diff --git a/api/v1alpha1/internaldns_access_test.go b/api/v1alpha1/internaldns_access_test.go index d4abda23..e67b646e 100644 --- a/api/v1alpha1/internaldns_access_test.go +++ b/api/v1alpha1/internaldns_access_test.go @@ -1,42 +1,24 @@ -// SPDX-License-Identifier: AGPL-3.0-only - package v1alpha1 import ( "encoding/json" "testing" - "time" ) -func TestAccessAuthorizationPreservesDeadlinePrecision(t *testing.T) { - const raw = `{"writerEpoch":1,"sequence":2,"validUntil":"2026-10-08T23:22:29.022357123Z"}` - var authorization DNSResolverAccessAuthorization - if err := json.Unmarshal([]byte(raw), &authorization); err != nil { +func TestResolverAccessHasNoExpiryOrClusterIdentity(t *testing.T) { + a := DNSResolverAccessAuthorization{WriterEpoch: 1, Sequence: 2} + b, err := json.Marshal(a) + if err != nil { t.Fatal(err) } - for i := 0; i < 3; i++ { - encoded, err := json.Marshal(authorization) - if err != nil { - t.Fatal(err) - } - if string(encoded) != raw { - t.Fatalf("authorization deadline changed on round trip: %s", encoded) - } - if err := json.Unmarshal(encoded, &authorization); err != nil { - t.Fatal(err) - } - } - if authorization.ValidUntil.Nanosecond() != 22357123 { - t.Fatal("authorization lost nanosecond precision") + if string(b) != `{"writerEpoch":1,"sequence":2}` { + t.Fatalf("unexpected access fence: %s", b) } - // The same instant in another time zone serializes to the same UTC fence. - authorization.ValidUntil.Time = authorization.ValidUntil.In(time.FixedZone("offset", 3600)) - encoded, err := json.Marshal(authorization) - if err != nil || string(encoded) != raw { - t.Fatalf("UTC normalization changed authorization: %s, %v", encoded, err) + b, err = json.Marshal(DNSProducerPrincipal{Subject: "system:serviceaccount:compute:dns-publisher"}) + if err != nil { + t.Fatal(err) } - encoded, err = json.Marshal(DNSResolverAccessAuthorization{}) - if err != nil || string(encoded) != `{"writerEpoch":0,"sequence":0,"validUntil":null}` { - t.Fatalf("zero value encoding changed field names or null deadline: %s, %v", encoded, err) + if string(b) != `{"subject":"system:serviceaccount:compute:dns-publisher"}` { + t.Fatalf("unexpected publisher identity: %s", b) } } diff --git a/api/v1alpha1/internaldns_binding_test.go b/api/v1alpha1/internaldns_binding_test.go index ce16887b..1f48e81d 100644 --- a/api/v1alpha1/internaldns_binding_test.go +++ b/api/v1alpha1/internaldns_binding_test.go @@ -5,8 +5,6 @@ package v1alpha1 import ( "encoding/json" "testing" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestResolverBindingUsesNestedContractAndIndependentCopies(t *testing.T) { @@ -14,7 +12,7 @@ func TestResolverBindingUsesNestedContractAndIndependentCopies(t *testing.T) { Source: DNSResolverBindingSource{ProjectUID: "project", ResolverContextRef: DNSObjectReference{Name: "context", UID: "context-uid"}, AccessBindingRef: DNSObjectReference{Name: "access", UID: "access-uid"}}, Placement: DNSResolverBindingPlacement{Region: "central", Shard: "shared"}, Configuration: DNSResolverBindingConfiguration{Generation: 1, Revision: 2, Listeners: DNSResolverBindingListeners{Node: DNSResolverBindingListener{Address: "fd53::1", Port: 53, Transports: []string{"UDP", "TCP"}}, Regional: DNSResolverBindingListener{Address: "fd54::1", Port: 53, Transports: []string{"UDP", "TCP"}}}, ZoneRefs: []DNSObjectReference{{Name: "private", UID: "zone-uid"}}}, - Authorization: DNSResolverAccessAuthorization{WriterEpoch: 3, Sequence: 4, ValidUntil: metav1.Now()}, + Authorization: DNSResolverAccessAuthorization{WriterEpoch: 3, Sequence: 4}, }} copy := original.DeepCopy() copy.Spec.Configuration.ZoneRefs[0].Name = "changed" diff --git a/api/v1alpha1/internaldns_project_types.go b/api/v1alpha1/internaldns_project_types.go index 6316ad84..f7f16461 100644 --- a/api/v1alpha1/internaldns_project_types.go +++ b/api/v1alpha1/internaldns_project_types.go @@ -3,9 +3,6 @@ package v1alpha1 import ( - "encoding/json" - "time" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ) @@ -115,24 +112,7 @@ type DNSResolverAccessAuthorization struct { // +kubebuilder:validation:Minimum=1 WriterEpoch int64 `json:"writerEpoch"` // +kubebuilder:validation:Minimum=1 - Sequence int64 `json:"sequence"` - ValidUntil metav1.Time `json:"validUntil"` -} - -// MarshalJSON preserves the source authorization deadline exactly. metav1.Time -// accepts fractional seconds on input but its default JSON encoder drops them, -// which would mutate a same-fence deadline when projecting an access binding. -func (a DNSResolverAccessAuthorization) MarshalJSON() ([]byte, error) { - var deadline *time.Time - if !a.ValidUntil.IsZero() { - instant := a.ValidUntil.UTC() - deadline = &instant - } - return json.Marshal(struct { - WriterEpoch int64 `json:"writerEpoch"` - Sequence int64 `json:"sequence"` - ValidUntil *time.Time `json:"validUntil"` - }{WriterEpoch: a.WriterEpoch, Sequence: a.Sequence, ValidUntil: deadline}) + Sequence int64 `json:"sequence"` } // +kubebuilder:validation:XValidation:rule="self.contextRef.name.size() > 0 && has(self.contextRef.uid) && self.contextRef.uid.size() > 0",message="contextRef requires name and UID" @@ -245,8 +225,6 @@ type DNSRegistrationList struct { } type DNSProducerPrincipal struct { - // +kubebuilder:validation:MinLength=1 - ClusterUID string `json:"clusterUID"` // +kubebuilder:validation:MinLength=1 Subject string `json:"subject"` } @@ -304,7 +282,6 @@ type DNSRecordContributionStatus struct { Sequence int64 `json:"sequence,omitempty"` Eligible bool `json:"eligible,omitempty"` Reason string `json:"reason,omitempty"` - ValidUntil *metav1.Time `json:"validUntil,omitempty"` PublishedRevision int64 `json:"publishedRevision,omitempty"` Conditions []metav1.Condition `json:"conditions,omitempty"` } diff --git a/api/v1alpha1/internaldns_service_types.go b/api/v1alpha1/internaldns_service_types.go index e97eb874..dc57e565 100644 --- a/api/v1alpha1/internaldns_service_types.go +++ b/api/v1alpha1/internaldns_service_types.go @@ -10,7 +10,6 @@ import ( // DNSResolverBinding is the DNS-service projection of a consumer context and access binding. // +kubebuilder:validation:XValidation:rule="self.configuration.generation >= oldSelf.configuration.generation && self.configuration.revision >= oldSelf.configuration.revision",message="configuration fences must not decrease" // +kubebuilder:validation:XValidation:rule="self.authorization.writerEpoch > oldSelf.authorization.writerEpoch || (self.authorization.writerEpoch == oldSelf.authorization.writerEpoch && self.authorization.sequence >= oldSelf.authorization.sequence)",message="authorization fences must not decrease" -// +kubebuilder:validation:XValidation:rule="self.authorization.writerEpoch != oldSelf.authorization.writerEpoch || self.authorization.sequence != oldSelf.authorization.sequence || self.authorization.validUntil == oldSelf.authorization.validUntil",message="changing an authorization deadline requires a new fence" // +kubebuilder:validation:XValidation:rule="self.configuration.zoneRefs == oldSelf.configuration.zoneRefs || (self.configuration.generation > oldSelf.configuration.generation && self.configuration.revision > oldSelf.configuration.revision)",message="zone membership changes require a new cache generation and configuration revision" // +kubebuilder:validation:XValidation:rule="(has(self.tombstone) && self.tombstone) == (has(oldSelf.tombstone) && oldSelf.tombstone) || self.configuration.revision > oldSelf.configuration.revision",message="changing withdrawal state requires a new configuration revision" // +kubebuilder:validation:XValidation:rule="!has(oldSelf.tombstone) || !oldSelf.tombstone || (has(self.tombstone) && self.tombstone) || self.authorization.writerEpoch > oldSelf.authorization.writerEpoch || (self.authorization.writerEpoch == oldSelf.authorization.writerEpoch && self.authorization.sequence > oldSelf.authorization.sequence)",message="reactivating withdrawn access requires a new source authorization fence" @@ -153,13 +152,12 @@ type DNSRegistrationFence struct { } // DNSContributionFence persists an authenticated high-water sequence even -// while its observation is unhealthy, rejected, or expired. +// while its observation is unhealthy or withdrawn. type DNSContributionFence struct { - UID types.UID `json:"uid"` - GrantUID types.UID `json:"grantUID"` - Epoch int64 `json:"epoch"` - Sequence int64 `json:"sequence"` - ValidUntil metav1.Time `json:"validUntil,omitempty"` + UID types.UID `json:"uid"` + GrantUID types.UID `json:"grantUID"` + Epoch int64 `json:"epoch"` + Sequence int64 `json:"sequence"` } type DNSPublicationManifestStatus struct { diff --git a/api/v1alpha1/internaldns_validation_envtest_test.go b/api/v1alpha1/internaldns_validation_envtest_test.go index 45bb76d7..a4673fa2 100644 --- a/api/v1alpha1/internaldns_validation_envtest_test.go +++ b/api/v1alpha1/internaldns_validation_envtest_test.go @@ -7,7 +7,6 @@ import ( "os" "path/filepath" "testing" - "time" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -57,7 +56,7 @@ func TestPrivateDNSValidationAtAPIServer(t *testing.T) { } newAccess := func(name string) *DNSResolverAccessBinding { return &DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "validation"}, Spec: DNSResolverAccessBindingSpec{ - ContextRef: DNSObjectReference{Name: "context", UID: "context-uid"}, Region: "central", QueryIdentity: DNSResolverQueryIdentity{Type: DNSResolverQueryIdentityDestinationAddress, Value: "fd53::1"}, Port: 53, Transports: []string{"UDP", "TCP"}, Authorization: DNSResolverAccessAuthorization{WriterEpoch: 1, Sequence: 1, ValidUntil: metav1.NewTime(time.Now().Add(time.Minute))}, + ContextRef: DNSObjectReference{Name: "context", UID: "context-uid"}, Region: "central", QueryIdentity: DNSResolverQueryIdentity{Type: DNSResolverQueryIdentityDestinationAddress, Value: "fd53::1"}, Port: 53, Transports: []string{"UDP", "TCP"}, Authorization: DNSResolverAccessAuthorization{WriterEpoch: 1, Sequence: 1}, }} } for _, tc := range []struct { @@ -86,9 +85,6 @@ func TestPrivateDNSValidationAtAPIServer(t *testing.T) { {"change-region", func(a *DNSResolverAccessBinding) { a.Spec.Region = "east" }}, {"change-address", func(a *DNSResolverAccessBinding) { a.Spec.QueryIdentity.Value = "fd53::2" }}, {"change-epoch", func(a *DNSResolverAccessBinding) { a.Spec.Authorization.WriterEpoch++ }}, - {"extend-without-sequence", func(a *DNSResolverAccessBinding) { - a.Spec.Authorization.ValidUntil = metav1.NewTime(a.Spec.Authorization.ValidUntil.Add(time.Minute)) - }}, {"decrease-sequence", func(a *DNSResolverAccessBinding) { a.Spec.Authorization.Sequence = 1 }}, } { t.Run(tc.name, func(t *testing.T) { @@ -101,13 +97,12 @@ func TestPrivateDNSValidationAtAPIServer(t *testing.T) { requireInvalid(t, cl.Update(ctx, a)) }) } - t.Run("renewal-and-status", func(t *testing.T) { + t.Run("sequence-and-status", func(t *testing.T) { a := newAccess("renewal") if err := cl.Create(ctx, a); err != nil { t.Fatal(err) } a.Spec.Authorization.Sequence++ - a.Spec.Authorization.ValidUntil = metav1.NewTime(a.Spec.Authorization.ValidUntil.Add(time.Minute)) if err := cl.Update(ctx, a); err != nil { t.Fatal(err) } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 36037304..ea2cbae6 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -132,7 +132,6 @@ func (in *DNSApplyAcknowledgement) DeepCopy() *DNSApplyAcknowledgement { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DNSContributionFence) DeepCopyInto(out *DNSContributionFence) { *out = *in - in.ValidUntil.DeepCopyInto(&out.ValidUntil) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DNSContributionFence. @@ -703,9 +702,7 @@ func (in *DNSPublicationManifestSpec) DeepCopyInto(out *DNSPublicationManifestSp if in.ContributionFences != nil { in, out := &in.ContributionFences, &out.ContributionFences *out = make([]DNSContributionFence, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } + copy(*out, *in) } if in.RegistrationFences != nil { in, out := &in.RegistrationFences, &out.RegistrationFences @@ -937,10 +934,6 @@ func (in *DNSRecordContributionSpec) DeepCopy() *DNSRecordContributionSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DNSRecordContributionStatus) DeepCopyInto(out *DNSRecordContributionStatus) { *out = *in - if in.ValidUntil != nil { - in, out := &in.ValidUntil, &out.ValidUntil - *out = (*in).DeepCopy() - } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]v1.Condition, len(*in)) @@ -1266,7 +1259,6 @@ func (in *DNSResolvedAdditionalNameRule) DeepCopy() *DNSResolvedAdditionalNameRu // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DNSResolverAccessAuthorization) DeepCopyInto(out *DNSResolverAccessAuthorization) { *out = *in - in.ValidUntil.DeepCopyInto(&out.ValidUntil) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DNSResolverAccessAuthorization. @@ -1348,7 +1340,7 @@ func (in *DNSResolverAccessBindingSpec) DeepCopyInto(out *DNSResolverAccessBindi *out = make([]string, len(*in)) copy(*out, *in) } - in.Authorization.DeepCopyInto(&out.Authorization) + out.Authorization = in.Authorization } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DNSResolverAccessBindingSpec. @@ -1543,7 +1535,7 @@ func (in *DNSResolverBindingSpec) DeepCopyInto(out *DNSResolverBindingSpec) { out.Source = in.Source out.Placement = in.Placement in.Configuration.DeepCopyInto(&out.Configuration) - in.Authorization.DeepCopyInto(&out.Authorization) + out.Authorization = in.Authorization } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DNSResolverBindingSpec. diff --git a/config/crd/bases/dns.networking.miloapis.com_dnscontributiongrants.yaml b/config/crd/bases/dns.networking.miloapis.com_dnscontributiongrants.yaml index 46a238eb..3614543d 100644 --- a/config/crd/bases/dns.networking.miloapis.com_dnscontributiongrants.yaml +++ b/config/crd/bases/dns.networking.miloapis.com_dnscontributiongrants.yaml @@ -44,14 +44,10 @@ spec: type: array principal: properties: - clusterUID: - minLength: 1 - type: string subject: minLength: 1 type: string required: - - clusterUID - subject type: object producerID: diff --git a/config/crd/bases/dns.networking.miloapis.com_dnspublicationmanifests.yaml b/config/crd/bases/dns.networking.miloapis.com_dnspublicationmanifests.yaml index 1a8845d5..018209d8 100644 --- a/config/crd/bases/dns.networking.miloapis.com_dnspublicationmanifests.yaml +++ b/config/crd/bases/dns.networking.miloapis.com_dnspublicationmanifests.yaml @@ -71,7 +71,7 @@ spec: items: description: |- DNSContributionFence persists an authenticated high-water sequence even - while its observation is unhealthy, rejected, or expired. + while its observation is unhealthy or withdrawn. properties: epoch: format: int64 @@ -91,9 +91,6 @@ spec: don't ONLY use UUIDs, this is an alias to string. Being a type captures intent and helps make sure that UIDs and names do not get conflated. type: string - validUntil: - format: date-time - type: string required: - epoch - grantUID diff --git a/config/crd/bases/dns.networking.miloapis.com_dnsrecordcontributions.yaml b/config/crd/bases/dns.networking.miloapis.com_dnsrecordcontributions.yaml index 163f509d..0f38938b 100644 --- a/config/crd/bases/dns.networking.miloapis.com_dnsrecordcontributions.yaml +++ b/config/crd/bases/dns.networking.miloapis.com_dnsrecordcontributions.yaml @@ -412,9 +412,6 @@ spec: sequence: format: int64 type: integer - validUntil: - format: date-time - type: string writerEpoch: format: int64 type: integer diff --git a/config/crd/bases/dns.networking.miloapis.com_dnsresolveraccessbindings.yaml b/config/crd/bases/dns.networking.miloapis.com_dnsresolveraccessbindings.yaml index 99cd47bf..f743b3d3 100644 --- a/config/crd/bases/dns.networking.miloapis.com_dnsresolveraccessbindings.yaml +++ b/config/crd/bases/dns.networking.miloapis.com_dnsresolveraccessbindings.yaml @@ -44,16 +44,12 @@ spec: format: int64 minimum: 1 type: integer - validUntil: - format: date-time - type: string writerEpoch: format: int64 minimum: 1 type: integer required: - sequence - - validUntil - writerEpoch type: object contextRef: diff --git a/config/crd/bases/dns.networking.miloapis.com_dnsresolverbindings.yaml b/config/crd/bases/dns.networking.miloapis.com_dnsresolverbindings.yaml index 4875ae95..4d78b627 100644 --- a/config/crd/bases/dns.networking.miloapis.com_dnsresolverbindings.yaml +++ b/config/crd/bases/dns.networking.miloapis.com_dnsresolverbindings.yaml @@ -45,16 +45,12 @@ spec: format: int64 minimum: 1 type: integer - validUntil: - format: date-time - type: string writerEpoch: format: int64 minimum: 1 type: integer required: - sequence - - validUntil - writerEpoch type: object configuration: @@ -263,10 +259,6 @@ spec: rule: self.authorization.writerEpoch > oldSelf.authorization.writerEpoch || (self.authorization.writerEpoch == oldSelf.authorization.writerEpoch && self.authorization.sequence >= oldSelf.authorization.sequence) - - message: changing an authorization deadline requires a new fence - rule: self.authorization.writerEpoch != oldSelf.authorization.writerEpoch - || self.authorization.sequence != oldSelf.authorization.sequence || - self.authorization.validUntil == oldSelf.authorization.validUntil - message: zone membership changes require a new cache generation and configuration revision rule: self.configuration.zoneRefs == oldSelf.configuration.zoneRefs diff --git a/config/internal-dns/README.md b/config/internal-dns/README.md index a1573c32..0b3ecd42 100644 --- a/config/internal-dns/README.md +++ b/config/internal-dns/README.md @@ -35,11 +35,11 @@ projection or publication for healthy sources. DNS always consumes `DNSResolverContext` and `DNSResolverAccessBinding` from the project API. It does not watch a networking API. A trusted integration principal bound to `internal-dns-vpc-integration` creates a context and writes -short-lived regional access intent. RBAC separates integration, grant issuer, +regional access intent. RBAC separates integration, grant issuer, product publisher, and DNS controller permissions. Product publishers cannot issue grants or modify resolver authority. -The context UID, access UID, authorization writer epoch, sequence, and original -deadline are carried into the internal `DNSResolverBinding`. +The context UID, access UID, writer epoch, and sequence are carried into the +internal `DNSResolverBinding`. Access persists until an explicit withdrawal. The control plane allocates a managed namespace when the context requests one. Compute discovers the context status and publishes through @@ -72,7 +72,7 @@ TCP. `verifier.publicName` can override this probe. BIND members need a writable Private packet caching stays disabled in dnsdist. Each BIND view has its own positive and negative cache. `max-ncache-ttl` defaults to five seconds and is bounded at 60 seconds. Public recursion retains DNSSEC validation; each view -adds validation exceptions only for its private apexes. Unknown or expired +adds validation exceptions only for its private apexes. Unknown or withdrawn destinations are rejected before cache lookup. Regional publication proof connects directly to that member's BIND listener, @@ -81,7 +81,7 @@ UDP and TCP, and verifies the installed SOA serial and fingerprint. Restrict `proxyPeers` to regional dnsdist and the member's local publication agent network. -## Transactions, health, and expiry +## Transactions, health, and outage recovery An attached zone that has no active publication gates its context until the configuration is complete. Pending contexts deny queries, recursion, and cache @@ -96,19 +96,20 @@ atomic only after every candidate validates. Reload wrappers must wait for the listener and backend pool before returning so the immediate DNS proof observes the activated state. -Publication ACKs are leases. A regional frontend and node may keep a context -available while at least one assigned regional member has a fresh current ACK; -the project-facing Ready condition can still report degraded membership. If no -member remains, the frontend gates the context. A regional member that cannot -apply or prove a publication fails closed and stops renewing its watchdog. +Publication acknowledgements report verified configuration versions. The fleet +retains accepted proofs during control plane or broker outages and continues +using local DNS probes and backend health checks. Status can report degraded +update delivery without withdrawing an applied resolver binding. -Agents preserve original record and access deadlines in their checkpoints. -They remove expired records, rewrite regional zone files, and flush affected -BIND names locally even while NATS or a source API is unavailable. Replay and -restart cannot extend a deadline or regress writer/configuration fences. +Records and access bindings have no expiry deadline. Product services withdraw +endpoints through explicit health updates or deletion. When delivery stops, DNS +retains the last reported endpoint state. Agents restore applied configuration +from persistent checkpoints before waiting for messaging connectivity. New +nodes require a complete configuration before accepting queries. Replay cannot +regress writer or configuration versions or restore a withdrawn lifetime. -The watchdog lease is at most five seconds. It is renewed only after local -expiry enforcement, a complete validated transaction, and required DNS proofs. +The local watchdog lease is at most five seconds. It is renewed only after a +complete validated transaction and required DNS proofs. The gate must stop or firewall the local private listener without depending on Kubernetes or NATS. Validation, reload, readiness wait, and proof must fit inside the watchdog budget; reduce shard size or improve local operations when @@ -142,8 +143,8 @@ raising those bounds. NATS uses TLS with a verified server name, client certificate, and one scoped identity per process. Control-plane credentials publish serving/publication subjects and consume ACKs. Each serving member can consume its assigned durable -subjects and publish only its own hashed ACK subject. Broker partitions do not -extend record or authorization deadlines. +subjects and publish only its own hashed ACK subject. Broker partitions delay +updates while the fleet continues serving its applied configuration. `fleet.example.yaml` provides node, regional frontend, and regional BIND JSON configurations plus one Deployment template. Instantiate the template for each @@ -157,12 +158,13 @@ The project API server authorizes DNS resource and status operations. Bind the roles in `rbac.yaml` to service principals in each project namespace. Customer access uses the protected resources and DNS roles in `config/iam`. -The webhook validates live references and lease limits. It also verifies that -changes to a publisher's contribution and freshness observation use that -publisher's current grant. Publication-result updates cannot renew a publisher's +The webhook validates live references and verifies that changes to a publisher's +contribution and eligibility observation use that publisher's current grant. +Grants identify the authenticated subject within the project API context. +Publication-result updates cannot change a publisher's observation. Resource permissions do not depend on webhook principal lists. -The supported internal envelope version is `v1alpha3` and the serving checkpoint -format is 4. Unsupported versions fail closed. Deploy matching controller and +The supported internal envelope version is `v1alpha4` and the serving checkpoint +format is 5. Unsupported versions fail closed. Deploy matching controller and fleet versions with fresh internal DNS transport state and checkpoints. Preserve any existing state for recovery. The deployment requires a coordinated rollout; it does not support mixed versions or automatic state conversion. diff --git a/config/internal-dns/control-plane.example.yaml b/config/internal-dns/control-plane.example.yaml index 9dd5065c..3f257fbf 100644 --- a/config/internal-dns/control-plane.example.yaml +++ b/config/internal-dns/control-plane.example.yaml @@ -14,7 +14,6 @@ data: { "name": "example-project", "projectUID": "00000000-0000-0000-0000-000000000001", - "sourceClusterUID": "00000000-0000-0000-0000-000000000010", "namespace": "example-project", "kubeAPI": {"qps":20,"burst":40,"requestTimeoutSeconds":10} } @@ -62,9 +61,7 @@ data: "admission": { "enabled": true, "port": 9443, - "certDir": "/var/run/internal-dns/admission", - "maxContributionLeaseSeconds": 90, - "maxAccessLeaseSeconds": 300 + "certDir": "/var/run/internal-dns/admission" } } --- diff --git a/config/internal-dns/fleet.example.yaml b/config/internal-dns/fleet.example.yaml index 66c96aa9..b5a43141 100644 --- a/config/internal-dns/fleet.example.yaml +++ b/config/internal-dns/fleet.example.yaml @@ -13,7 +13,7 @@ data: "region":"us-central1","shard":"shared-0","memberID":"node-us-central1-0","replicaID":"node-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"resolver", "render":{"role":"node","nodeDNSDist":{"path":"/etc/internal-dns/node-dnsdist.conf","listenAddress":"[::]:53","acls":["fd71:100::/64"],"poolName":"node_shared","backends":[{"memberID":"node-bind-0","address":"fd70:10::10","port":5300}]},"nodeBIND":{"path":"/etc/internal-dns/node-bind.conf","listenAddress":"fd70:10::10","port":5300,"proxyPeers":["fd70:10::20"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-dnsdist","args":["node","{stageDir}/node-dnsdist.conf"],"whenChanged":["/etc/internal-dns/node-dnsdist.conf"]},{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["node","{stageDir}/node-bind.conf"],"whenChanged":["/etc/internal-dns/node-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["node"],"whenChanged":["/etc/internal-dns/node-bind.conf"]},{"path":"/usr/local/sbin/internal-dns-reload-dnsdist","args":["node"],"whenChanged":["/etc/internal-dns/node-dnsdist.conf"]}]}, - "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["node","{name}"]}],"expiryInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"localServingLease":90000000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"resolverDNSProbe":{"timeout":3000000000},"resolverVerifyConcurrency":16 + "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["node","{name}"]}],"reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"resolverDNSProbe":{"timeout":3000000000},"resolverVerifyConcurrency":16 }, "watchdog":{"leasePath":"/run/internal-dns/lease.json","memberID":"node-us-central1-0","replicaID":"node-us-central1-0","maxLeaseSeconds":5,"startupGraceSeconds":0,"intervalMillis":250,"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["node"]}]} } @@ -30,7 +30,7 @@ data: "region":"us-central1","shard":"shared-0","memberID":"regional-front-us-central1-0","replicaID":"regional-front-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"resolver", "render":{"role":"regional-dnsdist","clusterDNSDist":{"path":"/etc/internal-dns/cluster-dnsdist.conf","listenAddress":"[::]:53","acls":["fd71:200::/64"],"poolName":"regional_shared","backends":[{"memberID":"regional-bind-us-central1-0","address":"fd70:20::10","port":5300},{"memberID":"regional-bind-us-central1-1","address":"fd70:20::11","port":5300}]}}, "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-dnsdist","args":["regional","{stageDir}/cluster-dnsdist.conf"],"whenChanged":["/etc/internal-dns/cluster-dnsdist.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-dnsdist","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-dnsdist.conf"]}]}, - "expiryInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"localServingLease":90000000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"resolverDNSProbe":{"timeout":3000000000},"resolverVerifyConcurrency":16 + "reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"resolverDNSProbe":{"timeout":3000000000},"resolverVerifyConcurrency":16 }, "watchdog":{"leasePath":"/run/internal-dns/lease.json","memberID":"regional-front-us-central1-0","replicaID":"regional-front-us-central1-0","maxLeaseSeconds":5,"startupGraceSeconds":0,"intervalMillis":250,"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["regional-front"]}]} } @@ -47,7 +47,7 @@ data: "region":"us-central1","shard":"shared-0","memberID":"regional-bind-us-central1-0","replicaID":"regional-bind-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"combined", "render":{"role":"regional-bind","clusterBIND":{"path":"/etc/internal-dns/cluster-bind.conf","listenAddress":"fd70:20::10","port":5300,"proxyPeers":["fd70:20::20","fd70:20::30"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["regional","-z","{stageDir}/cluster-bind.conf"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}]}, - "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["regional","{name}"]}],"expiryInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"publicationDNSProbe":{"server":"[fd70:20::10]:5300","timeout":3000000000} + "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["regional","{name}"]}],"reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"publicationDNSProbe":{"server":"[fd70:20::10]:5300","timeout":3000000000} }, "watchdog":{"leasePath":"/run/internal-dns/lease.json","memberID":"regional-bind-us-central1-0","replicaID":"regional-bind-us-central1-0","maxLeaseSeconds":5,"startupGraceSeconds":0,"intervalMillis":250,"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["regional-bind"]}]} } @@ -67,7 +67,7 @@ data: "region":"us-central1","shard":"shared-0","memberID":"regional-bind-us-central1-1","replicaID":"regional-bind-us-central1-1","stateDir":"/var/lib/internal-dns","mode":"combined", "render":{"role":"regional-bind","clusterBIND":{"path":"/etc/internal-dns/cluster-bind.conf","listenAddress":"fd70:20::11","port":5300,"proxyPeers":["fd70:20::20","fd70:20::31"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["regional","-z","{stageDir}/cluster-bind.conf"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}]}, - "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["regional","{name}"]}],"expiryInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"publicationDNSProbe":{"server":"[fd70:20::11]:5300","timeout":3000000000} + "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["regional","{name}"]}],"reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"publicationDNSProbe":{"server":"[fd70:20::11]:5300","timeout":3000000000} }, "watchdog":{"leasePath":"/run/internal-dns/lease.json","memberID":"regional-bind-us-central1-1","replicaID":"regional-bind-us-central1-1","maxLeaseSeconds":5,"startupGraceSeconds":0,"intervalMillis":250,"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["regional-bind"]}]} } diff --git a/internal/internaldns/admission/handler.go b/internal/internaldns/admission/handler.go index 461a3a97..8f062898 100644 --- a/internal/internaldns/admission/handler.go +++ b/internal/internaldns/admission/handler.go @@ -11,7 +11,6 @@ import ( "fmt" "net/http" "reflect" - "time" dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" "go.miloapis.com/dns-operator/internal/internaldns/model" @@ -28,17 +27,11 @@ const contributionResource = "dnsrecordcontributions" const Path = "/validate-internal-dns" -// Handler uses an uncached project client for grant and registration lookups. -// TrustedClusterUID is supplied by the server configuration, never by a record -// publisher. For project-aware deployments, ClientForRequest must resolve the -// authenticated project context and return its immutable source cluster UID. +// Handler uses an uncached client selected by authenticated project context. type Handler struct { - Client client.Client - TrustedClusterUID string - MaxContributionLease time.Duration - MaxAccessLease time.Duration - Now func() time.Time - ClientForRequest func(context.Context, kubeadmission.Request) (client.Client, string, error) + Client client.Client + + ClientForRequest func(context.Context, kubeadmission.Request) (client.Client, error) } var _ kubeadmission.Handler = (*Handler)(nil) @@ -56,14 +49,14 @@ func (h *Handler) Handle(ctx context.Context, req kubeadmission.Request) kubeadm } switch req.Resource.Resource { case "dnsregistrations", contributionResource, "dnscontributiongrants", "dnszoneassociations", "dnsnamingpolicies", "dnsresolveraccessbindings": - // These resources require live references or bounded freshness checks. + // These resources require live references. default: return kubeadmission.Allowed("API server authorization and schema validation apply") } - cl, clusterUID := h.Client, h.TrustedClusterUID + cl := h.Client if h.ClientForRequest != nil { var err error - cl, clusterUID, err = h.ClientForRequest(ctx, req) + cl, err = h.ClientForRequest(ctx, req) if err != nil { return kubeadmission.Errored(http.StatusServiceUnavailable, err) } @@ -78,7 +71,7 @@ func (h *Handler) Handle(ctx context.Context, req kubeadmission.Request) kubeadm case "dnsregistrations": err = h.registration(ctx, cl, req) case contributionResource: - err = h.contribution(ctx, cl, clusterUID, req) + err = h.contribution(ctx, cl, req) case "dnscontributiongrants": err = h.grant(ctx, cl, req) case "dnszoneassociations": @@ -99,7 +92,6 @@ func (h *Handler) resolverAccess(ctx context.Context, cl client.Client, req kube if err := decode(req, ¤t); err != nil { return err } - a := current.Spec.Authorization var resolverContext dnsv1alpha1.DNSResolverContext if err := cl.Get(ctx, client.ObjectKey{Namespace: req.Namespace, Name: current.Spec.ContextRef.Name}, &resolverContext); err != nil { return fmt.Errorf("resolver context lookup: %w", err) @@ -114,24 +106,10 @@ func (h *Handler) resolverAccess(ctx context.Context, cl client.Client, req kube if ready == nil || ready.Status != metav1.ConditionTrue || ready.ObservedGeneration != resolverContext.Generation { return fmt.Errorf("resolver context is not ready for its current generation") } - max := h.MaxAccessLease - if max <= 0 { - max = 5 * time.Minute - } - now := time.Now() - if h.Now != nil { - now = h.Now() - } - if !a.ValidUntil.After(now) { - return fmt.Errorf("access deadline is already expired") - } - if a.ValidUntil.After(now.Add(max)) { - return fmt.Errorf("access deadline exceeds platform lease limit") - } return nil } -func (h *Handler) contribution(ctx context.Context, cl client.Client, clusterUID string, req kubeadmission.Request) error { +func (h *Handler) contribution(ctx context.Context, cl client.Client, req kubeadmission.Request) error { var c, old dnsv1alpha1.DNSRecordContribution if err := decode(req, &c); err != nil { return err @@ -149,8 +127,8 @@ func (h *Handler) contribution(ctx context.Context, cl client.Client, clusterUID if reflect.DeepEqual(previous, current) { return nil } - // Binding an issued epoch cannot introduce or renew endpoint eligibility. - if previous.WriterEpoch == 0 && previous.ObservedGeneration == 0 && previous.Sequence == 0 && !previous.Eligible && previous.ValidUntil == nil && current.WriterEpoch > 0 { + // Binding an issued epoch cannot introduce endpoint eligibility. + if previous.WriterEpoch == 0 && previous.ObservedGeneration == 0 && previous.Sequence == 0 && !previous.Eligible && current.WriterEpoch > 0 { current.WriterEpoch = 0 bindEpoch = reflect.DeepEqual(previous, current) } @@ -162,7 +140,7 @@ func (h *Handler) contribution(ctx context.Context, cl client.Client, clusterUID if g.UID != c.Spec.GrantRef.UID || !g.DeletionTimestamp.IsZero() { return fmt.Errorf("grant lifetime is not current") } - if clusterUID == "" || g.Spec.Principal.ClusterUID != clusterUID || (!bindEpoch && g.Spec.Principal.Subject != req.UserInfo.Username) { + if !bindEpoch && g.Spec.Principal.Subject != req.UserInfo.Username { return fmt.Errorf("caller does not match the grant's authenticated producer principal") } if g.Spec.RegistrationRef.UID != c.Spec.RegistrationRef.UID || g.Spec.RegistrationRef.Name != c.Spec.RegistrationRef.Name { @@ -214,23 +192,6 @@ func (h *Handler) contribution(ctx context.Context, cl client.Client, clusterUID if c.Status.ObservedGeneration != c.Generation { return fmt.Errorf("producer observation must match the contribution generation") } - if c.Status.ValidUntil == nil { - return fmt.Errorf("producer observation requires an original freshness deadline") - } - now := time.Now() - if h.Now != nil { - now = h.Now() - } - maxLease := h.MaxContributionLease - if maxLease <= 0 { - maxLease = 90 * time.Second - } - if c.Status.ValidUntil.After(now.Add(maxLease)) { - return fmt.Errorf("freshness deadline exceeds platform lease limit") - } - if c.Status.Eligible && !c.Status.ValidUntil.After(now) { - return fmt.Errorf("eligible observation is already expired") - } } else if req.Operation != admissionv1.Delete && !reflect.DeepEqual(old.Status, c.Status) && req.Operation != admissionv1.Create { return fmt.Errorf("producer observations must use the status subresource") } diff --git a/internal/internaldns/admission/handler_test.go b/internal/internaldns/admission/handler_test.go index ee606c87..4d498004 100644 --- a/internal/internaldns/admission/handler_test.go +++ b/internal/internaldns/admission/handler_test.go @@ -21,7 +21,6 @@ import ( ) func TestContributionAuthenticatedWriter(t *testing.T) { - now := time.Date(2026, 10, 6, 18, 0, 0, 0, time.UTC) producer := "system:serviceaccount:compute-system:dns-publisher" registration := &dnsv1alpha1.DNSRegistration{ ObjectMeta: metav1.ObjectMeta{Name: "api", Namespace: "project", UID: "reg-uid", Generation: 3}, @@ -31,7 +30,7 @@ func TestContributionAuthenticatedWriter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "compute", Namespace: "project", UID: "grant-uid"}, Spec: dnsv1alpha1.DNSContributionGrantSpec{ RegistrationRef: dnsv1alpha1.DNSObjectReference{Name: "api", UID: "reg-uid"}, - Principal: dnsv1alpha1.DNSProducerPrincipal{ClusterUID: "cluster-uid", Subject: producer}, + Principal: dnsv1alpha1.DNSProducerPrincipal{Subject: producer}, RecordTypes: []dnsv1alpha1.RRType{"AAAA"}, }, Status: dnsv1alpha1.DNSContributionGrantStatus{ActiveWriterEpoch: 8, ObservedRegistrationGeneration: 3}, @@ -42,13 +41,13 @@ func TestContributionAuthenticatedWriter(t *testing.T) { RegistrationRef: dnsv1alpha1.DNSObjectReference{Name: "api", UID: "reg-uid", Generation: 3}, GrantRef: dnsv1alpha1.DNSObjectReference{Name: "compute", UID: "grant-uid"}, }, - Status: dnsv1alpha1.DNSRecordContributionStatus{ObservedGeneration: 2, WriterEpoch: 8, Sequence: 18, Eligible: true, ValidUntil: ptrTime(now.Add(30 * time.Second))}, + Status: dnsv1alpha1.DNSRecordContributionStatus{ObservedGeneration: 2, WriterEpoch: 8, Sequence: 18, Eligible: true}, } scheme := runtime.NewScheme() if err := dnsv1alpha1.AddToScheme(scheme); err != nil { t.Fatal(err) } - h := Handler{Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(registration, grant, &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "private", Namespace: "project"}, Spec: dnsv1alpha1.DNSZoneSpec{DomainName: "corp.internal", Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}}).Build(), TrustedClusterUID: "cluster-uid", Now: func() time.Time { return now }} + h := Handler{Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(registration, grant, &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "private", Namespace: "project"}, Spec: dnsv1alpha1.DNSZoneSpec{DomainName: "corp.internal", Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}}).Build()} cases := []struct { name string user string @@ -70,8 +69,6 @@ func TestContributionAuthenticatedWriter(t *testing.T) { {"forged grant UID", producer, func(c *dnsv1alpha1.DNSRecordContribution) { c.Spec.GrantRef.UID = "other-grant" }, false}, {"retired registration UID", producer, func(c *dnsv1alpha1.DNSRecordContribution) { c.Spec.RegistrationRef.UID = "old-reg" }, false}, {"publication fields", producer, func(c *dnsv1alpha1.DNSRecordContribution) { c.Status.PublishedRevision = 1000 }, false}, - {"overlong lease", producer, func(c *dnsv1alpha1.DNSRecordContribution) { c.Status.ValidUntil = ptrTime(now.Add(time.Hour)) }, false}, - {"expired eligible lease", producer, func(c *dnsv1alpha1.DNSRecordContribution) { c.Status.ValidUntil = ptrTime(now.Add(-time.Second)) }, false}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -117,8 +114,6 @@ func TestUnavailableProjectContextFailsClosed(t *testing.T) { } } -func ptrTime(t time.Time) *metav1.Time { v := metav1.NewTime(t); return &v } - func request(resource, username string, obj, old any) kubeadmission.Request { raw, _ := json.Marshal(obj) var oldRaw []byte @@ -160,7 +155,7 @@ func TestRetiredProductIntentCanBeDeletedWithoutLiveGrant(t *testing.T) { } } -func TestResolverAccessRequiresCurrentContextAndBoundedLease(t *testing.T) { +func TestResolverAccessRequiresCurrentContext(t *testing.T) { now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) scheme := runtime.NewScheme() if err := dnsv1alpha1.AddToScheme(scheme); err != nil { @@ -168,20 +163,13 @@ func TestResolverAccessRequiresCurrentContextAndBoundedLease(t *testing.T) { } resolverContext := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "ctx", Namespace: "project", UID: "ctx-uid", Generation: 1}, Status: dnsv1alpha1.DNSResolverContextStatus{AccessWriterEpoch: 3}} setAdmissionCondition(&resolverContext.Status.Conditions, "Ready", metav1.ConditionTrue, 1, now) - h := Handler{Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(resolverContext).Build(), MaxAccessLease: 5 * time.Minute, Now: func() time.Time { return now }} - old := &dnsv1alpha1.DNSResolverAccessBinding{Spec: dnsv1alpha1.DNSResolverAccessBindingSpec{ContextRef: dnsv1alpha1.DNSObjectReference{Name: "ctx", UID: "ctx-uid"}, Region: "central", QueryIdentity: dnsv1alpha1.DNSResolverQueryIdentity{Type: dnsv1alpha1.DNSResolverQueryIdentityDestinationAddress, Value: "fd70:100::10"}, Port: 53, Transports: []string{"UDP", "TCP"}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 3, Sequence: 9, ValidUntil: metav1.NewTime(now.Add(2 * time.Minute))}}} + h := Handler{Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(resolverContext).Build()} + old := &dnsv1alpha1.DNSResolverAccessBinding{Spec: dnsv1alpha1.DNSResolverAccessBindingSpec{ContextRef: dnsv1alpha1.DNSObjectReference{Name: "ctx", UID: "ctx-uid"}, Region: "central", QueryIdentity: dnsv1alpha1.DNSResolverQueryIdentity{Type: dnsv1alpha1.DNSResolverQueryIdentityDestinationAddress, Value: "fd70:100::10"}, Port: 53, Transports: []string{"UDP", "TCP"}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 3, Sequence: 9}}} current := old.DeepCopy() current.Spec.Authorization.Sequence = 10 - current.Spec.Authorization.ValidUntil = metav1.NewTime(now.Add(3 * time.Minute)) req := request("dnsresolveraccessbindings", "system:serviceaccount:network-system:integration", current, old) if res := h.Handle(context.Background(), req); !res.Allowed { - t.Fatalf("fresh integration renewal denied: %v", res.Result) - } - overlong := current.DeepCopy() - overlong.Spec.Authorization.ValidUntil = metav1.NewTime(now.Add(6 * time.Minute)) - req = request("dnsresolveraccessbindings", "system:serviceaccount:network-system:integration", overlong, old) - if h.Handle(context.Background(), req).Allowed { - t.Fatal("overlong access lease accepted") + t.Fatalf("current integration update denied: %v", res.Result) } forged := current.DeepCopy() forged.Spec.ContextRef.UID = "replacement" @@ -286,9 +274,9 @@ func TestRegistrationRequiresCurrentZone(t *testing.T) { } } -func TestPublicationStatusDoesNotRenewPublisherObservation(t *testing.T) { +func TestPublicationStatusDoesNotChangePublisherObservation(t *testing.T) { h := Handler{Client: fake.NewClientBuilder().WithScheme(runtime.NewScheme()).Build()} - old := &dnsv1alpha1.DNSRecordContribution{Status: dnsv1alpha1.DNSRecordContributionStatus{WriterEpoch: 4, Sequence: 10, Eligible: true, ValidUntil: ptrTime(time.Now().Add(-time.Minute))}} + old := &dnsv1alpha1.DNSRecordContribution{Status: dnsv1alpha1.DNSRecordContributionStatus{WriterEpoch: 4, Sequence: 10, Eligible: true}} current := old.DeepCopy() current.Status.PublishedRevision = 12 current.Status.Conditions = []metav1.Condition{{Type: "Published", Status: metav1.ConditionTrue}} @@ -297,10 +285,10 @@ func TestPublicationStatusDoesNotRenewPublisherObservation(t *testing.T) { if result := h.Handle(context.Background(), req); !result.Allowed { t.Fatalf("publication result needs no producer grant: %v", result.Result) } - current.Status.ValidUntil = ptrTime(time.Now().Add(time.Minute)) + current.Status.Sequence++ req = request("dnsrecordcontributions", "rbac-authorized-controller", current, old) req.SubResource = "status" if h.Handle(context.Background(), req).Allowed { - t.Fatal("publication update renewed producer freshness without a grant") + t.Fatal("publication update changed the producer observation without a grant") } } diff --git a/internal/internaldns/controlplane/access_recovery_envtest_test.go b/internal/internaldns/controlplane/access_recovery_envtest_test.go index 4fd1edc2..3586d3ab 100644 --- a/internal/internaldns/controlplane/access_recovery_envtest_test.go +++ b/internal/internaldns/controlplane/access_recovery_envtest_test.go @@ -2,7 +2,6 @@ package controlplane import ( "context" - "encoding/json" "os" "path/filepath" "testing" @@ -85,22 +84,9 @@ func TestAccessRecoveryPreservesSourceFencesWithCEL(t *testing.T) { t.Run("name-only customer references", func(t *testing.T) { verifyNamedReferencesWithAPI(t, cl) }) - deadline := metav1.NewTime(now.Add(time.Minute + 22357123*time.Nanosecond)) - access := &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "access", Namespace: "project"}, Spec: dnsv1alpha1.DNSResolverAccessBindingSpec{ContextRef: dnsv1alpha1.DNSObjectReference{Name: resolverContext.Name, UID: resolverContext.UID}, Region: "central", QueryIdentity: dnsv1alpha1.DNSResolverQueryIdentity{Type: dnsv1alpha1.DNSResolverQueryIdentityDestinationAddress, Value: "fd70:100::10"}, Port: 53, Transports: []string{"UDP", "TCP"}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 1, Sequence: 2, ValidUntil: deadline}}} - access.APIVersion = dnsv1alpha1.GroupVersion.String() - access.Kind = "DNSResolverAccessBinding" - raw, err := json.Marshal(access) - if err != nil { - t.Fatal(err) - } - // Submit unstructured JSON, as kubectl does, so the API initially stores a - // fractional deadline independently of the typed client encoder. - rawAccess := &unstructured.Unstructured{} - requireAccessTestSuccess(t, json.Unmarshal(raw, &rawAccess.Object)) - rawAccess.Object["spec"].(map[string]interface{})["authorization"].(map[string]interface{})["validUntil"] = deadline.UTC().Format(time.RFC3339Nano) - requireAccessTestSuccess(t, cl.Create(ctx, rawAccess)) - requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(access), access)) - r := &Reconciler{Client: cl, Scheme: scheme, Options: ReconcilerOptions{ProjectUID: "project-uid", PlatformNamespace: "platform", Region: "central", Shard: "shared", MaxAccessLease: 5 * time.Minute, AddressAllocator: &platform.Allocator{Client: cl, Namespace: "platform"}}} + access := &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "access", Namespace: "project"}, Spec: dnsv1alpha1.DNSResolverAccessBindingSpec{ContextRef: dnsv1alpha1.DNSObjectReference{Name: resolverContext.Name, UID: resolverContext.UID}, Region: "central", QueryIdentity: dnsv1alpha1.DNSResolverQueryIdentity{Type: dnsv1alpha1.DNSResolverQueryIdentityDestinationAddress, Value: "fd70:100::10"}, Port: 53, Transports: []string{"UDP", "TCP"}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 1, Sequence: 2}}} + requireAccessTestSuccess(t, cl.Create(ctx, access)) + r := &Reconciler{Client: cl, Scheme: scheme, Options: ReconcilerOptions{ProjectUID: "project-uid", PlatformNamespace: "platform", Region: "central", Shard: "shared", AddressAllocator: &platform.Allocator{Client: cl, Namespace: "platform"}}} reconcile := func(at time.Time) { t.Helper() requireAccessTestSuccess(t, r.reconcileContextBindings(ctx, "project", nil, map[string]*dnsv1alpha1.DNSZone{}, at)) @@ -112,213 +98,60 @@ func TestAccessRecoveryPreservesSourceFencesWithCEL(t *testing.T) { t.Fatalf("got %d bindings", len(bindings.Items)) } binding := bindings.Items[0] - if !binding.Spec.Authorization.ValidUntil.Equal(&deadline) { - t.Fatalf("binding projection changed fractional deadline: %v != %v", binding.Spec.Authorization.ValidUntil, deadline) - } - bindingKey := client.ObjectKeyFromObject(&binding) - readBinding := func() { - t.Helper() - requireAccessTestSuccess(t, cl.Get(ctx, bindingKey, &binding)) - } - reconcile(now) - readBinding() - if binding.Spec.Tombstone || !binding.Spec.Authorization.ValidUntil.Equal(&deadline) { - t.Fatal("unchanged fractional authorization was withdrawn on second reconcile") + key := client.ObjectKeyFromObject(&binding) + reconcile(now.Add(365 * 24 * time.Hour)) + requireAccessTestSuccess(t, cl.Get(ctx, key, &binding)) + if binding.Spec.Tombstone || binding.Spec.Authorization.Sequence != 2 || binding.Spec.Configuration.Revision != 1 { + t.Fatalf("elapsed time changed access: %#v", binding.Spec) } requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(access), access)) - if accepted := apimeta.FindStatusCondition(access.Status.Conditions, "Accepted"); accepted == nil || accepted.Status != metav1.ConditionTrue { - t.Fatal("fractional source authorization lost Accepted status") - } - subsecondMutation := binding.DeepCopy() - subsecondMutation.Spec.Authorization.ValidUntil = metav1.NewTime(deadline.Add(time.Nanosecond)) - if err := cl.Update(ctx, subsecondMutation); !apierrors.IsInvalid(err) { - t.Fatalf("same-fence nanosecond deadline mutation accepted: %v", err) - } - // False is omitted from the wire. An ordinary active update must still - // evaluate the optional tombstone transition rules without a CEL error. - active := binding.DeepCopy() - active.Labels["validation"] = "active-update" - if err := cl.Update(ctx, active); err != nil { - t.Fatalf("active update with omitted false tombstone failed: %v", err) - } - readBinding() - for _, change := range []struct { - name string - mutate func(*dnsv1alpha1.DNSResolverBinding) - }{ - {"context UID", func(b *dnsv1alpha1.DNSResolverBinding) { b.Spec.Source.ResolverContextRef.UID = "replacement-context" }}, - {"access UID", func(b *dnsv1alpha1.DNSResolverBinding) { b.Spec.Source.AccessBindingRef.UID = "replacement-access" }}, - {"placement", func(b *dnsv1alpha1.DNSResolverBinding) { b.Spec.Placement.Shard = "replacement-shard" }}, - {"listener", func(b *dnsv1alpha1.DNSResolverBinding) { b.Spec.Configuration.Listeners.Node.Address = "fd70:100::11" }}, - } { - mutated := binding.DeepCopy() - change.mutate(mutated) - mutated.Spec.Configuration.Revision++ - if err := cl.Update(ctx, mutated); !apierrors.IsInvalid(err) { - t.Fatalf("immutable %s change accepted: %v", change.name, err) + // Schema validation preserves listener identity and ordered source revisions. + for _, mutate := range []func(*dnsv1alpha1.DNSResolverAccessBinding){func(a *dnsv1alpha1.DNSResolverAccessBinding) { a.Spec.QueryIdentity.Value = "fd70:100::20" }, func(a *dnsv1alpha1.DNSResolverAccessBinding) { a.Spec.Authorization.Sequence = 1 }, func(a *dnsv1alpha1.DNSResolverAccessBinding) { a.Spec.ContextRef.UID = "another-context" }} { + candidate := access.DeepCopy() + mutate(candidate) + if err := cl.Update(ctx, candidate); !apierrors.IsInvalid(err) { + t.Fatalf("identity or revision mutation accepted: %v", err) } } - // Expiry advances the DNS-owned configuration fence, preserving the exact - // integration-owned authorization. Source sequence 3 remains available. - expiredAt := now.Add(61 * time.Second) - reconcile(expiredAt) - readBinding() - if !binding.Spec.Tombstone || binding.Spec.Configuration.Revision != 2 || binding.Spec.Authorization.Sequence != 2 || !binding.Spec.Authorization.ValidUntil.Equal(&deadline) { - t.Fatalf("withdrawal consumed source authority: %#v", binding.Spec) - } - requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(access), access)) - staleExpiredAccess := *access.DeepCopy() - // A direct same-fence replay cannot reopen the retired configuration. - replay := binding.DeepCopy() - replay.Spec.Tombstone = false - replay.Spec.Configuration.Revision++ - if err := cl.Update(ctx, replay); !apierrors.IsInvalid(err) { - t.Fatalf("same-fence reactivation accepted: %v", err) - } - mutated := binding.DeepCopy() - mutated.Spec.Authorization.ValidUntil = metav1.NewTime(expiredAt.Add(time.Minute + 987654321*time.Nanosecond)) - if err := cl.Update(ctx, mutated); !apierrors.IsInvalid(err) { - t.Fatalf("same-fence deadline mutation accepted: %v", err) - } - // Source CEL rejects deadline changes without an increasing sequence. - requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(access), access)) - baseAccess := access.DeepCopy() - renewedDeadline := metav1.NewTime(expiredAt.Add(time.Minute + 987654321*time.Nanosecond)) - access.Spec.Authorization.ValidUntil = renewedDeadline - if err := cl.Patch(ctx, access, client.MergeFromWithOptions(baseAccess, client.MergeFromWithOptimisticLock{})); !apierrors.IsInvalid(err) { - t.Fatalf("same-sequence source deadline mutation accepted: %v", err) - } - requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(access), access)) - if !access.Spec.Authorization.ValidUntil.Equal(&deadline) { - t.Fatal("rejected source patch changed its retained deadline") - } - reconcile(expiredAt) - readBinding() - if !binding.Spec.Tombstone { - t.Fatal("same source fence reopened withdrawn access") - } - // Ordinary next-sequence renewal is independent of private DNS counters. - requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(access), access)) - access.Spec.Authorization.Sequence = 3 - access.Spec.Authorization.ValidUntil = renewedDeadline - requireAccessTestSuccess(t, cl.Update(ctx, access)) - requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(access), access)) - if !access.Spec.Authorization.ValidUntil.Equal(&renewedDeadline) { - t.Fatal("source renewal lost deadline precision") - } - reconcile(expiredAt) - readBinding() - if binding.Spec.Tombstone || binding.Spec.Authorization.Sequence != 3 || binding.Spec.Configuration.Revision != 3 || !binding.Spec.Authorization.ValidUntil.Equal(&access.Spec.Authorization.ValidUntil) { - t.Fatalf("next source sequence did not recover access: %#v", binding.Spec) + // An incomplete cached List cannot revoke a live access binding. + stale := &Reconciler{Client: &accessListSnapshotClient{Client: cl, accesses: nil}, Scheme: scheme, Options: r.Options} + requireAccessTestSuccess(t, stale.reconcileContextBindings(ctx, "project", nil, nil, now.Add(365*24*time.Hour))) + requireAccessTestSuccess(t, cl.Get(ctx, key, &binding)) + if binding.Spec.Tombstone { + t.Fatal("incomplete List withdrew live access") } - // Another controller can List expired source sequence 2 before this renewed - // platform binding is visible. Cleanup must freshly read sequence 3 instead - // of converting an unchanged stale rejection status into revocation. - staleController := &Reconciler{Client: &accessListSnapshotClient{Client: cl, accesses: []dnsv1alpha1.DNSResolverAccessBinding{staleExpiredAccess}}, Scheme: scheme, Options: r.Options} - requireAccessTestSuccess(t, staleController.reconcileContextBindings(ctx, "project", nil, map[string]*dnsv1alpha1.DNSZone{}, expiredAt)) - readBinding() - if binding.Spec.Tombstone || binding.Spec.Authorization.Sequence != 3 || binding.Spec.Configuration.Revision != 3 { - t.Fatalf("stale source List withdrew renewed binding: %#v", binding.Spec) - } - staleController.Client = &accessListSnapshotClient{Client: cl} - requireAccessTestSuccess(t, staleController.reconcileContextBindings(ctx, "project", nil, map[string]*dnsv1alpha1.DNSZone{}, expiredAt)) - readBinding() - if binding.Spec.Tombstone || binding.Spec.Configuration.Revision != 3 { - t.Fatal("omitted source List withdrew renewed binding") - } - // Deleting the context revokes access before its source deadline. Recreating - // the same name cannot satisfy the old UID-pinned integration reference. - oldContextUID := resolverContext.UID requireAccessTestSuccess(t, cl.Delete(ctx, resolverContext)) - reconcile(expiredAt) - readBinding() - if !binding.Spec.Tombstone || binding.Spec.Authorization.Sequence != 3 || !binding.Spec.Authorization.ValidUntil.Equal(&access.Spec.Authorization.ValidUntil) { - t.Fatalf("context withdrawal rewrote source authority: %#v", binding.Spec) - } - freshContext := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "context", Namespace: "project"}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "opaque-consumer"}} - readyContext(freshContext) - if freshContext.UID == oldContextUID { - t.Fatal("context UID did not rotate") - } - reconcile(expiredAt) - readBinding() + reconcile(now.Add(365 * 24 * time.Hour)) + requireAccessTestSuccess(t, cl.Get(ctx, key, &binding)) + if !binding.Spec.Tombstone || binding.Spec.Authorization.Sequence != 2 { + t.Fatal("explicit context deletion did not withdraw access") + } + replacement := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: resolverContext.Name, Namespace: "project"}, Spec: resolverContext.Spec} + readyContext(replacement) + reconcile(now.Add(365 * 24 * time.Hour)) + requireAccessTestSuccess(t, cl.Get(ctx, key, &binding)) if !binding.Spec.Tombstone { - t.Fatal("recreated context name restored old access") + t.Fatal("replacement context inherited retired access") } - // A new access UID cannot reuse a quarantined destination. A fresh protected - // destination creates a separate binding and leaves the retired one closed. requireAccessTestSuccess(t, cl.Delete(ctx, access)) - freshAccess := &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "access", Namespace: "project"}, Spec: access.Spec} - freshAccess.Spec.ContextRef = dnsv1alpha1.DNSObjectReference{Name: freshContext.Name, UID: freshContext.UID} - freshAccess.Spec.Authorization.Sequence = 1 - requireAccessTestSuccess(t, cl.Create(ctx, freshAccess)) - requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(freshAccess), freshAccess)) - if !freshAccess.Spec.Authorization.ValidUntil.Equal(&renewedDeadline) { - t.Fatal("typed create lost source deadline precision") - } - reconcile(expiredAt) + access = &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "replacement", Namespace: "project"}, Spec: access.Spec} + access.Spec.ContextRef.UID = replacement.UID + access.Spec.QueryIdentity.Value = "fd70:100::20" + access.Spec.Authorization.Sequence = 1 + requireAccessTestSuccess(t, cl.Create(ctx, access)) + reconcile(now.Add(365 * 24 * time.Hour)) requireAccessTestSuccess(t, cl.List(ctx, &bindings, client.InNamespace("platform"))) - if len(bindings.Items) != 1 { - t.Fatal("new access lifetime reused a protected destination") - } - requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(freshAccess), freshAccess)) - freshAccess.Spec.QueryIdentity.Value = "fd70:100::11" - freshAccess.Spec.Authorization.Sequence = 2 - if err := cl.Update(ctx, freshAccess); !apierrors.IsInvalid(err) { - t.Fatalf("access identity changed in place: %v", err) - } - freshSpec := freshAccess.Spec - requireAccessTestSuccess(t, cl.Delete(ctx, freshAccess)) - freshAccess = &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "access", Namespace: "project"}, Spec: freshSpec} - requireAccessTestSuccess(t, cl.Create(ctx, freshAccess)) - requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(freshAccess), freshAccess)) - reconcile(expiredAt) - requireAccessTestSuccess(t, cl.List(ctx, &bindings, client.InNamespace("platform"))) - if len(bindings.Items) != 2 { - t.Fatalf("fresh protected identity created %d bindings", len(bindings.Items)) - } - readBinding() - if !binding.Spec.Tombstone { - t.Fatal("old binding reopened after access UID recreation") - } - // A renewal committed after the source confirmation but before destructive - // platform Patch must win through the platform resourceVersion fence. - var freshBinding dnsv1alpha1.DNSResolverBinding - for _, candidate := range bindings.Items { - if candidate.Spec.Source.AccessBindingRef.UID == freshAccess.UID { - freshBinding = candidate - } - } - if freshBinding.UID == "" { - t.Fatal("fresh access binding missing") - } - requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(freshAccess), freshAccess)) - freshAccess.Spec.Authorization.Sequence = 3 - freshAccess.Spec.Authorization.ValidUntil = metav1.NewTime(expiredAt.Add(-time.Second)) - requireAccessTestSuccess(t, cl.Update(ctx, freshAccess)) - raceClient := &bindingPatchRaceClient{Client: cl, beforeWithdrawal: func(ctx context.Context) error { - if err := cl.Get(ctx, client.ObjectKeyFromObject(freshAccess), freshAccess); err != nil { - return err - } - freshAccess.Spec.Authorization.Sequence = 4 - freshAccess.Spec.Authorization.ValidUntil = renewedDeadline - if err := cl.Update(ctx, freshAccess); err != nil { - return err - } - if err := cl.Get(ctx, client.ObjectKeyFromObject(&freshBinding), &freshBinding); err != nil { - return err + active := 0 + for _, b := range bindings.Items { + if !b.Spec.Tombstone { + active++ + if b.Spec.Source.ResolverContextRef.UID != replacement.UID { + t.Fatal("new binding uses retired context") + } } - freshBinding.Spec.Authorization = freshAccess.Spec.Authorization - freshBinding.Spec.Configuration.Revision++ - return cl.Update(ctx, &freshBinding) - }} - raceController := &Reconciler{Client: &accessListSnapshotClient{Client: raceClient}, Scheme: scheme, Options: r.Options} - if err := raceController.reconcileContextBindings(ctx, "project", nil, nil, expiredAt); !apierrors.IsConflict(err) { - t.Fatalf("concurrent platform renewal did not fence withdrawal: %v", err) } - requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(&freshBinding), &freshBinding)) - if freshBinding.Spec.Tombstone || freshBinding.Spec.Authorization.Sequence != 4 || !freshBinding.Spec.Authorization.ValidUntil.Equal(&renewedDeadline) { - t.Fatal("destructive cleanup overwrote concurrent platform renewal") + if active != 1 { + t.Fatalf("active bindings=%d", active) } } diff --git a/internal/internaldns/controlplane/compiler.go b/internal/internaldns/controlplane/compiler.go index aac95a6a..70838bc2 100644 --- a/internal/internaldns/controlplane/compiler.go +++ b/internal/internaldns/controlplane/compiler.go @@ -14,9 +14,7 @@ import ( "k8s.io/apimachinery/pkg/types" ) -// PublicationPlan is serialized into DNSPublicationChunk payloads. It carries -// original observation deadlines so a disconnected serving agent can expire an -// endpoint without receiving a central deletion. +// PublicationPlan is the canonical record configuration serialized into publication chunks. type PublicationPlan struct { ZoneUID types.UID `json:"zoneUID"` ZoneApex string `json:"zoneApex"` @@ -40,15 +38,13 @@ type PublicationRRSet struct { TTLSeconds int32 `json:"ttlSeconds"` Records []dnsv1alpha1.RecordEntry `json:"records"` ContributionUIDs []types.UID `json:"contributionUIDs,omitempty"` - ValidUntil *time.Time `json:"validUntil,omitempty"` } type ContributionFence struct { - UID types.UID `json:"uid"` - GrantUID types.UID `json:"grantUID"` - Epoch int64 `json:"epoch"` - Sequence int64 `json:"sequence"` - ValidUntil time.Time `json:"validUntil"` + UID types.UID `json:"uid"` + GrantUID types.UID `json:"grantUID"` + Epoch int64 `json:"epoch"` + Sequence int64 `json:"sequence"` } type RegistrationFence struct { @@ -86,10 +82,6 @@ func Compile(in CompileInput) (CompileResult, error) { return CompileResult{}, err } - now := in.Now - if now.IsZero() { - now = time.Now().UTC() - } result := CompileResult{Accepted: map[types.UID]bool{}, Reasons: map[types.UID]string{}, Sequences: map[types.UID]int64{}} result.Plan.ZoneUID = in.Zone.UID result.Plan.ZoneApex = canonicalName(in.Zone.Spec.DomainName) @@ -193,7 +185,6 @@ func Compile(in CompileInput) (CompileResult, error) { previousSeq := map[types.UID]ContributionFence{} if in.Previous != nil { for _, f := range in.Previous.Contributions { - f.ValidUntil = f.ValidUntil.UTC() previousSeq[f.UID] = f } } @@ -208,7 +199,6 @@ func Compile(in CompileInput) (CompileResult, error) { ttl int32 records []dnsv1alpha1.RecordEntry uids []types.UID - until *time.Time } aggregates := map[ownershipKey]*aggregate{} for i := range in.Contributions { @@ -220,11 +210,7 @@ func Compile(in CompileInput) (CompileResult, error) { result.Reasons[c.UID] = reason continue } - until := time.Time{} - if c.Status.ValidUntil != nil { - until = c.Status.ValidUntil.UTC() - } - currentFence := ContributionFence{UID: c.UID, GrantUID: grant.UID, Epoch: c.Status.WriterEpoch, Sequence: c.Status.Sequence, ValidUntil: until} + currentFence := ContributionFence{UID: c.UID, GrantUID: grant.UID, Epoch: c.Status.WriterEpoch, Sequence: c.Status.Sequence} previousFence := previousSeq[c.UID] if previousFence.UID == "" || currentFence.Epoch > previousFence.Epoch || currentFence.Sequence > previousFence.Sequence { publicationFences[c.UID] = currentFence @@ -238,10 +224,6 @@ func Compile(in CompileInput) (CompileResult, error) { result.Reasons[c.UID] = "Ineligible" continue } - if c.Status.ValidUntil == nil || !c.Status.ValidUntil.After(now) { - result.Reasons[c.UID] = "Expired" - continue - } validBundle := true for _, set := range c.Spec.RecordSets { if len(set.Records) == 0 || !containsType(reg.Spec.RecordTypes, set.RecordType) || !containsType(grant.Spec.RecordTypes, set.RecordType) { @@ -279,10 +261,6 @@ func Compile(in CompileInput) (CompileResult, error) { record.Name = key.name a.records = append(a.records, record) a.uids = append(a.uids, c.UID) - if a.until == nil || until.Before(*a.until) { - u := until - a.until = &u - } } } } @@ -309,7 +287,7 @@ func Compile(in CompileInput) (CompileResult, error) { } for key, a := range aggregates { sortAggregate(a.records, a.uids) - result.Plan.RRsets = append(result.Plan.RRsets, PublicationRRSet{Name: key.name, RecordType: key.rrtype, TTLSeconds: a.ttl, Records: a.records, ContributionUIDs: a.uids, ValidUntil: a.until}) + result.Plan.RRsets = append(result.Plan.RRsets, PublicationRRSet{Name: key.name, RecordType: key.rrtype, TTLSeconds: a.ttl, Records: a.records, ContributionUIDs: a.uids}) } sort.Slice(result.Plan.RRsets, func(i, j int) bool { if result.Plan.RRsets[i].Name == result.Plan.RRsets[j].Name { @@ -340,9 +318,6 @@ func validateContribution(c *dnsv1alpha1.DNSRecordContribution, reg *dnsv1alpha1 if c.Status.Sequence <= 0 { return "InvalidSequence" } - if c.Status.ValidUntil == nil || c.Status.ValidUntil.IsZero() { - return "MissingValidityDeadline" - } if previous.UID != "" { if previous.GrantUID != "" && previous.GrantUID != grant.UID { return "FenceIdentityMismatch" @@ -354,9 +329,6 @@ func validateContribution(c *dnsv1alpha1.DNSRecordContribution, reg *dnsv1alpha1 if c.Status.Sequence < previous.Sequence { return dnsValueNonIncreasingSequence } - if c.Status.Sequence == previous.Sequence && !c.Status.ValidUntil.Time.Equal(previous.ValidUntil) { - return "FenceDeadlineMismatch" - } } } return "" diff --git a/internal/internaldns/controlplane/compiler_test.go b/internal/internaldns/controlplane/compiler_test.go index 545debea..fe5b6c80 100644 --- a/internal/internaldns/controlplane/compiler_test.go +++ b/internal/internaldns/controlplane/compiler_test.go @@ -76,25 +76,23 @@ func TestCompileRejectsStaleGenerationAndWholeInvalidBundle(t *testing.T) { }) } -func TestCompileExpiryStillAdvancesHighWaterAndRejectsRollback(t *testing.T) { +func TestCompileWithdrawalAdvancesHighWaterAndRejectsRollback(t *testing.T) { now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) in := fixture(now) previous := in.Contributions[0] - in.Previous = &PublicationPlan{Contributions: []ContributionFence{{UID: previous.UID, GrantUID: previous.Spec.GrantRef.UID, Epoch: 7, Sequence: 18, ValidUntil: now}}} + in.Previous = &PublicationPlan{Contributions: []ContributionFence{{UID: previous.UID, GrantUID: previous.Spec.GrantRef.UID, Epoch: 7, Sequence: 18}}} in.Contributions[0].Status.Sequence = 19 - expired := metav1.NewTime(now.Add(-time.Second)) - in.Contributions[0].Status.ValidUntil = &expired + in.Contributions[0].Status.Eligible = false got, err := Compile(in) if err != nil { t.Fatal(err) } - if got.Reasons[previous.UID] != "Expired" || len(got.Plan.RRsets) != 0 || got.Plan.Contributions[0].Sequence != 19 { - t.Fatalf("expiry fence incorrect: %#v", got) + if got.Reasons[previous.UID] != "Ineligible" || len(got.Plan.RRsets) != 0 || got.Plan.Contributions[0].Sequence != 19 { + t.Fatalf("withdrawal fence incorrect: %#v", got) } in.Previous = &got.Plan in.Contributions[0].Status.Sequence = 18 - fresh := metav1.NewTime(now.Add(time.Minute)) - in.Contributions[0].Status.ValidUntil = &fresh + in.Contributions[0].Status.Eligible = true rolled, err := Compile(in) if err != nil { t.Fatal(err) @@ -141,28 +139,6 @@ func TestCompileRetainsHighWaterAcrossRejectedPublications(t *testing.T) { } } -func TestCompileRejectsEqualSequenceDeadlineMutation(t *testing.T) { - now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) - in := fixture(now) - first, err := Compile(in) - if err != nil { - t.Fatal(err) - } - in.Previous = &first.Plan - changed := metav1.NewTime(in.Contributions[0].Status.ValidUntil.Add(time.Minute)) - in.Contributions[0].Status.ValidUntil = &changed - got, err := Compile(in) - if err != nil { - t.Fatal(err) - } - if got.Reasons["contribution-a"] != "FenceDeadlineMismatch" || len(got.Plan.RRsets) != 0 { - t.Fatalf("equal sequence changed its immutable deadline: %#v", got) - } - if len(got.Plan.Contributions) != 1 || !got.Plan.Contributions[0].ValidUntil.Equal(first.Plan.Contributions[0].ValidUntil) { - t.Fatalf("deadline mutation replaced durable fence: %#v", got.Plan.Contributions) - } -} - func TestCompileNormalizesFQDNAndEnforcesCNAMEExclusivity(t *testing.T) { in := fixture(time.Now().UTC()) in.StaticRecords = []dnsv1alpha1.DNSRecordSet{{ @@ -215,8 +191,7 @@ func fixture(now time.Time) CompileInput { } reg := dnsv1alpha1.DNSRegistration{ObjectMeta: metav1.ObjectMeta{Name: "api", UID: "registration-a", Generation: 3}, Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name, UID: zone.UID, Generation: zone.Generation}, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeAAAA}, PublicationPolicy: dnsv1alpha1.DNSPublicationPolicyEligibleContributions, TTLSeconds: 30}} grant := dnsv1alpha1.DNSContributionGrant{ObjectMeta: metav1.ObjectMeta{Name: "compute", UID: "grant-a", Generation: 2}, Spec: dnsv1alpha1.DNSContributionGrantSpec{RegistrationRef: dnsv1alpha1.DNSObjectReference{Name: reg.Name, UID: reg.UID, Generation: reg.Generation}, ProducerID: "compute-east", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeAAAA}}, Status: dnsv1alpha1.DNSContributionGrantStatus{ActiveWriterEpoch: 7, ObservedGrantGeneration: 2, ObservedRegistrationGeneration: 3}} - until := metav1.NewTime(now.Add(time.Minute)) - c := dnsv1alpha1.DNSRecordContribution{ObjectMeta: metav1.ObjectMeta{Name: "endpoint-a", UID: "contribution-a", Generation: 4}, Spec: dnsv1alpha1.DNSRecordContributionSpec{RegistrationRef: dnsv1alpha1.DNSObjectReference{Name: reg.Name, UID: reg.UID, Generation: reg.Generation}, GrantRef: dnsv1alpha1.DNSObjectReference{Name: grant.Name, UID: grant.UID}, RecordSets: []dnsv1alpha1.DNSContributionRecordSet{{RecordType: dnsv1alpha1.RRTypeAAAA, Records: []dnsv1alpha1.RecordEntry{{Name: "api.corp.internal", AAAA: &dnsv1alpha1.AAAARecordSpec{Content: "fd00::1"}}}}}}, Status: dnsv1alpha1.DNSRecordContributionStatus{ObservedGeneration: 4, WriterEpoch: 7, Sequence: 18, Eligible: true, ValidUntil: &until}} + c := dnsv1alpha1.DNSRecordContribution{ObjectMeta: metav1.ObjectMeta{Name: "endpoint-a", UID: "contribution-a", Generation: 4}, Spec: dnsv1alpha1.DNSRecordContributionSpec{RegistrationRef: dnsv1alpha1.DNSObjectReference{Name: reg.Name, UID: reg.UID, Generation: reg.Generation}, GrantRef: dnsv1alpha1.DNSObjectReference{Name: grant.Name, UID: grant.UID}, RecordSets: []dnsv1alpha1.DNSContributionRecordSet{{RecordType: dnsv1alpha1.RRTypeAAAA, Records: []dnsv1alpha1.RecordEntry{{Name: "api.corp.internal", AAAA: &dnsv1alpha1.AAAARecordSpec{Content: "fd00::1"}}}}}}, Status: dnsv1alpha1.DNSRecordContributionStatus{ObservedGeneration: 4, WriterEpoch: 7, Sequence: 18, Eligible: true}} return CompileInput{Zone: zone, Associations: []dnsv1alpha1.DNSZoneAssociation{assoc("b", "vpc-b"), assoc("a", "vpc-a")}, Registrations: []dnsv1alpha1.DNSRegistration{reg}, Grants: []dnsv1alpha1.DNSContributionGrant{grant}, Contributions: []dnsv1alpha1.DNSRecordContribution{c}, Now: now} } @@ -316,3 +291,32 @@ func TestCompileRejectsUnboundedDynamicTTL(t *testing.T) { t.Fatal("unbounded TTL published") } } + +func TestCompileRetainsHealthyRecordsUntilExplicitWithdrawal(t *testing.T) { + now := time.Now().UTC() + input := fixture(now) + first, err := Compile(input) + if err != nil { + t.Fatal(err) + } + input.Now = now.Add(365 * 24 * time.Hour) + input.Previous = &first.Plan + after, err := Compile(input) + if err != nil { + t.Fatal(err) + } + beforeJSON, _ := json.Marshal(first.Plan) + afterJSON, _ := json.Marshal(after.Plan) + if string(beforeJSON) != string(afterJSON) { + t.Fatal("elapsed time changed the last reported endpoint state") + } + input.Contributions[0].Status.Eligible = false + input.Contributions[0].Status.Sequence++ + withdrawn, err := Compile(input) + if err != nil { + t.Fatal(err) + } + if len(withdrawn.Plan.RRsets) != 0 { + t.Fatal("explicit unhealthy observation did not withdraw the endpoint") + } +} diff --git a/internal/internaldns/controlplane/context_reconciler.go b/internal/internaldns/controlplane/context_reconciler.go index 488d804a..eb29962f 100644 --- a/internal/internaldns/controlplane/context_reconciler.go +++ b/internal/internaldns/controlplane/context_reconciler.go @@ -363,11 +363,11 @@ func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, as spec := a.Spec contextObject := contextByUID[spec.ContextRef.UID] contextReady := resolverContextReady(contextObject) - valid := a.DeletionTimestamp.IsZero() && contextObject != nil && contextObject.DeletionTimestamp.IsZero() && contextObject.Name == spec.ContextRef.Name && contextReady && contextObject.Status.AccessWriterEpoch > 0 && spec.Authorization.WriterEpoch == contextObject.Status.AccessWriterEpoch && spec.Authorization.Sequence >= a.Status.ObservedSequence && spec.Authorization.ValidUntil.After(now) && !spec.Authorization.ValidUntil.After(now.Add(r.Options.MaxAccessLease)) + valid := a.DeletionTimestamp.IsZero() && contextObject != nil && contextObject.DeletionTimestamp.IsZero() && contextObject.Name == spec.ContextRef.Name && contextReady && contextObject.Status.AccessWriterEpoch > 0 && spec.Authorization.WriterEpoch == contextObject.Status.AccessWriterEpoch && spec.Authorization.Sequence >= a.Status.ObservedSequence bindingName := "binding-" + model.OpaqueToken(string(r.Options.ProjectUID) + "/" + string(a.UID))[:20] if prior := bindingsByName[bindingName]; prior != nil { compare := model.Compare(uint64(spec.Authorization.WriterEpoch), uint64(spec.Authorization.Sequence), uint64(prior.Spec.Authorization.WriterEpoch), uint64(prior.Spec.Authorization.Sequence)) - if compare < 0 || (compare == 0 && (!spec.Authorization.ValidUntil.Equal(&prior.Spec.Authorization.ValidUntil) || prior.Spec.Tombstone)) { + if compare < 0 || (compare == 0 && prior.Spec.Tombstone) { valid = false } } @@ -433,7 +433,7 @@ func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, as return err } } - binding := &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: r.Options.PlatformNamespace, Labels: map[string]string{"internal-dns.miloapis.com/context-uid": string(contextObject.UID), "internal-dns.miloapis.com/access-uid": string(a.UID), "internal-dns.miloapis.com/region": model.SafeToken(spec.Region), "internal-dns.miloapis.com/shard": model.SafeToken(r.shardForRegion(spec.Region))}}, Spec: dnsv1alpha1.DNSResolverBindingSpec{Source: dnsv1alpha1.DNSResolverBindingSource{ProjectUID: r.Options.ProjectUID, ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: contextObject.Name, UID: contextObject.UID}, AccessBindingRef: dnsv1alpha1.DNSObjectReference{Name: a.Name, UID: a.UID}}, Placement: dnsv1alpha1.DNSResolverBindingPlacement{Region: spec.Region, Shard: r.shardForRegion(spec.Region)}, Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Generation: generation, Revision: revision, Listeners: dnsv1alpha1.DNSResolverBindingListeners{Node: dnsv1alpha1.DNSResolverBindingListener{Address: spec.QueryIdentity.Value, Port: spec.Port, Transports: append([]string(nil), spec.Transports...)}, Regional: dnsv1alpha1.DNSResolverBindingListener{Address: cluster, Port: spec.Port, Transports: append([]string(nil), spec.Transports...)}}, ZoneRefs: zoneReferences(zids, zones)}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: spec.Authorization.WriterEpoch, Sequence: spec.Authorization.Sequence, ValidUntil: spec.Authorization.ValidUntil}}} + binding := &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: r.Options.PlatformNamespace, Labels: map[string]string{"internal-dns.miloapis.com/context-uid": string(contextObject.UID), "internal-dns.miloapis.com/access-uid": string(a.UID), "internal-dns.miloapis.com/region": model.SafeToken(spec.Region), "internal-dns.miloapis.com/shard": model.SafeToken(r.shardForRegion(spec.Region))}}, Spec: dnsv1alpha1.DNSResolverBindingSpec{Source: dnsv1alpha1.DNSResolverBindingSource{ProjectUID: r.Options.ProjectUID, ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: contextObject.Name, UID: contextObject.UID}, AccessBindingRef: dnsv1alpha1.DNSObjectReference{Name: a.Name, UID: a.UID}}, Placement: dnsv1alpha1.DNSResolverBindingPlacement{Region: spec.Region, Shard: r.shardForRegion(spec.Region)}, Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Generation: generation, Revision: revision, Listeners: dnsv1alpha1.DNSResolverBindingListeners{Node: dnsv1alpha1.DNSResolverBindingListener{Address: spec.QueryIdentity.Value, Port: spec.Port, Transports: append([]string(nil), spec.Transports...)}, Regional: dnsv1alpha1.DNSResolverBindingListener{Address: cluster, Port: spec.Port, Transports: append([]string(nil), spec.Transports...)}}, ZoneRefs: zoneReferences(zids, zones)}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: spec.Authorization.WriterEpoch, Sequence: spec.Authorization.Sequence}}} if !exists { if err := r.Create(ctx, binding); err != nil { return err @@ -472,7 +472,7 @@ func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, as if b.Labels["internal-dns.miloapis.com/context-uid"] == "" { continue } - withdraw, err := r.confirmAccessWithdrawal(ctx, ns, b, now) + withdraw, err := r.confirmAccessWithdrawal(ctx, ns, b) if err != nil { return err } @@ -493,8 +493,8 @@ func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, as // confirmAccessWithdrawal prevents a stale or incomplete source List from // withdrawing a binding renewed by another controller. Read uncertainty leaves -// the existing authorization intact; serving agents still enforce its deadline. -func (r *Reconciler) confirmAccessWithdrawal(ctx context.Context, namespace string, binding *dnsv1alpha1.DNSResolverBinding, now time.Time) (bool, error) { +// the existing authorization intact; serving agents retain the applied configuration. +func (r *Reconciler) confirmAccessWithdrawal(ctx context.Context, namespace string, binding *dnsv1alpha1.DNSResolverBinding) (bool, error) { var access dnsv1alpha1.DNSResolverAccessBinding if err := r.Get(ctx, client.ObjectKey{Namespace: namespace, Name: binding.Spec.Source.AccessBindingRef.Name}, &access); err != nil { if apierrors.IsNotFound(err) { @@ -508,7 +508,7 @@ func (r *Reconciler) confirmAccessWithdrawal(ctx context.Context, namespace stri authorization := access.Spec.Authorization prior := binding.Spec.Authorization compare := model.Compare(uint64(authorization.WriterEpoch), uint64(authorization.Sequence), uint64(prior.WriterEpoch), uint64(prior.Sequence)) - if compare < 0 || (compare == 0 && !authorization.ValidUntil.Equal(&prior.ValidUntil)) { + if compare < 0 { return false, nil } if access.Spec.ContextRef.Name != binding.Spec.Source.ResolverContextRef.Name || access.Spec.ContextRef.UID != binding.Spec.Source.ResolverContextRef.UID { @@ -525,11 +525,11 @@ func (r *Reconciler) confirmAccessWithdrawal(ctx context.Context, namespace stri return true, nil } ready := apimeta.FindStatusCondition(resolverContext.Status.Conditions, "Ready") - if !authorization.ValidUntil.After(now) || resolverContext.Status.AccessWriterEpoch > authorization.WriterEpoch { + if resolverContext.Status.AccessWriterEpoch > authorization.WriterEpoch { return true, nil } // Readiness or epoch propagation can lag a healthy authorization. Such - // uncertainty is not retirement evidence and must not consume its lease. + // uncertainty is not retirement evidence and must not withdraw its binding. if ready == nil || ready.Status != metav1.ConditionTrue || ready.ObservedGeneration != resolverContext.Generation || resolverContext.Status.AccessWriterEpoch < authorization.WriterEpoch { return false, nil } @@ -572,7 +572,7 @@ func bindingAckCurrent(binding *dnsv1alpha1.DNSResolverBinding, now time.Time) b // past the exact configuration/authorization generation or its local ACK // lease. An ACK lease may be shorter than the access deadline, but never // longer because AckSink caps it before persisting the row. - if ack.Phase == string(model.AckVerified) && ack.Revision == binding.Spec.Configuration.Revision && ack.AuthorizationIssuerEpoch == binding.Spec.Authorization.WriterEpoch && ack.AuthorizationRevision == binding.Spec.Authorization.Sequence && ack.ValidUntil != nil && ack.ValidUntil.After(now) && !ack.ValidUntil.After(binding.Spec.Authorization.ValidUntil.Time) { + if ack.Phase == string(model.AckVerified) && ack.Revision == binding.Spec.Configuration.Revision && ack.AuthorizationIssuerEpoch == binding.Spec.Authorization.WriterEpoch && ack.AuthorizationRevision == binding.Spec.Authorization.Sequence && ack.ValidUntil != nil && ack.ValidUntil.After(now) { return true } } @@ -661,8 +661,6 @@ func BindingAddressCandidates(uid types.UID) (string, string) { func bindingEqual(a, b dnsv1alpha1.DNSResolverBindingSpec) bool { x := a y := b - x.Authorization.ValidUntil = metav1.Time{} - y.Authorization.ValidUntil = metav1.Time{} xb, _ := json.Marshal(x) yb, _ := json.Marshal(y) return string(xb) == string(yb) diff --git a/internal/internaldns/controlplane/context_reconciler_test.go b/internal/internaldns/controlplane/context_reconciler_test.go index 882725ca..ffe14cf1 100644 --- a/internal/internaldns/controlplane/context_reconciler_test.go +++ b/internal/internaldns/controlplane/context_reconciler_test.go @@ -28,22 +28,6 @@ type accessListSnapshotClient struct { getErrorFor string } -type bindingPatchRaceClient struct { - client.Client - beforeWithdrawal func(context.Context) error -} - -func (c *bindingPatchRaceClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { - if binding, ok := obj.(*dnsv1alpha1.DNSResolverBinding); ok && binding.Spec.Tombstone && c.beforeWithdrawal != nil { - before := c.beforeWithdrawal - c.beforeWithdrawal = nil - if err := before(ctx); err != nil { - return err - } - } - return c.Client.Patch(ctx, obj, patch, opts...) -} - func (c *accessListSnapshotClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { _, access := obj.(*dnsv1alpha1.DNSResolverAccessBinding) _, resolverContext := obj.(*dnsv1alpha1.DNSResolverContext) @@ -69,7 +53,7 @@ func (testAllocator) ClaimDestination(context.Context, types.UID, types.UID, str return nil } -func TestContextAccessProjectsExactAuthorizationAndExpiresClosed(t *testing.T) { +func TestContextAccessPersistsUntilExplicitWithdrawal(t *testing.T) { ctx := context.Background() now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) s := runtime.NewScheme() @@ -81,10 +65,9 @@ func TestContextAccessProjectsExactAuthorizationAndExpiresClosed(t *testing.T) { zone := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "zone", Namespace: "project", UID: "zone-uid", Generation: 1}, Spec: dnsv1alpha1.DNSZoneSpec{Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}} association := &dnsv1alpha1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: "assoc", Namespace: "project", UID: "assoc-uid", Generation: 1}, Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: "zone"}, ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: "ctx"}}, Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: "zone", UID: "zone-uid", Generation: 1}, ResolvedResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: "ctx", UID: "context-uid"}}} setCondition(&association.Status.Conditions, "Accepted", metav1.ConditionTrue, "Accepted", "", 1, now) - until := metav1.NewTime(now.Add(2 * time.Minute)) - access := &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "access", Namespace: "project", UID: "access-uid", Generation: 1}, Spec: dnsv1alpha1.DNSResolverAccessBindingSpec{ContextRef: dnsv1alpha1.DNSObjectReference{Name: "ctx", UID: "context-uid"}, Region: "central", QueryIdentity: dnsv1alpha1.DNSResolverQueryIdentity{Type: dnsv1alpha1.DNSResolverQueryIdentityDestinationAddress, Value: "fd70:100::10"}, Port: 53, Transports: []string{"UDP", "TCP"}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 4, Sequence: 12, ValidUntil: until}}} + access := &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "access", Namespace: "project", UID: "access-uid", Generation: 1}, Spec: dnsv1alpha1.DNSResolverAccessBindingSpec{ContextRef: dnsv1alpha1.DNSObjectReference{Name: "ctx", UID: "context-uid"}, Region: "central", QueryIdentity: dnsv1alpha1.DNSResolverQueryIdentity{Type: dnsv1alpha1.DNSResolverQueryIdentityDestinationAddress, Value: "fd70:100::10"}, Port: 53, Transports: []string{"UDP", "TCP"}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 4, Sequence: 12}}} c := fake.NewClientBuilder().WithScheme(s).WithObjects(contextObject, zone, association, access).WithStatusSubresource(&dnsv1alpha1.DNSResolverContext{}, &dnsv1alpha1.DNSResolverAccessBinding{}, &dnsv1alpha1.DNSResolverAccessBinding{}, &dnsv1alpha1.DNSResolverBinding{}).Build() - r := &Reconciler{Client: c, Scheme: s, Options: ReconcilerOptions{ProjectUID: "project-uid", PlatformNamespace: "platform", Region: "central", Shard: "shared", MaxAccessLease: 5 * time.Minute, AddressAllocator: testAllocator{}}} + r := &Reconciler{Client: c, Scheme: s, Options: ReconcilerOptions{ProjectUID: "project-uid", PlatformNamespace: "platform", Region: "central", Shard: "shared", AddressAllocator: testAllocator{}}} if err := r.reconcileContextBindings(ctx, "project", []dnsv1alpha1.DNSZoneAssociation{*association}, map[string]*dnsv1alpha1.DNSZone{"zone": zone}, now); err != nil { t.Fatal(err) } @@ -96,7 +79,7 @@ func TestContextAccessProjectsExactAuthorizationAndExpiresClosed(t *testing.T) { t.Fatalf("got %d bindings", len(bindings.Items)) } b := bindings.Items[0] - if b.Spec.Configuration.Listeners.Node.Address != "fd70:100::10" || b.Spec.Source.ResolverContextRef.UID != "context-uid" || b.Spec.Authorization.WriterEpoch != 4 || b.Spec.Authorization.Sequence != 12 || !b.Spec.Authorization.ValidUntil.Equal(&until) { + if b.Spec.Configuration.Listeners.Node.Address != "fd70:100::10" || b.Spec.Source.ResolverContextRef.UID != "context-uid" || b.Spec.Authorization.WriterEpoch != 4 || b.Spec.Authorization.Sequence != 12 { t.Fatalf("access projection drifted: %#v", b.Spec) } if b.Spec.Source.ProjectUID != "project-uid" || b.Spec.Source.ResolverContextRef.Name != "ctx" || b.Spec.Source.AccessBindingRef.Name != "access" || b.Spec.Source.AccessBindingRef.UID != "access-uid" || b.Spec.Configuration.ZoneRefs[0].Name != "zone" || b.Spec.Configuration.ZoneRefs[0].UID != "zone-uid" || b.Spec.Placement.Region != "central" { @@ -125,13 +108,17 @@ func TestContextAccessProjectsExactAuthorizationAndExpiresClosed(t *testing.T) { t.Fatalf("zone membership change reused configuration fences: %#v", b.Spec.Configuration) } + contextObject.Status.AccessWriterEpoch++ + if err := c.Status().Update(ctx, contextObject); err != nil { + t.Fatal(err) + } if err := r.reconcileContextBindings(ctx, "project", []dnsv1alpha1.DNSZoneAssociation{*association}, map[string]*dnsv1alpha1.DNSZone{"zone": zone}, now.Add(3*time.Minute)); err != nil { t.Fatal(err) } if err := c.Get(ctx, client.ObjectKeyFromObject(&b), &b); err != nil { t.Fatal(err) } - if !b.Spec.Tombstone || b.Spec.Authorization.Sequence != 12 || !b.Spec.Authorization.ValidUntil.Equal(&until) || b.Spec.Configuration.Revision != 3 { + if !b.Spec.Tombstone || b.Spec.Authorization.Sequence != 12 || b.Spec.Configuration.Revision != 3 { t.Fatalf("withdrawal changed integration authority or failed to advance configuration: %#v", b.Spec) } if err := c.Get(ctx, client.ObjectKeyFromObject(access), access); err != nil { @@ -141,8 +128,7 @@ func TestContextAccessProjectsExactAuthorizationAndExpiresClosed(t *testing.T) { if ready == nil || ready.Status != metav1.ConditionFalse { t.Fatalf("invalid access retained Ready: %#v", access.Status.Conditions) } - staleExpiredAccess := *access.DeepCopy() - access.Spec.Authorization.ValidUntil = metav1.NewTime(now.Add(4 * time.Minute)) + staleAccess := *access.DeepCopy() if err := c.Update(ctx, access); err != nil { t.Fatal(err) } @@ -153,11 +139,12 @@ func TestContextAccessProjectsExactAuthorizationAndExpiresClosed(t *testing.T) { t.Fatal(err) } if !b.Spec.Tombstone { - t.Fatal("same-fence source deadline replay reopened access") + t.Fatal("same-fence replay reopened access") } if err := c.Get(ctx, client.ObjectKeyFromObject(access), access); err != nil { t.Fatal(err) } + access.Spec.Authorization.WriterEpoch = contextObject.Status.AccessWriterEpoch access.Spec.Authorization.Sequence++ if err := c.Update(ctx, access); err != nil { t.Fatal(err) @@ -168,10 +155,10 @@ func TestContextAccessProjectsExactAuthorizationAndExpiresClosed(t *testing.T) { if err := c.Get(ctx, client.ObjectKeyFromObject(&b), &b); err != nil { t.Fatal(err) } - if b.Spec.Tombstone || b.Spec.Authorization.Sequence != 13 || !b.Spec.Authorization.ValidUntil.Equal(&access.Spec.Authorization.ValidUntil) || b.Spec.Configuration.Revision != 4 { + if b.Spec.Tombstone || b.Spec.Authorization.Sequence != 13 || b.Spec.Configuration.Revision != 4 { t.Fatalf("ordinary next source sequence failed to renew access: %#v", b.Spec) } - for _, snapshot := range [][]dnsv1alpha1.DNSResolverAccessBinding{{staleExpiredAccess}, nil} { + for _, snapshot := range [][]dnsv1alpha1.DNSResolverAccessBinding{{staleAccess}, nil} { staleController := &Reconciler{Client: &accessListSnapshotClient{Client: c, accesses: snapshot}, Scheme: s, Options: r.Options} if err := staleController.reconcileContextBindings(ctx, "project", nil, nil, now.Add(3*time.Minute)); err != nil { t.Fatal(err) @@ -216,9 +203,8 @@ func TestContextAccessProjectsExactAuthorizationAndExpiresClosed(t *testing.T) { func TestBindingAckCurrentRequiresVerifiedExactAuthorization(t *testing.T) { now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) - deadline := metav1.NewTime(now.Add(time.Minute)) ackDeadline := metav1.NewTime(now.Add(30 * time.Second)) - b := &dnsv1alpha1.DNSResolverBinding{Spec: dnsv1alpha1.DNSResolverBindingSpec{Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Revision: 7}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 3, Sequence: 11, ValidUntil: deadline}}, Status: dnsv1alpha1.DNSResolverBindingStatus{MemberAcknowledgements: []dnsv1alpha1.DNSApplyAcknowledgement{{Phase: string(model.AckVerified), Revision: 7, AuthorizationIssuerEpoch: 3, AuthorizationRevision: 11, ValidUntil: &ackDeadline}}}} + b := &dnsv1alpha1.DNSResolverBinding{Spec: dnsv1alpha1.DNSResolverBindingSpec{Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Revision: 7}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 3, Sequence: 11}}, Status: dnsv1alpha1.DNSResolverBindingStatus{MemberAcknowledgements: []dnsv1alpha1.DNSApplyAcknowledgement{{Phase: string(model.AckVerified), Revision: 7, AuthorizationIssuerEpoch: 3, AuthorizationRevision: 11, ValidUntil: &ackDeadline}}}} if !bindingAckCurrent(b, now) { t.Fatal("exact verified ACK was rejected") } @@ -232,11 +218,6 @@ func TestBindingAckCurrentRequiresVerifiedExactAuthorization(t *testing.T) { t.Fatal("stale authorization ACK was accepted") } b.Status.MemberAcknowledgements[0].AuthorizationRevision = 11 - overlong := metav1.NewTime(now.Add(2 * time.Minute)) - b.Status.MemberAcknowledgements[0].ValidUntil = &overlong - if bindingAckCurrent(b, now) { - t.Fatal("ACK beyond access deadline was accepted") - } } func TestContextNamingPolicyResolvesMultipleZonesWithoutNetworking(t *testing.T) { diff --git a/internal/internaldns/controlplane/publication_reconciler.go b/internal/internaldns/controlplane/publication_reconciler.go index cca97be2..9df5ca03 100644 --- a/internal/internaldns/controlplane/publication_reconciler.go +++ b/internal/internaldns/controlplane/publication_reconciler.go @@ -101,7 +101,7 @@ func (r *Reconciler) reconcileProject(ctx context.Context, ns string) (ctrl.Resu return ctrl.Result{RequeueAfter: nextProjectRequeue(now, earliest, r.Options.LeaseDuration)}, nil } -// nextProjectRequeue never lets a later contribution or authorization expiry +// nextProjectRequeue never lets a later scheduled reconciliation // postpone ownership renewal. A standby polls more frequently while another // holder owns a zone, but the active holder must still run within half a lease // even when the compiled publication is otherwise quiescent. @@ -311,7 +311,7 @@ func (r *Reconciler) compileZone(ctx context.Context, z *dnsv1alpha1.DNSZone, as return false, time.Time{}, err } } - return r.tombstoneAcknowledged(ctx, own), nextExpiry(ownPlan, now), nil + return r.tombstoneAcknowledged(ctx, own), time.Time{}, nil } previousRevision, err := r.activeRevision(ctx, own) if err != nil { @@ -342,7 +342,7 @@ func (r *Reconciler) compileZone(ctx context.Context, z *dnsv1alpha1.DNSZone, as if err := r.Patch(ctx, own, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { return false, time.Time{}, err } - return false, nextExpiry(ownPlan, now), nil + return false, time.Time{}, nil } func (r *Reconciler) activeRevision(ctx context.Context, own *dnsv1alpha1.DNSPublicationOwnership) (int64, error) { @@ -430,7 +430,7 @@ func (r *Reconciler) persistPublication(ctx context.Context, z *dnsv1alpha1.DNSZ } apiFences := make([]dnsv1alpha1.DNSContributionFence, 0, len(fences)) for _, f := range fences { - apiFences = append(apiFences, dnsv1alpha1.DNSContributionFence{UID: f.UID, GrantUID: f.GrantUID, Epoch: f.Epoch, Sequence: f.Sequence, ValidUntil: metav1.NewTime(f.ValidUntil)}) + apiFences = append(apiFences, dnsv1alpha1.DNSContributionFence{UID: f.UID, GrantUID: f.GrantUID, Epoch: f.Epoch, Sequence: f.Sequence}) } apiRegistrations := make([]dnsv1alpha1.DNSRegistrationFence, 0, len(registrations)) for _, f := range registrations { @@ -548,7 +548,7 @@ func (r *Reconciler) loadPrevious(ctx context.Context, o *dnsv1alpha1.DNSPublica // fields may lose fractional seconds during API round trips, so rebuilding // from manifest metadata can make an unchanged observation appear mutated. for _, f := range wire.ObservationFences { - p.Contributions = append(p.Contributions, ContributionFence{UID: types.UID(f.ContributionUID), GrantUID: types.UID(f.GrantUID), Epoch: int64(f.WriterEpoch), Sequence: int64(f.Sequence), ValidUntil: f.ValidUntil.UTC()}) + p.Contributions = append(p.Contributions, ContributionFence{UID: types.UID(f.ContributionUID), GrantUID: types.UID(f.GrantUID), Epoch: int64(f.WriterEpoch), Sequence: int64(f.Sequence)}) } return p, nil } @@ -672,13 +672,3 @@ func chunk(b []byte, size int) [][]byte { } return out } - -func nextExpiry(p PublicationPlan, now time.Time) time.Time { - var next time.Time - for _, f := range p.Contributions { - if f.ValidUntil.After(now) && (next.IsZero() || f.ValidUntil.Before(next)) { - next = f.ValidUntil - } - } - return next -} diff --git a/internal/internaldns/controlplane/publication_reconciler_test.go b/internal/internaldns/controlplane/publication_reconciler_test.go index e47e8fc7..25ef84b6 100644 --- a/internal/internaldns/controlplane/publication_reconciler_test.go +++ b/internal/internaldns/controlplane/publication_reconciler_test.go @@ -58,14 +58,14 @@ func TestExpiredOwnershipTakeoverFencesResumedWriter(t *testing.T) { } } -func TestContributionDeadlineCannotDelayOwnershipRenewal(t *testing.T) { +func TestScheduledReconciliationCannotDelayOwnershipRenewal(t *testing.T) { now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) lease := 10 * time.Second if got := nextProjectRequeue(now, now.Add(90*time.Second), lease); got != 5*time.Second { - t.Fatalf("long-lived contribution delayed lease renewal: got %s want 5s", got) + t.Fatalf("later reconciliation delayed lease renewal: got %s want 5s", got) } if got := nextProjectRequeue(now, now.Add(2*time.Second), lease); got != 2*time.Second { - t.Fatalf("near contribution expiry was not honored: got %s want 2s", got) + t.Fatalf("earlier reconciliation was not honored: got %s want 2s", got) } } @@ -95,12 +95,10 @@ func TestQuiescentSurvivorRenewsOwnershipBeyondLeaseWindow(t *testing.T) { contribution.Namespace = "project" contribution.Status.WriterEpoch = 0 contribution.Status.Sequence = 1 - until := metav1.NewTime(now.Add(90 * time.Second)) - contribution.Status.ValidUntil = &until c := fake.NewClientBuilder().WithScheme(s).WithObjects(network, zone, assoc, reg, grant, contribution). WithStatusSubresource(&dnsv1alpha1.DNSResolverContext{}, &dnsv1alpha1.DNSResolverAccessBinding{}, &dnsv1alpha1.DNSZoneAssociation{}, &dnsv1alpha1.DNSRegistration{}, &dnsv1alpha1.DNSContributionGrant{}, &dnsv1alpha1.DNSRecordContribution{}, &dnsv1alpha1.DNSPublicationManifest{}).Build() options := ReconcilerOptions{ - ProjectUID: "project-a", SourceClusterUID: "cluster-a", ProjectNamespace: "project", PlatformNamespace: "platform", + ProjectUID: "project-a", ProjectNamespace: "project", PlatformNamespace: "platform", Identity: "writer-a", LeaseDuration: 10 * time.Second, ServingRegions: []ServingRegion{{Region: "east", Shard: "shared"}}, AddressAllocator: testAllocator{}, Now: func() time.Time { return now }, } @@ -153,12 +151,11 @@ func TestQuiescentSurvivorRenewsOwnershipBeyondLeaseWindow(t *testing.T) { func TestOwnershipTakeoverLoadsOldEpochHighWater(t *testing.T) { ctx := context.Background() now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) - deadline := now.Add(-time.Minute).Add(101582 * time.Microsecond) s := runtime.NewScheme() if err := dnsv1alpha1.AddToScheme(s); err != nil { t.Fatal(err) } - payload, err := json.Marshal(model.PublicationPlan{ZoneUID: "zone-a", Apex: "corp.internal", ObservationFences: []model.ObservationFence{{ContributionUID: "contribution-a", GrantUID: "grant-a", WriterEpoch: 7, Sequence: 19, ValidUntil: deadline}}, RRSets: []model.RRSet{}}) + payload, err := json.Marshal(model.PublicationPlan{ZoneUID: "zone-a", Apex: "corp.internal", ObservationFences: []model.ObservationFence{{ContributionUID: "contribution-a", GrantUID: "grant-a", WriterEpoch: 7, Sequence: 19}}, RRSets: []model.RRSet{}}) if err != nil { t.Fatal(err) } @@ -174,7 +171,7 @@ func TestOwnershipTakeoverLoadsOldEpochHighWater(t *testing.T) { ContentHash: model.Hash(payload), // The metadata deliberately represents the precision an API server may // retain. Recovery must use the exact immutable wire observation. - ContributionFences: []dnsv1alpha1.DNSContributionFence{{UID: "contribution-a", GrantUID: "grant-a", Epoch: 7, Sequence: 19, ValidUntil: metav1.NewTime(deadline.Truncate(time.Second))}}, + ContributionFences: []dnsv1alpha1.DNSContributionFence{{UID: "contribution-a", GrantUID: "grant-a", Epoch: 7, Sequence: 19}}, }, } owner := &dnsv1alpha1.DNSPublicationOwnership{ @@ -197,9 +194,6 @@ func TestOwnershipTakeoverLoadsOldEpochHighWater(t *testing.T) { if len(previous.Contributions) != 1 || previous.Contributions[0].Sequence != 19 { t.Fatalf("old epoch high-water fence was lost: %#v", previous) } - if !previous.Contributions[0].ValidUntil.Equal(deadline) { - t.Fatalf("wire deadline precision was lost: got %s want %s", previous.Contributions[0].ValidUntil, deadline) - } in := fixture(now) in.Previous = previous in.Contributions[0].Status.Sequence = 18 @@ -302,14 +296,14 @@ func TestReconcilerPersistsFencedMultiRegionOutbox(t *testing.T) { contrib.Namespace = "project" contrib.Status.WriterEpoch = 0 contrib.Status.Sequence = 1 - accessEast := &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "prod-east", Namespace: "project", UID: "access-east"}, Spec: dnsv1alpha1.DNSResolverAccessBindingSpec{ContextRef: dnsv1alpha1.DNSObjectReference{Name: "prod", UID: "vpc-a"}, Region: "east", QueryIdentity: dnsv1alpha1.DNSResolverQueryIdentity{Type: dnsv1alpha1.DNSResolverQueryIdentityDestinationAddress, Value: "fd53::11"}, Port: 53, Transports: []string{"UDP", "TCP"}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 1, Sequence: 1, ValidUntil: metav1.NewTime(now.Add(3 * time.Minute))}}} + accessEast := &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "prod-east", Namespace: "project", UID: "access-east"}, Spec: dnsv1alpha1.DNSResolverAccessBindingSpec{ContextRef: dnsv1alpha1.DNSObjectReference{Name: "prod", UID: "vpc-a"}, Region: "east", QueryIdentity: dnsv1alpha1.DNSResolverQueryIdentity{Type: dnsv1alpha1.DNSResolverQueryIdentityDestinationAddress, Value: "fd53::11"}, Port: 53, Transports: []string{"UDP", "TCP"}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 1, Sequence: 1}}} accessWest := accessEast.DeepCopy() accessWest.Name = "prod-west" accessWest.UID = "access-west" accessWest.Spec.Region = "west" accessWest.Spec.QueryIdentity.Value = "fd53::12" c := fake.NewClientBuilder().WithScheme(s).WithObjects(network, zone, assoc, reg, grant, contrib, accessEast, accessWest).WithStatusSubresource(&dnsv1alpha1.DNSResolverContext{}, &dnsv1alpha1.DNSResolverAccessBinding{}, &dnsv1alpha1.DNSZoneAssociation{}, &dnsv1alpha1.DNSRegistration{}, &dnsv1alpha1.DNSContributionGrant{}, &dnsv1alpha1.DNSRecordContribution{}, &dnsv1alpha1.DNSPublicationManifest{}).Build() - r := &Reconciler{Client: c, Scheme: s, Options: ReconcilerOptions{ProjectUID: "project-a", SourceClusterUID: "cluster-a", ProjectNamespace: "project", PlatformNamespace: "internal-dns-system", Identity: "compiler-a", ServingRegions: []ServingRegion{{Region: "east", Shard: "s1"}, {Region: "west", Shard: "s2"}}, AddressAllocator: testAllocator{}, Now: func() time.Time { return now }}} + r := &Reconciler{Client: c, Scheme: s, Options: ReconcilerOptions{ProjectUID: "project-a", ProjectNamespace: "project", PlatformNamespace: "internal-dns-system", Identity: "compiler-a", ServingRegions: []ServingRegion{{Region: "east", Shard: "s1"}, {Region: "west", Shard: "s2"}}, AddressAllocator: testAllocator{}, Now: func() time.Time { return now }}} for i := 0; i < 8; i++ { if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "project", Name: "project"}}); err != nil { t.Fatalf("reconcile %d: %v", i, err) diff --git a/internal/internaldns/controlplane/reconciler.go b/internal/internaldns/controlplane/reconciler.go index ebb96781..25930e98 100644 --- a/internal/internaldns/controlplane/reconciler.go +++ b/internal/internaldns/controlplane/reconciler.go @@ -20,7 +20,6 @@ import ( type ReconcilerOptions struct { ProjectUID types.UID - SourceClusterUID string ProjectNamespace string PlatformNamespace string Region string @@ -32,7 +31,6 @@ type ReconcilerOptions struct { AddressAllocator AddressAllocator PrivateZoneClassName string ServingRegions []ServingRegion - MaxAccessLease time.Duration } type ServingRegion struct { @@ -68,9 +66,6 @@ func (o *ReconcilerOptions) defaults() { o.LeaseDuration = 30 * time.Second } - if o.MaxAccessLease <= 0 { - o.MaxAccessLease = 5 * time.Minute - } if o.Now == nil { o.Now = func() time.Time { return time.Now().UTC() } } @@ -85,9 +80,7 @@ func (o ReconcilerOptions) servingRegions() []ServingRegion { type Options = ReconcilerOptions -// Reconciler compiles one project's desired DNS state. ProjectUID and -// SourceClusterUID are trusted configuration supplied by the replication -// bridge; Kubernetes namespace identity is never treated as project identity. +// Reconciler compiles records within a configured project scope. type Reconciler struct { client.Client Scheme *runtime.Scheme @@ -96,8 +89,8 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { r.Options.defaults() - if r.Options.ProjectUID == "" || r.Options.SourceClusterUID == "" { - return ctrl.Result{}, errors.New("projectUID and sourceClusterUID are required") + if r.Options.ProjectUID == "" { + return ctrl.Result{}, errors.New("projectUID is required") } ns := r.Options.ProjectNamespace if ns == "" { diff --git a/internal/internaldns/controlplane/wire.go b/internal/internaldns/controlplane/wire.go index bab5827a..c7a9e38c 100644 --- a/internal/internaldns/controlplane/wire.go +++ b/internal/internaldns/controlplane/wire.go @@ -22,7 +22,7 @@ func WirePlan(plan PublicationPlan) (model.PublicationPlan, error) { for _, contribution := range plan.Contributions { out.ObservationFences = append(out.ObservationFences, model.ObservationFence{ ContributionUID: string(contribution.UID), GrantUID: string(contribution.GrantUID), - WriterEpoch: uint64(contribution.Epoch), Sequence: uint64(contribution.Sequence), ValidUntil: contribution.ValidUntil, + WriterEpoch: uint64(contribution.Epoch), Sequence: uint64(contribution.Sequence), }) } sort.Slice(out.ObservationFences, func(i, j int) bool { @@ -56,7 +56,6 @@ func WirePlan(plan PublicationPlan) (model.PublicationPlan, error) { if f.UID == uid { record.WriterEpoch = uint64(f.Epoch) record.Sequence = uint64(f.Sequence) - record.ValidUntil = f.ValidUntil break } } diff --git a/internal/internaldns/model/model.go b/internal/internaldns/model/model.go index aef57230..039ac1c9 100644 --- a/internal/internaldns/model/model.go +++ b/internal/internaldns/model/model.go @@ -17,7 +17,7 @@ import ( "github.com/miekg/dns" ) -const APIVersion = "internal.dns.networking.miloapis.com/v1alpha3" +const APIVersion = "internal.dns.networking.miloapis.com/v1alpha4" type Kind string @@ -78,9 +78,8 @@ const ( ) type Authorization struct { - IssuerEpoch uint64 `json:"issuerEpoch"` - Revision uint64 `json:"revision"` - ValidUntil time.Time `json:"validUntil"` + IssuerEpoch uint64 `json:"issuerEpoch"` + Revision uint64 `json:"revision"` } type Backend struct { @@ -192,7 +191,7 @@ func (s ServingSnapshot) Validate() error { } seenVIP[raw] = b.BindingUID } - if b.Port == 0 || len(b.Transports) == 0 || b.Authorization.IssuerEpoch == 0 || b.Authorization.Revision == 0 || b.Authorization.ValidUntil.IsZero() { + if b.Port == 0 || len(b.Transports) == 0 || b.Authorization.IssuerEpoch == 0 || b.Authorization.Revision == 0 { return fmt.Errorf("binding %q has incomplete transport or authorization", b.BindingUID) } if b.ConfigurationPending && len(b.Zones) != 0 { @@ -323,12 +322,11 @@ func (c PublicationChunk) Validate() error { } type RRRecord struct { - Content string `json:"content"` - ContributionUID string `json:"contributionUID,omitempty"` - WriterEpoch uint64 `json:"writerEpoch,omitempty"` - Sequence uint64 `json:"sequence,omitempty"` - Eligible bool `json:"eligible"` - ValidUntil time.Time `json:"validUntil,omitempty"` + Content string `json:"content"` + ContributionUID string `json:"contributionUID,omitempty"` + WriterEpoch uint64 `json:"writerEpoch,omitempty"` + Sequence uint64 `json:"sequence,omitempty"` + Eligible bool `json:"eligible"` } type RRSet struct { @@ -352,15 +350,12 @@ type RegistrationFence struct { Generation uint64 `json:"generation"` } -// ObservationFence carries the accepted high-water mark even when an -// unhealthy or withdrawn observation produces no records. ValidUntil is the -// original contributor deadline and is part of the immutable fence identity. +// ObservationFence carries the accepted high-water mark, including withdrawn observations. type ObservationFence struct { - ContributionUID string `json:"contributionUID"` - GrantUID string `json:"grantUID"` - WriterEpoch uint64 `json:"writerEpoch"` - Sequence uint64 `json:"sequence"` - ValidUntil time.Time `json:"validUntil"` + ContributionUID string `json:"contributionUID"` + GrantUID string `json:"grantUID"` + WriterEpoch uint64 `json:"writerEpoch"` + Sequence uint64 `json:"sequence"` } func (p PublicationPlan) Validate() error { @@ -384,7 +379,7 @@ func (p PublicationPlan) Validate() error { observationByUID := make(map[string]ObservationFence, len(p.ObservationFences)) previousContribution := "" for _, observation := range p.ObservationFences { - if observation.ContributionUID == "" || observation.GrantUID == "" || observation.WriterEpoch == 0 || observation.Sequence == 0 || observation.ValidUntil.IsZero() { + if observation.ContributionUID == "" || observation.GrantUID == "" || observation.WriterEpoch == 0 || observation.Sequence == 0 { return errors.New("publication observation fence is incomplete") } if previousContribution != "" && observation.ContributionUID <= previousContribution { @@ -433,12 +428,12 @@ func (p PublicationPlan) Validate() error { return fmt.Errorf("RRset %q contains an invalid record", key) } if rec.ContributionUID == "" { - if rec.WriterEpoch != 0 || rec.Sequence != 0 || !rec.ValidUntil.IsZero() { + if rec.WriterEpoch != 0 || rec.Sequence != 0 { return fmt.Errorf("RRset %q static record contains contribution fence data", key) } } else { observation, ok := observationByUID[rec.ContributionUID] - if !ok || rec.WriterEpoch != observation.WriterEpoch || rec.Sequence != observation.Sequence || !rec.ValidUntil.Equal(observation.ValidUntil) { + if !ok || rec.WriterEpoch != observation.WriterEpoch || rec.Sequence != observation.Sequence { return fmt.Errorf("RRset %q record fence does not match contribution observation %q", key, rec.ContributionUID) } } diff --git a/internal/internaldns/model/model_test.go b/internal/internaldns/model/model_test.go index 8cdef63c..5365bf09 100644 --- a/internal/internaldns/model/model_test.go +++ b/internal/internaldns/model/model_test.go @@ -42,7 +42,7 @@ func TestServingSnapshotRejectsSharedDestination(t *testing.T) { t.Parallel() now := time.Now() b := func(uid, vpc string) Binding { - return Binding{BindingUID: uid, ProjectUID: "project", ContextUID: vpc, BindingGeneration: 1, ConfigurationRevision: 1, ConsumerAddress: "2001:db8::1", ClusterAddress: "2001:db8::2", Port: 53, Transports: []Transport{TransportUDP, TransportTCP}, Authorization: Authorization{IssuerEpoch: 1, Revision: 1, ValidUntil: now.Add(time.Minute)}} + return Binding{BindingUID: uid, ProjectUID: "project", ContextUID: vpc, BindingGeneration: 1, ConfigurationRevision: 1, ConsumerAddress: "2001:db8::1", ClusterAddress: "2001:db8::2", Port: 53, Transports: []Transport{TransportUDP, TransportTCP}, Authorization: Authorization{IssuerEpoch: 1, Revision: 1}} } s := ServingSnapshot{Region: "east", Shard: "s1", ConfigurationEpoch: 1, ConfigurationRevision: 1, GeneratedAt: now, Bindings: []Binding{b("one", "vpc-a"), b("two", "vpc-b")}} if err := s.Validate(); err == nil { @@ -94,7 +94,7 @@ func TestEnvelopeRejectsObsoleteWireVersion(t *testing.T) { func TestPendingConfigurationCannotExposePartialZoneAttachments(t *testing.T) { now := time.Now().UTC() - binding := Binding{BindingUID: "binding", ProjectUID: "project", ContextUID: "context", BindingGeneration: 1, ConfigurationRevision: 1, ConsumerAddress: "fd53::a", ClusterAddress: "fd54::a", Port: 53, Transports: []Transport{TransportUDP}, Authorization: Authorization{IssuerEpoch: 1, Revision: 1, ValidUntil: now.Add(time.Minute)}, ConfigurationPending: true, Zones: []ZoneAttachment{{ZoneUID: "zone", Apex: "private.internal", RequiredPublicationEpoch: 1, RequiredPublicationRevision: 1}}} + binding := Binding{BindingUID: "binding", ProjectUID: "project", ContextUID: "context", BindingGeneration: 1, ConfigurationRevision: 1, ConsumerAddress: "fd53::a", ClusterAddress: "fd54::a", Port: 53, Transports: []Transport{TransportUDP}, Authorization: Authorization{IssuerEpoch: 1, Revision: 1}, ConfigurationPending: true, Zones: []ZoneAttachment{{ZoneUID: "zone", Apex: "private.internal", RequiredPublicationEpoch: 1, RequiredPublicationRevision: 1}}} snapshot := ServingSnapshot{Region: "east", Shard: "shared", ConfigurationEpoch: 1, ConfigurationRevision: 1, GeneratedAt: now, Bindings: []Binding{binding}} if err := snapshot.Validate(); err == nil { t.Fatal("partial pending context accepted") diff --git a/internal/internaldns/platform/acks.go b/internal/internaldns/platform/acks.go index 28b6fd1c..321db0cc 100644 --- a/internal/internaldns/platform/acks.go +++ b/internal/internaldns/platform/acks.go @@ -161,10 +161,6 @@ func (s *AckSink) serving(ctx context.Context, a model.MemberAck, row dnsv1alpha bindingRow.SnapshotRevision = int64(a.Revision) bindingRow.AuthorizationIssuerEpoch = binding.Spec.Authorization.WriterEpoch bindingRow.AuthorizationRevision = binding.Spec.Authorization.Sequence - if bindingRow.ValidUntil != nil && bindingRow.ValidUntil.After(b.Authorization.ValidUntil) { - u := metav1.NewTime(b.Authorization.ValidUntil) - bindingRow.ValidUntil = &u - } binding.Status.MemberAcknowledgements = upsertAck(binding.Status.MemberAcknowledgements, bindingRow) complete := s.bindingVerified(binding, e, snapshot, now) if complete { @@ -283,7 +279,7 @@ func (s *AckSink) projectPublicationForScope(ctx context.Context, project Projec continue } for _, record := range rrset.Records { - if record.Eligible && (record.ContributionUID == "" || record.ValidUntil.After(now)) { + if record.Eligible { available = true } } @@ -385,7 +381,7 @@ func hasApplied(rows []dnsv1alpha1.DNSApplyAcknowledgement, epoch, revision int6 return false } -// Refresh withdraws status readiness when a member lease or authorization expires, +// Refresh withdraws status readiness when member acknowledgment freshness expires, // even if its process or event stream has stopped producing acknowledgements. func (s *AckSink) Refresh(ctx context.Context) error { now := time.Now().UTC() @@ -528,7 +524,7 @@ func LoadCommittedServingSnapshot(ctx context.Context, c client.Client, namespac } func (s *AckSink) bindingVerified(b *dnsv1alpha1.DNSResolverBinding, envelope *model.Envelope, snapshot *model.ServingSnapshot, now time.Time) bool { - if envelope == nil || snapshot == nil || b.Spec.Tombstone || !b.Spec.Authorization.ValidUntil.After(now) { + if envelope == nil || snapshot == nil || b.Spec.Tombstone { return false } present := false diff --git a/internal/internaldns/platform/planner.go b/internal/internaldns/platform/planner.go index e5c974b6..e79cca26 100644 --- a/internal/internaldns/platform/planner.go +++ b/internal/internaldns/platform/planner.go @@ -145,7 +145,7 @@ func (p *Planner) Step(ctx context.Context) error { if b.Configuration.Listeners.Node.Port != 53 || b.Configuration.Listeners.Regional.Port != 53 || !reflect.DeepEqual(b.Configuration.Listeners.Node.Transports, b.Configuration.Listeners.Regional.Transports) { return fmt.Errorf("binding %s requires matching DNS listener ports and transports", obj.Name) } - view := model.Binding{BindingUID: string(obj.UID), ProjectUID: string(b.Source.ProjectUID), ContextUID: string(b.Source.ResolverContextRef.UID), BindingGeneration: uint64(b.Configuration.Generation), ConfigurationRevision: uint64(b.Configuration.Revision), ConsumerAddress: b.Configuration.Listeners.Node.Address, ClusterAddress: b.Configuration.Listeners.Regional.Address, Port: uint16(b.Configuration.Listeners.Node.Port), NodeBackends: c.NodeBackends, ClusterBackends: c.ClusterBackends, Authorization: model.Authorization{IssuerEpoch: uint64(b.Authorization.WriterEpoch), Revision: uint64(b.Authorization.Sequence), ValidUntil: b.Authorization.ValidUntil.Time}} + view := model.Binding{BindingUID: string(obj.UID), ProjectUID: string(b.Source.ProjectUID), ContextUID: string(b.Source.ResolverContextRef.UID), BindingGeneration: uint64(b.Configuration.Generation), ConfigurationRevision: uint64(b.Configuration.Revision), ConsumerAddress: b.Configuration.Listeners.Node.Address, ClusterAddress: b.Configuration.Listeners.Regional.Address, Port: uint16(b.Configuration.Listeners.Node.Port), NodeBackends: c.NodeBackends, ClusterBackends: c.ClusterBackends, Authorization: model.Authorization{IssuerEpoch: uint64(b.Authorization.WriterEpoch), Revision: uint64(b.Authorization.Sequence)}} for _, t := range b.Configuration.Listeners.Node.Transports { view.Transports = append(view.Transports, model.Transport(t)) } diff --git a/internal/internaldns/platform/platform_test.go b/internal/internaldns/platform/platform_test.go index b14b1767..1adaec72 100644 --- a/internal/internaldns/platform/platform_test.go +++ b/internal/internaldns/platform/platform_test.go @@ -114,7 +114,7 @@ func TestSharedPlannerPreservesOtherProjects(t *testing.T) { mname := fmt.Sprintf("manifest-%d", i) objects = append(objects, &dnsv1alpha1.DNSPublicationOwnership{ObjectMeta: metav1.ObjectMeta{Name: "owner-" + model.OpaqueToken(string(zuid))[:20], Namespace: ns}, Spec: dnsv1alpha1.DNSPublicationOwnershipSpec{ZoneUID: zuid, WriterEpoch: 1, ActiveManifestName: mname}}) objects = append(objects, &dnsv1alpha1.DNSPublicationManifest{ObjectMeta: metav1.ObjectMeta{Name: mname, Namespace: ns}, Spec: dnsv1alpha1.DNSPublicationManifestSpec{ZoneRef: dnsv1alpha1.DNSObjectReference{UID: zuid}, ZoneApex: "prod.internal", WriterEpoch: 1, Revision: 1, ServingTargets: []dnsv1alpha1.DNSPublicationTarget{{Region: region, Shard: shard}}}}) - objects = append(objects, &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("binding-%d", i), Namespace: ns, UID: types.UID(fmt.Sprintf("binding-uid-%d", i)), Labels: map[string]string{"internal-dns.miloapis.com/region": region, "internal-dns.miloapis.com/shard": shard}}, Spec: dnsv1alpha1.DNSResolverBindingSpec{Source: dnsv1alpha1.DNSResolverBindingSource{ProjectUID: types.UID(fmt.Sprintf("project-%d", i)), ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: string(vuid), UID: vuid}}, Placement: dnsv1alpha1.DNSResolverBindingPlacement{Region: region, Shard: shard}, Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Generation: 1, Revision: 1, Listeners: dnsv1alpha1.DNSResolverBindingListeners{Node: dnsv1alpha1.DNSResolverBindingListener{Address: fmt.Sprintf("fd53::%d", i), Port: 53, Transports: []string{"UDP", "TCP"}}, Regional: dnsv1alpha1.DNSResolverBindingListener{Address: fmt.Sprintf("fd54::%d", i), Port: 53, Transports: []string{"UDP", "TCP"}}}, ZoneRefs: zoneReferences([]types.UID{zuid})}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 1, Sequence: 1, ValidUntil: metav1.NewTime(now.Add(time.Minute))}}}) + objects = append(objects, &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("binding-%d", i), Namespace: ns, UID: types.UID(fmt.Sprintf("binding-uid-%d", i)), Labels: map[string]string{"internal-dns.miloapis.com/region": region, "internal-dns.miloapis.com/shard": shard}}, Spec: dnsv1alpha1.DNSResolverBindingSpec{Source: dnsv1alpha1.DNSResolverBindingSource{ProjectUID: types.UID(fmt.Sprintf("project-%d", i)), ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: string(vuid), UID: vuid}}, Placement: dnsv1alpha1.DNSResolverBindingPlacement{Region: region, Shard: shard}, Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Generation: 1, Revision: 1, Listeners: dnsv1alpha1.DNSResolverBindingListeners{Node: dnsv1alpha1.DNSResolverBindingListener{Address: fmt.Sprintf("fd53::%d", i), Port: 53, Transports: []string{"UDP", "TCP"}}, Regional: dnsv1alpha1.DNSResolverBindingListener{Address: fmt.Sprintf("fd54::%d", i), Port: 53, Transports: []string{"UDP", "TCP"}}}, ZoneRefs: zoneReferences([]types.UID{zuid})}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 1, Sequence: 1}}}) } cl := testClient(t, objects...) p := Planner{Client: cl, Allocator: &Allocator{Client: cl, Namespace: ns}, Config: PlannerConfig{Namespace: ns, Region: region, Shard: shard, Identity: "planner", ClusterBackends: []model.Backend{{MemberID: "cluster-1", Address: "fd00::20", Port: 5300}}}, Now: func() time.Time { return now }} @@ -159,7 +159,7 @@ func TestSharedPlannerPreservesOtherProjects(t *testing.T) { if len(pending.Bindings) != 2 || !pending.Bindings[0].ConfigurationPending || pending.Bindings[0].Tombstone || len(pending.Bindings[0].Zones) != 0 || pending.Bindings[1].ConfigurationPending { t.Fatal("incomplete zone addition withdrew authorization, exposed partial zones, or gated the healthy project") } - if pending.Bindings[0].Authorization.Revision != uint64(originalAuthorization.Sequence) || !pending.Bindings[0].Authorization.ValidUntil.Equal(originalAuthorization.ValidUntil.Time) { + if pending.Bindings[0].Authorization.Revision != uint64(originalAuthorization.Sequence) { t.Fatal("pending configuration renewed the original authorization") } for _, obj := range []client.Object{ @@ -381,7 +381,7 @@ func TestReadinessCannotMixAuthorizationRenewals(t *testing.T) { func TestReadinessExpiresWithoutAnotherAck(t *testing.T) { now := time.Now().UTC() until := metav1.NewTime(now.Add(-time.Second)) - b := &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{Name: "binding", Namespace: "platform", Labels: map[string]string{"internal-dns.miloapis.com/region": "east", "internal-dns.miloapis.com/shard": "shared"}}, Spec: dnsv1alpha1.DNSResolverBindingSpec{Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Revision: 1}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 1, Sequence: 1, ValidUntil: metav1.NewTime(now.Add(time.Minute))}}, Status: dnsv1alpha1.DNSResolverBindingStatus{Phase: "Serving", MemberAcknowledgements: []dnsv1alpha1.DNSApplyAcknowledgement{{MemberID: "resolver", WriterEpoch: 1, Revision: 1, AuthorizationIssuerEpoch: 1, AuthorizationRevision: 1, Phase: string(model.AckVerified), ValidUntil: &until}}}} + b := &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{Name: "binding", Namespace: "platform", Labels: map[string]string{"internal-dns.miloapis.com/region": "east", "internal-dns.miloapis.com/shard": "shared"}}, Spec: dnsv1alpha1.DNSResolverBindingSpec{Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Revision: 1}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 1, Sequence: 1}}, Status: dnsv1alpha1.DNSResolverBindingStatus{Phase: "Serving", MemberAcknowledgements: []dnsv1alpha1.DNSApplyAcknowledgement{{MemberID: "resolver", WriterEpoch: 1, Revision: 1, AuthorizationIssuerEpoch: 1, AuthorizationRevision: 1, Phase: string(model.AckVerified), ValidUntil: &until}}}} cl := testClient(t, b) s := AckSink{Client: cl, Namespace: "platform", Region: "east", Shard: "shared", Members: []model.Member{{MemberID: "resolver", Role: "resolver"}}, Now: func() time.Time { return now }} if err := s.Refresh(context.Background()); err != nil { @@ -455,7 +455,7 @@ func TestOutOfOrderAckDoesNotReplaceWithdrawalInSameSecond(t *testing.T) { func TestBindingReadinessRequiresCurrentCommittedSnapshot(t *testing.T) { now := time.Now().UTC() until := metav1.NewTime(now.Add(time.Minute)) - b := &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{UID: "binding-uid"}, Spec: dnsv1alpha1.DNSResolverBindingSpec{Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Generation: 1, Revision: 7}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 3, Sequence: 5, ValidUntil: until}}, Status: dnsv1alpha1.DNSResolverBindingStatus{MemberAcknowledgements: []dnsv1alpha1.DNSApplyAcknowledgement{ + b := &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{UID: "binding-uid"}, Spec: dnsv1alpha1.DNSResolverBindingSpec{Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Generation: 1, Revision: 7}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 3, Sequence: 5}}, Status: dnsv1alpha1.DNSResolverBindingStatus{MemberAcknowledgements: []dnsv1alpha1.DNSApplyAcknowledgement{ {MemberID: "node", WriterEpoch: 2, SnapshotRevision: 8, Revision: 7, AuthorizationIssuerEpoch: 3, AuthorizationRevision: 5, Phase: string(model.AckVerified), ValidUntil: &until}, {MemberID: "cluster", WriterEpoch: 2, SnapshotRevision: 8, Revision: 7, AuthorizationIssuerEpoch: 3, AuthorizationRevision: 5, Phase: string(model.AckVerified), ValidUntil: &until}, }}} diff --git a/internal/internaldns/runtime/config.go b/internal/internaldns/runtime/config.go index 66077f55..b07f539e 100644 --- a/internal/internaldns/runtime/config.go +++ b/internal/internaldns/runtime/config.go @@ -27,12 +27,12 @@ const ( ) type ProjectConfig struct { - Name string `json:"name,omitempty"` - ProjectUID types.UID `json:"projectUID"` - SourceClusterUID string `json:"sourceClusterUID"` - Namespace string `json:"namespace"` - Kubeconfig string `json:"kubeconfig,omitempty"` - KubeAPI KubeAPIClientConfig `json:"kubeAPI,omitempty"` + Name string `json:"name,omitempty"` + ProjectUID types.UID `json:"projectUID"` + + Namespace string `json:"namespace"` + Kubeconfig string `json:"kubeconfig,omitempty"` + KubeAPI KubeAPIClientConfig `json:"kubeAPI,omitempty"` } type KubeAPIClientConfig struct { @@ -42,11 +42,9 @@ type KubeAPIClientConfig struct { } type AdmissionConfig struct { - Enabled bool `json:"enabled"` - Port int `json:"port,omitempty"` - CertDir string `json:"certDir,omitempty"` - MaxContributionLeaseSeconds int64 `json:"maxContributionLeaseSeconds,omitempty"` - MaxAccessLeaseSeconds int64 `json:"maxAccessLeaseSeconds,omitempty"` + Enabled bool `json:"enabled"` + Port int `json:"port,omitempty"` + CertDir string `json:"certDir,omitempty"` } type RegionalTransportConfig struct { @@ -115,13 +113,6 @@ func (c *Config) defaults() { if c.Admission.Port == 0 { c.Admission.Port = 9443 } - if c.Admission.MaxContributionLeaseSeconds <= 0 { - c.Admission.MaxContributionLeaseSeconds = 90 - } - if c.Admission.MaxAccessLeaseSeconds <= 0 { - c.Admission.MaxAccessLeaseSeconds = 300 - } - if c.NATS.Name == "" { c.NATS.Name = "internal-dns" } @@ -209,8 +200,8 @@ func (c *Config) Validate(role string) error { uids := map[types.UID]bool{} namespaces := map[string]bool{} for i, p := range c.Projects { - if p.ProjectUID == "" || p.SourceClusterUID == "" || p.Namespace == "" { - return fmt.Errorf("project %d requires projectUID, sourceClusterUID, and namespace", i) + if p.ProjectUID == "" || p.Namespace == "" { + return fmt.Errorf("project %d requires projectUID and namespace", i) } if p.Namespace == c.PlatformNamespace { return fmt.Errorf("project %d namespace collides with platformNamespace", i) @@ -274,10 +265,6 @@ func (c *Config) Validate(role string) error { return nil } -func (c AdmissionConfig) maxAccessLease() time.Duration { - return time.Duration(c.MaxAccessLeaseSeconds) * time.Second -} - func validateTransportConfig(nats transport.Config, stream transport.StreamConfig, ensure bool) error { if nats.URL == "" || stream.Name == "" { return errors.New("nats.url and stream.name are required") @@ -341,6 +328,3 @@ func (c KubeAPIClientConfig) requestTimeout() time.Duration { func (c Config) ownershipLease() time.Duration { return time.Duration(c.OwnershipLeaseSeconds) * time.Second } -func (c AdmissionConfig) maxLease() time.Duration { - return time.Duration(c.MaxContributionLeaseSeconds) * time.Second -} diff --git a/internal/internaldns/runtime/config_test.go b/internal/internaldns/runtime/config_test.go index 629e668e..a3a1c433 100644 --- a/internal/internaldns/runtime/config_test.go +++ b/internal/internaldns/runtime/config_test.go @@ -41,7 +41,7 @@ func TestKubeAPIRateLimitDefaultsOverridesAndValidation(t *testing.T) { Region: "central", Shard: "one", NATS: transport.Config{URL: "nats://fixture:4222", AllowInsecure: true}, Stream: transport.StreamConfig{Name: "DNS", SnapshotsBucket: "DNS_BOOTSTRAP"}, - Projects: []ProjectConfig{{Name: "project", ProjectUID: "p1", SourceClusterUID: "c1", Namespace: "project", + Projects: []ProjectConfig{{Name: "project", ProjectUID: "p1", Namespace: "project", KubeAPI: KubeAPIClientConfig{QPS: 30, Burst: 60}}}, Members: []model.Member{{MemberID: "regional-1", Role: "regional"}}, } @@ -102,7 +102,7 @@ func TestControlPlaneRejectsAmbiguousProjectsAndMemberIDs(t *testing.T) { base := Config{ NATS: transport.Config{URL: "nats://fixture:4222", AllowInsecure: true}, Stream: transport.StreamConfig{Name: "DNS", SnapshotsBucket: "DNS_BOOTSTRAP"}, - Projects: []ProjectConfig{{ProjectUID: "p1", SourceClusterUID: "c1", Namespace: "n1"}, {ProjectUID: "p2", SourceClusterUID: "c2", Namespace: "n2"}}, + Projects: []ProjectConfig{{ProjectUID: "p1", Namespace: "n1"}, {ProjectUID: "p2", Namespace: "n2"}}, } if err := base.Validate("control-plane"); err == nil { t.Fatal("unnamed multi-project configuration was accepted") @@ -152,3 +152,56 @@ func TestConfigLoaderRejectsRemovedPrivateServingFields(t *testing.T) { t.Fatal("multiple config objects accepted") } } + +func TestAgentRestoresCheckpointWhileBrokerUnavailable(t *testing.T) { + dir := t.TempDir() + cfg := Config{Region: "central", Shard: "one", NATS: transport.Config{URL: "nats://127.0.0.1:1", AllowInsecure: true}, Stream: transport.StreamConfig{Name: "DNS", SnapshotsBucket: "DNS_BOOTSTRAP"}, Agent: serving.Config{MemberID: "node-0", StateDir: filepath.Join(dir, "state"), Mode: serving.ModeResolver, Render: serving.RenderConfig{Role: serving.RenderNode, NodeDNSDist: serving.DNSDistRenderConfig{Path: filepath.Join(dir, "dnsdist.conf"), ListenAddress: "[::1]:53", PoolName: "node"}, NodeBIND: serving.BINDRenderConfig{Path: filepath.Join(dir, "named.conf"), ListenAddress: "::1", Port: 5353}}, Transaction: serving.TransactionConfig{StageDir: filepath.Join(dir, "stage")}, UnsafeAllowNoFailClosed: true, UnsafeAllowNoDNSVerify: true, WatchdogLeasePath: filepath.Join(dir, "lease.json"), ReconcileInterval: 10 * time.Millisecond}} + if err := cfg.Validate(roleAgent); err != nil { + t.Fatal(err) + } + seed, err := serving.NewAgent(cfg.Agent, serving.Dependencies{}) + if err != nil { + t.Fatal(err) + } + snapshot := model.ServingSnapshot{Region: "central", Shard: "one", ConfigurationEpoch: 1, ConfigurationRevision: 1, GeneratedAt: time.Now(), Bindings: []model.Binding{{BindingUID: "binding", ProjectUID: "project", ContextUID: "context", BindingGeneration: 1, ConfigurationRevision: 1, ConsumerAddress: "fd53::1", ClusterAddress: "fd53::2", Port: 53, Transports: []model.Transport{model.TransportUDP, model.TransportTCP}, Authorization: model.Authorization{IssuerEpoch: 1, Revision: 1}}}} + env, err := model.NewEnvelope(model.KindServingSnapshot, "initial", "central", "one", "shard", 1, 1, time.Now(), snapshot) + if err != nil { + t.Fatal(err) + } + if err := seed.Handle(context.Background(), env); err != nil { + t.Fatal(err) + } + if err := os.Remove(cfg.Agent.WatchdogLeasePath); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- runAgent(ctx, cfg) }() + deadline := time.Now().Add(3 * time.Second) + for { + if _, err := os.Stat(cfg.Agent.WatchdogLeasePath); err == nil { + break + } + if time.Now().After(deadline) { + t.Fatal("checkpoint activation waited for broker connectivity") + } + time.Sleep(10 * time.Millisecond) + } + data, err := os.ReadFile(cfg.Agent.Render.NodeDNSDist.Path) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), "fd53::1") || !strings.Contains(string(data), "ready=true") { + t.Fatalf("persisted resolver mapping was not restored: %s", data) + } + cancel() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(3 * time.Second): + t.Fatal("agent did not stop") + } +} diff --git a/internal/internaldns/runtime/run.go b/internal/internaldns/runtime/run.go index eb72d02d..8c9e2192 100644 --- a/internal/internaldns/runtime/run.go +++ b/internal/internaldns/runtime/run.go @@ -117,13 +117,12 @@ func runControlPlane(ctx context.Context, cfg Config, kubeconfig string) error { // contexts, access bindings, associations, registrations, or producer observations. routed := &RoutingClient{Client: platformClient, Source: direct, PlatformNamespace: cfg.PlatformNamespace} reconciler := &controlplane.Reconciler{Client: routed, Scheme: scheme, Options: controlplane.ReconcilerOptions{ - ProjectUID: project.ProjectUID, SourceClusterUID: project.SourceClusterUID, + ProjectUID: project.ProjectUID, ProjectNamespace: project.Namespace, PlatformNamespace: cfg.PlatformNamespace, Region: cfg.Region, Shard: cfg.Shard, Identity: identity, LeaseDuration: cfg.ownershipLease(), AddressAllocator: allocator, PrivateZoneClassName: cfg.PrivateZoneClassName, ServingRegions: regions, - MaxAccessLease: cfg.Admission.maxAccessLease(), }} if err := reconciler.SetupWithManager(mgr); err != nil { return fmt.Errorf("setup reconciler: %w", err) @@ -146,9 +145,7 @@ func runControlPlane(ctx context.Context, cfg Config, kubeconfig string) error { return fmt.Errorf("create admission manager: %w", err) } handler := &dnsadmission.Handler{ - MaxContributionLease: cfg.Admission.maxLease(), - MaxAccessLease: cfg.Admission.maxAccessLease(), - ClientForRequest: projectSelector(projects), + ClientForRequest: projectSelector(projects), } admissionManager.GetWebhookServer().Register(dnsadmission.Path, &kubeadmission.Webhook{Handler: handler}) } @@ -218,7 +215,7 @@ func runProjectSource(ctx context.Context, project projectRuntime) { err = errors.New("source manager stopped unexpectedly") } log.Error(err, "source manager unavailable; rebuilding and retrying") - if !waitForRetry(ctx, time.Second) { + if !waitForRetry(ctx) { return } } @@ -246,7 +243,7 @@ func runControlPlaneRegion(ctx context.Context, cfg Config, region string, plann return nil } log.Error(err, "regional transport setup failed; retrying") - if !waitForRetry(ctx, time.Second) { + if !waitForRetry(ctx) { return nil } continue @@ -274,7 +271,7 @@ func runControlPlaneRegion(ctx context.Context, cfg Config, region string, plann } if setupFailed { regional.Close() - if !waitForRetry(ctx, time.Second) { + if !waitForRetry(ctx) { return nil } continue @@ -294,8 +291,8 @@ func runControlPlaneRegion(ctx context.Context, cfg Config, region string, plann return nil } -func waitForRetry(ctx context.Context, delay time.Duration) bool { - timer := time.NewTimer(delay) +func waitForRetry(ctx context.Context) bool { + timer := time.NewTimer(time.Second) defer timer.Stop() select { case <-ctx.Done(): @@ -306,45 +303,79 @@ func waitForRetry(ctx context.Context, delay time.Duration) bool { } func runAgent(ctx context.Context, cfg Config) error { - regional, err := connectRegional(ctx, cfg, []string{cfg.Agent.Region}) + log := ctrl.LoggerFrom(ctx).WithName("internal-dns-agent") + publisher := &agentPublisher{} + agent, err := serving.NewAgent(cfg.Agent, serving.Dependencies{AckPublisher: publisher, OnError: func(err error) { log.Error(err, "serving agent operation failed; retrying") }}) if err != nil { return err } - defer regional.Close() - route, _ := regional.route(cfg.Agent.Region) - consumer := func(suffix string, filters ...string) (*transport.Consumer, error) { - return route.bus.Consumer(ctx, transport.ConsumerConfig{Stream: route.stream.Name, - Durable: durable(cfg.Agent.MemberID, suffix), FilterSubjects: filters, - AckWait: 30 * time.Second, MaxAckPending: 64}) + workerCtx, cancel := context.WithCancel(ctx) + defer cancel() + done := make(chan error, 1) + go func() { done <- agent.Start(workerCtx) }() + go func() { + for workerCtx.Err() == nil { + err := runAgentTransport(workerCtx, cfg, agent, publisher) + if workerCtx.Err() != nil { + return + } + if err != nil { + log.Error(err, "DNS state delivery unavailable; retrying") + } + if !waitForRetry(workerCtx) { + return + } + } + }() + select { + case <-ctx.Done(): + return <-done + case err := <-done: + return err } - servingSource, err := consumer("serving", model.ServingSubject(cfg.Agent.Region, cfg.Agent.Shard)) +} + +// agentPublisher does not block local activation while the transport connects. +type agentPublisher struct { + mu sync.RWMutex + delegate transport.ExportPublisher +} + +func (p *agentPublisher) set(delegate transport.ExportPublisher) { + p.mu.Lock() + defer p.mu.Unlock() + p.delegate = delegate +} +func (p *agentPublisher) Publish(ctx context.Context, subject string, env model.Envelope) (transport.PublishAck, error) { + p.mu.RLock() + delegate := p.delegate + p.mu.RUnlock() + if delegate == nil { + return transport.PublishAck{}, errors.New("DNS state transport unavailable") + } + return delegate.Publish(ctx, subject, env) +} +func runAgentTransport(ctx context.Context, cfg Config, agent *serving.Agent, publisher *agentPublisher) error { + regional, err := connectRegional(ctx, cfg, []string{cfg.Agent.Region}) if err != nil { return err } - var recordSource, ackSource *transport.Consumer - if cfg.Agent.Mode == serving.ModeResolver || cfg.Agent.Mode == serving.ModeCombined { - recordSource, err = consumer("records", model.RecordConsumerFilter(cfg.Agent.Region, cfg.Agent.Shard)) - if err != nil { - return err - } + defer regional.Close() + route, _ := regional.route(cfg.Agent.Region) + consumer := func(suffix string, filters ...string) (*transport.Consumer, error) { + return route.bus.Consumer(ctx, transport.ConsumerConfig{Stream: route.stream.Name, Durable: durable(cfg.Agent.MemberID, suffix), FilterSubjects: filters, AckWait: 30 * time.Second, MaxAckPending: 64}) } - if cfg.Agent.Mode == serving.ModeResolver || cfg.Agent.Mode == serving.ModeCombined { - ackSource, err = consumer("acks", ackFilter(cfg.Agent.Region, cfg.Agent.Shard)) + var sources []serving.EventSource + for _, feed := range []struct{ suffix, filter string }{{"serving", model.ServingSubject(cfg.Agent.Region, cfg.Agent.Shard)}, {"records", model.RecordConsumerFilter(cfg.Agent.Region, cfg.Agent.Shard)}, {"acks", ackFilter(cfg.Agent.Region, cfg.Agent.Shard)}} { + source, err := consumer(feed.suffix, feed.filter) if err != nil { return err } + sources = append(sources, source) } - log := ctrl.LoggerFrom(ctx).WithName("internal-dns-agent") - agent, err := serving.NewAgent(cfg.Agent, serving.Dependencies{ - ServingSource: servingSource, RecordSource: recordSource, AckSource: ackSource, - Bootstrap: serving.SnapshotBootstrap{Store: route.store, Region: cfg.Agent.Region, Shard: cfg.Agent.Shard}, - AckPublisher: route.bus, - OnError: func(err error) { log.Error(err, "serving agent operation failed; retrying") }, - }) - if err != nil { - return err - } - return agent.Start(ctx) + publisher.set(route.bus) + defer publisher.set(nil) + return agent.RunTransport(ctx, serving.SnapshotBootstrap{Store: route.store, Region: cfg.Agent.Region, Shard: cfg.Agent.Shard}, sources...) } func newScheme() (*runtime.Scheme, error) { @@ -437,25 +468,25 @@ func consumeAcks(ctx context.Context, source *transport.Consumer, sink *platform } } -func projectSelector(projects []projectRuntime) func(context.Context, kubeadmission.Request) (client.Client, string, error) { - return func(_ context.Context, req kubeadmission.Request) (client.Client, string, error) { +func projectSelector(projects []projectRuntime) func(context.Context, kubeadmission.Request) (client.Client, error) { + return func(_ context.Context, req kubeadmission.Request) (client.Client, error) { parents := req.UserInfo.Extra[parentNameExtra] if len(parents) == 0 && len(projects) == 1 { p := projects[0] if req.Namespace != p.config.Namespace { - return nil, "", fmt.Errorf("request namespace is outside the configured project") + return nil, fmt.Errorf("request namespace is outside the configured project") } - return p.direct, p.config.SourceClusterUID, nil + return p.direct, nil } if len(parents) != 1 { - return nil, "", fmt.Errorf("authenticated project parent is missing or ambiguous") + return nil, fmt.Errorf("authenticated project parent is missing or ambiguous") } for _, p := range projects { if p.config.Name == parents[0] && req.Namespace == p.config.Namespace { - return p.direct, p.config.SourceClusterUID, nil + return p.direct, nil } } - return nil, "", fmt.Errorf("authenticated project parent is not configured for this namespace") + return nil, fmt.Errorf("authenticated project parent is not configured for this namespace") } } diff --git a/internal/internaldns/serving/agent.go b/internal/internaldns/serving/agent.go index 94ada8c0..c61d0d13 100644 --- a/internal/internaldns/serving/agent.go +++ b/internal/internaldns/serving/agent.go @@ -37,12 +37,11 @@ type Config struct { Render RenderConfig `json:"render"` Transaction TransactionConfig `json:"transaction"` CacheFlush []Command `json:"cacheFlush,omitempty"` - ExpiryInterval time.Duration `json:"expiryInterval,omitempty"` + ReconcileInterval time.Duration `json:"reconcileInterval,omitempty"` RetryDelay time.Duration `json:"retryDelay,omitempty"` AckInterval time.Duration `json:"ackInterval,omitempty"` AckLease time.Duration `json:"ackLease,omitempty"` AckPublishTimeout time.Duration `json:"ackPublishTimeout,omitempty"` - LocalServingLease time.Duration `json:"localServingLease,omitempty"` MaxBindings int `json:"maxBindings,omitempty"` WatchdogLeasePath string `json:"watchdogLeasePath,omitempty"` WatchdogLease time.Duration `json:"watchdogLease,omitempty"` @@ -201,8 +200,7 @@ type Agent struct { installedPublications map[string]installedPublication } -// This identity describes the exact files successfully activated, not a -// fingerprint recomputed later when an observation deadline may have passed. +// installedPublication identifies the exact files activated by a validated transaction. type installedPublication struct { Identity string EffectiveHash string @@ -255,13 +253,12 @@ func NewAgent(c Config, d Dependencies) (*Agent, error) { } func (a *Agent) Start(ctx context.Context) error { - defer func() { _ = a.tx.FailClosed(context.Background()) }() - // A restarted regional materialization must enforce deadlines persisted in its local - // checkpoint before waiting on any remote service. The watchdog lease is not - // renewed here, so an inconsistent or unavailable bootstrap stays gated. - if err := a.expirePersisted(ctx); err != nil { - a.report(fmt.Errorf("enforce persisted expiry at startup: %w", err)) - } + defer func() { + a.mu.Lock() + defer a.mu.Unlock() + _ = a.failClosed(context.Background()) + }() + // Verify the persisted configuration locally before serving after restart. if !a.gateUntilClosed(ctx) { return nil } @@ -280,9 +277,6 @@ func (a *Agent) Start(ctx context.Context) error { return nil } a.report(err) - if !a.gateUntilClosed(ctx) { - return nil - } retry := a.cfg.RetryDelay if retry <= 0 { retry = time.Second @@ -303,7 +297,7 @@ func (a *Agent) Start(ctx context.Context) error { if a.cfg.Mode == ModeResolver || a.cfg.Mode == ModeCombined { start(a.deps.AckSource) } - interval := a.cfg.ExpiryInterval + interval := a.cfg.ReconcileInterval if interval <= 0 { interval = time.Second } @@ -341,7 +335,10 @@ func (a *Agent) gateUntilClosed(ctx context.Context) bool { retry = time.Second } for { - if err := a.failClosed(ctx); err == nil { + a.mu.Lock() + err := a.failClosed(ctx) + a.mu.Unlock() + if err == nil { return true } else { a.report(fmt.Errorf("gate serving during startup: %w", err)) @@ -352,40 +349,10 @@ func (a *Agent) gateUntilClosed(ctx context.Context) bool { } } -func (a *Agent) expirePersisted(ctx context.Context) error { - if a.cfg.Mode != ModeCombined { - return nil - } - a.mu.Lock() - defer a.mu.Unlock() - return a.expireRecords(ctx) -} - func (a *Agent) bootstrapAndStep(ctx context.Context) error { if a.deps.Bootstrap != nil { - envs, err := a.deps.Bootstrap.Load(ctx) - if err != nil { - return fmt.Errorf("load bootstrap snapshot: %w", err) - } - sort.SliceStable(envs, func(i, j int) bool { - order := func(k model.Kind) int { - switch k { - case model.KindPublicationChunk: - return 0 - case model.KindServingSnapshot: - // Context destination mappings are required to query-proof a - // publication before its manifest can be acknowledged. - return 1 - default: - return 2 - } - } - return order(envs[i].Kind) < order(envs[j].Kind) - }) - for _, env := range envs { - if err := a.Handle(ctx, env); err != nil { - return fmt.Errorf("apply bootstrap %s: %w", env.Kind, err) - } + if err := a.loadBootstrap(ctx, a.deps.Bootstrap); err != nil { + return err } } if err := a.Step(ctx); err != nil { @@ -394,6 +361,49 @@ func (a *Agent) bootstrapAndStep(ctx context.Context) error { return nil } +func (a *Agent) loadBootstrap(ctx context.Context, loader BootstrapSource) error { + envs, err := loader.Load(ctx) + if err != nil { + return fmt.Errorf("load bootstrap snapshot: %w", err) + } + order := func(k model.Kind) int { + switch k { + case model.KindPublicationChunk: + return 0 + case model.KindServingSnapshot: + return 1 + default: + return 2 + } + } + sort.SliceStable(envs, func(i, j int) bool { return order(envs[i].Kind) < order(envs[j].Kind) }) + for _, env := range envs { + if err := a.Handle(ctx, env); err != nil { + return fmt.Errorf("apply bootstrap %s: %w", env.Kind, err) + } + } + return nil +} + +// RunTransport synchronizes desired state independently of local DNS serving. +func (a *Agent) RunTransport(ctx context.Context, loader BootstrapSource, sources ...EventSource) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + if err := a.loadBootstrap(ctx, loader); err != nil { + return err + } + results := make(chan error, len(sources)) + for _, source := range sources { + go func() { results <- a.consume(ctx, source) }() + } + select { + case <-ctx.Done(): + return ctx.Err() + case err := <-results: + return err + } +} + func (a *Agent) consume(ctx context.Context, source EventSource) error { retry := a.cfg.RetryDelay if retry <= 0 { @@ -454,6 +464,9 @@ func (a *Agent) Handle(ctx context.Context, env model.Envelope) error { } a.mu.Lock() defer a.mu.Unlock() + if err := ctx.Err(); err != nil { + return err + } switch env.Kind { case model.KindServingSnapshot: return a.handleServing(ctx, env) @@ -609,7 +622,7 @@ func (a *Agent) handleManifest(ctx context.Context, env model.Envelope) error { return errors.New("conflicting publication manifest at the same epoch and revision") } if err := a.verifyPublication(ctx, old); err != nil { - _ = a.emitAck(ctx, model.KindPublicationManifest, m.ZoneUID, m.WriterEpoch, m.Revision, model.AckRejected, time.Time{}, err.Error()) + _ = a.emitAck(ctx, model.KindPublicationManifest, m.ZoneUID, m.WriterEpoch, m.Revision, model.AckRejected, err.Error()) return err } for _, name := range publicationNames(old.Plan) { @@ -660,7 +673,7 @@ func (a *Agent) handleManifest(ctx context.Context, env model.Envelope) error { if err := a.store.Save(a.state); err != nil { return err } - next := publicationState{Fence: fence{m.WriterEpoch, m.Revision}, Manifest: m, Plan: plan, EffectiveHash: effectivePlanHash(plan, a.deps.Clock())} + next := publicationState{Fence: fence{m.WriterEpoch, m.Revision}, Manifest: m, Plan: plan, EffectiveHash: effectivePlanHash(plan)} if a.cfg.Mode == ModeResolver { return a.recordResolverPublication(ctx, m.ZoneUID, next, old) } @@ -781,7 +794,7 @@ func (a *Agent) handleAck(ctx context.Context, env model.Envelope) error { if cmp < 0 || (cmp == 0 && !ack.ObservedAt.After(old.ObservedAt)) { return nil } - a.state.Acks[ack.ResourceUID][key] = replicaAck{Fence: fence{ack.Epoch, ack.Revision}, ObservedAt: ack.ObservedAt} + a.state.Acks[ack.ResourceUID][key] = replicaAck{Fence: fence{ack.Epoch, ack.Revision}, ObservedAt: ack.ObservedAt, Rejected: true} if err := a.store.Save(a.state); err != nil { return err } @@ -829,25 +842,20 @@ func (a *Agent) handleAck(ctx context.Context, env model.Envelope) error { func (a *Agent) Step(ctx context.Context) error { a.mu.Lock() defer a.mu.Unlock() + if a.state.Snapshot == nil { + return errors.New("no verified serving configuration is available") + } if a.cfg.Mode == ModeResolver { if err := a.retryPendingPublications(ctx); err != nil { _ = a.failClosed(ctx) return err } - if err := a.expireResolverCache(ctx); err != nil { - _ = a.failClosed(ctx) - return err - } } if a.cfg.Mode == ModeCombined { if err := a.retryPendingPublications(ctx); err != nil { _ = a.failClosed(ctx) return err } - if err := a.expireRecords(ctx); err != nil { - _ = a.failClosed(ctx) - return err - } if a.needsActivation || len(a.installedPublications) == 0 { if err := a.reconcile(ctx); err != nil { return err @@ -860,12 +868,6 @@ func (a *Agent) Step(ctx context.Context) error { if err := a.reconcile(ctx); err != nil { return err } - if a.cfg.Mode == ModeCombined { - if err := a.expireRecords(ctx); err != nil { - _ = a.failClosed(ctx) - return err - } - } return a.renewSafeWatchdogLease(ctx) } @@ -874,17 +876,7 @@ func (a *Agent) Step(ctx context.Context) error { // lease. This matters when a sustained JetStream backlog repeatedly acquires // the agent lock and delays the ticker. func (a *Agent) finishEvent(ctx context.Context) error { - if a.cfg.Mode == ModeResolver { - if err := a.expireResolverCache(ctx); err != nil { - _ = a.failClosed(ctx) - return err - } - } if a.cfg.Mode == ModeCombined { - if err := a.expireRecords(ctx); err != nil { - _ = a.failClosed(ctx) - return err - } if err := a.heartbeatPublications(ctx); err != nil { return err } @@ -897,29 +889,15 @@ func (a *Agent) reconcile(ctx context.Context) error { return nil } - now := a.deps.Clock() active := map[string]bool{} filtered := *a.state.Snapshot filtered.Bindings = nil for _, b := range a.state.Snapshot.Bindings { - eligible := !b.Tombstone && now.Before(b.Authorization.ValidUntil) && a.publicationsReady(b) + eligible := !b.Tombstone && a.publicationsReady(b) if eligible { active[b.BindingUID] = true } - if !b.Tombstone && now.Before(b.Authorization.ValidUntil) { - lease := a.cfg.LocalServingLease - if lease <= 0 { - lease = 90 * time.Second - } - horizon := now.Add(lease) - quantum := a.cfg.AckInterval - if quantum <= 0 { - quantum = 30 * time.Second - } - horizon = horizon.Truncate(quantum).Add(quantum) - if horizon.Before(b.Authorization.ValidUntil) { - b.Authorization.ValidUntil = horizon - } + if !b.Tombstone { filtered.Bindings = append(filtered.Bindings, b) } } @@ -928,7 +906,7 @@ func (a *Agent) reconcile(ctx context.Context) error { for _, binding := range filtered.Bindings { renderConfig.ReadyMembers[binding.BindingUID] = a.eligibleRegionalMembers(binding) } - rendered, err := renderPublications(renderConfig, filtered, active, a.state.Publications, now) + rendered, err := renderPublications(renderConfig, filtered, active, a.state.Publications) if err != nil { return err } @@ -953,7 +931,7 @@ func (a *Agent) reconcile(ctx context.Context) error { if a.cfg.Mode == ModeCombined { installed := map[string]installedPublication{} for uid, p := range a.state.Publications { - installed[uid] = installedPublication{Identity: publicationIdentity(p), EffectiveHash: effectivePlanHash(p.Plan, now), Views: a.publicationViewsAt(p, now)} + installed[uid] = installedPublication{Identity: publicationIdentity(p), EffectiveHash: effectivePlanHash(p.Plan), Views: a.publicationViewsAt(p)} } a.installedPublications = installed for _, p := range a.state.Publications { @@ -970,7 +948,7 @@ func (a *Agent) reconcile(ctx context.Context) error { // Any binding that fails either UDP or TCP is immediately removed again. probeFailed := a.verifyResolverBindings(ctx, filtered.Bindings, active) if probeFailed { - gated, err := renderPublications(renderConfig, filtered, active, a.state.Publications, now) + gated, err := renderPublications(renderConfig, filtered, active, a.state.Publications) if err != nil { return err } @@ -986,13 +964,7 @@ func (a *Agent) reconcile(ctx context.Context) error { } } } - // Resolver health is a local safety decision. Once every eligible binding - // has passed its end-to-end proof (and every failed binding has been - // durably re-rendered gated), renew the independent watchdog before any - // best-effort broker ACK. A NATS partition must expire remote readiness - // leases, but it must not consume the local five-second watchdog window for - // a dataplane that has just been proved healthy. Combined mode renews later, - // after its regional materialization expiry and health checks also complete. + // Local DNS proof renews the watchdog independently of broker acknowledgements. if a.cfg.Mode == ModeResolver { if err := a.renewSafeWatchdogLease(ctx); err != nil { return err @@ -1005,9 +977,6 @@ func (a *Agent) reconcile(ctx context.Context) error { p := model.AckVerified if !active[b.BindingUID] { p = model.AckApplied - if !now.Before(b.Authorization.ValidUntil) { - p = model.AckExpired - } } if err := a.emitServingAck(ctx, b, p); err != nil { a.report(fmt.Errorf("publish serving ACK %s: %w", b.BindingUID, err)) @@ -1150,7 +1119,7 @@ func (a *Agent) eligibleRegionalMembers(b model.Binding) map[string]bool { } } proof := a.state.Acks[z.ZoneUID][member.MemberID+"/"+member.ReplicaID] - if !a.deps.Clock().Before(proof.ValidUntil) || proof.Fence != p.Fence { + if proof.Rejected || proof.ObservedAt.IsZero() || proof.Fence != p.Fence { ready = false break } @@ -1180,21 +1149,21 @@ func (a *Agent) verifyPublication(ctx context.Context, p publicationState) error } func (a *Agent) publicationViews(p publicationState) []PublicationView { - return a.publicationViewsAt(p, a.deps.Clock()) + return a.publicationViewsAt(p) } -func (a *Agent) publicationViewsAt(p publicationState, now time.Time) []PublicationView { +func (a *Agent) publicationViewsAt(p publicationState) []PublicationView { if a.state.Snapshot == nil { return nil } var out []PublicationView for _, b := range a.state.Snapshot.Bindings { - if b.Tombstone || b.ConfigurationPending || !now.Before(b.Authorization.ValidUntil) { + if b.Tombstone || b.ConfigurationPending { continue } for _, z := range b.Zones { if z.ZoneUID == p.Plan.ZoneUID && model.Compare(p.Fence.Epoch, p.Fence.Revision, z.RequiredPublicationEpoch, z.RequiredPublicationRevision) >= 0 { - out = append(out, PublicationView{View: b.ViewName(), DestinationAddress: b.ClusterAddress, DestinationPort: b.Port, Serial: publicationSerial(p, now), FingerprintMailbox: publicationMailbox(p, now)}) + out = append(out, PublicationView{View: b.ViewName(), DestinationAddress: b.ClusterAddress, DestinationPort: b.Port, Serial: publicationSerial(p), FingerprintMailbox: publicationMailbox(p)}) } } } @@ -1227,84 +1196,11 @@ func (a *Agent) retryPendingPublications(ctx context.Context) error { } } p.Verified = true - p.EffectiveHash = effectivePlanHash(p.Plan, a.deps.Clock()) - a.state.Publications[uid] = p - if err := a.store.Save(a.state); err != nil { - return err - } - } - return nil -} - -// A node retains the publication deadlines so a broker outage cannot leave -// healthy records in its positive cache after the regional member withdraws. -func (a *Agent) expireResolverCache(ctx context.Context) error { - now := a.deps.Clock() - for uid, p := range a.state.Publications { - if p.Manifest.Tombstone { - continue - } - hash := effectivePlanHash(p.Plan, now) - if hash == p.EffectiveHash { - continue - } - for _, name := range publicationNames(p.Plan) { - if err := a.flushName(ctx, name); err != nil { - return err - } - } - p.EffectiveHash = hash - p.LocalRevision++ - a.state.Publications[uid] = p - if err := a.store.Save(a.state); err != nil { - return err - } - } - return nil -} - -func (a *Agent) expireRecords(ctx context.Context) error { - now := a.deps.Clock() - for uid, p := range a.state.Publications { - if !p.Verified || p.Manifest.Tombstone { - continue - } - hash := effectivePlanHash(p.Plan, now) - if hash == p.EffectiveHash { - continue - } - p.LocalRevision++ - p.EffectiveHash = hash - p.Verified = false - a.state.Publications[uid] = p - if err := a.store.Save(a.state); err != nil { - return err - } - if err := a.reconcile(ctx); err != nil { - return err - } - if len(a.publicationViews(p)) == 0 { - continue - } - if err := a.verifyPublication(ctx, p); err != nil { - return err - } - for _, name := range publicationNames(p.Plan) { - if err := a.flushName(ctx, name); err != nil { - return err - } - } - p.Verified = true + p.EffectiveHash = effectivePlanHash(p.Plan) a.state.Publications[uid] = p if err := a.store.Save(a.state); err != nil { return err } - if err := a.reconcile(ctx); err != nil { - return err - } - if err := a.emitAck(ctx, model.KindPublicationManifest, uid, p.Fence.Epoch, p.Fence.Revision, model.AckExpired, time.Time{}, ""); err != nil { - a.report(err) - } } return nil } @@ -1341,7 +1237,7 @@ func (a *Agent) flushPublicationZone(ctx context.Context, zoneUID string) error return nil } -func (a *Agent) emitAck(ctx context.Context, kind model.Kind, uid string, epoch, revision uint64, phase model.AckPhase, validUntil time.Time, message string) error { +func (a *Agent) emitAck(ctx context.Context, kind model.Kind, uid string, epoch, revision uint64, phase model.AckPhase, message string) error { if a.deps.AckPublisher == nil { return nil } @@ -1351,9 +1247,6 @@ func (a *Agent) emitAck(ctx context.Context, kind model.Kind, uid string, epoch, lease = 90 * time.Second } leaseUntil := now.Add(lease) - if !validUntil.IsZero() && validUntil.Before(leaseUntil) { - leaseUntil = validUntil - } ack := model.MemberAck{MemberID: a.cfg.MemberID, ReplicaID: a.cfg.ReplicaID, ResourceUID: uid, Kind: kind, Epoch: epoch, Revision: revision, Phase: phase, ObservedAt: now, ValidUntil: leaseUntil, Error: message} eventID := fmt.Sprintf("ack-%s-%s-%d-%d-%s-%d", model.OpaqueToken(a.cfg.MemberID+"/"+a.cfg.ReplicaID), model.OpaqueToken(uid), epoch, revision, phase, now.UnixNano()) env, err := model.NewEnvelope(model.KindMemberAck, eventID, a.cfg.Region, a.cfg.Shard, uid, epoch, revision, a.deps.Clock(), ack) @@ -1371,13 +1264,11 @@ func (a *Agent) emitAck(ctx context.Context, kind model.Kind, uid string, epoch, } func (a *Agent) emitVerifiedPublicationAck(ctx context.Context, uid string, epoch, revision uint64) error { - // The file transaction, proof and original-deadline enforcement are local - // safety gates. Broker delivery is independent: renew only a safe installed - // state first, then let remote readiness expire if ACK transport is down. + // Renew local readiness before best-effort acknowledgement delivery. if err := a.renewSafeWatchdogLease(ctx); err != nil { return err } - if err := a.emitAck(ctx, model.KindPublicationManifest, uid, epoch, revision, model.AckVerified, time.Time{}, ""); err != nil { + if err := a.emitAck(ctx, model.KindPublicationManifest, uid, epoch, revision, model.AckVerified, ""); err != nil { a.report(fmt.Errorf("publish verified publication ACK %s: %w", uid, err)) return nil } @@ -1398,7 +1289,7 @@ func (a *Agent) emitServingAck(ctx context.Context, b model.Binding, phase model if last.phase == phase && now.Sub(last.at) < interval { return nil } - if err := a.emitAck(ctx, model.KindServingSnapshot, b.BindingUID, a.state.ServingFence.Epoch, a.state.ServingFence.Revision, phase, b.Authorization.ValidUntil, ""); err != nil { + if err := a.emitAck(ctx, model.KindServingSnapshot, b.BindingUID, a.state.ServingFence.Epoch, a.state.ServingFence.Revision, phase, ""); err != nil { return err } a.lastServingAck[b.BindingUID] = servingAckState{phase: phase, at: now} @@ -1424,18 +1315,15 @@ func (a *Agent) heartbeatPublications(ctx context.Context) error { continue } if err := a.verifyPublication(ctx, p); err != nil { - _ = a.emitAck(ctx, model.KindPublicationManifest, uid, p.Fence.Epoch, p.Fence.Revision, model.AckRejected, time.Time{}, err.Error()) + _ = a.emitAck(ctx, model.KindPublicationManifest, uid, p.Fence.Epoch, p.Fence.Revision, model.AckRejected, err.Error()) return fmt.Errorf("reverify publication %s before ACK renewal: %w", uid, err) } due = append(due, duePublication{uid: uid, p: p}) } - // Local watchdog eligibility depends on the complete local verification - // above, not broker availability. Stop after the first stalled publish so a - // partition consumes at most one bounded publish timeout per Step. Missing - // remote ACK leases still expire independently and gate resolver eligibility. + // Publish acknowledgements within a bounded budget after local DNS verification. for _, publication := range due { p := publication.p - if err := a.emitAck(ctx, model.KindPublicationManifest, publication.uid, p.Fence.Epoch, p.Fence.Revision, model.AckVerified, time.Time{}, ""); err != nil { + if err := a.emitAck(ctx, model.KindPublicationManifest, publication.uid, p.Fence.Epoch, p.Fence.Revision, model.AckVerified, ""); err != nil { a.report(fmt.Errorf("publish verified publication ACK %s: %w", publication.uid, err)) break } @@ -1506,9 +1394,6 @@ func (a *Agent) validateContributionFences(plan model.PublicationPlan) error { return fmt.Errorf("contribution %q regressed from epoch %d sequence %d", observation.ContributionUID, current.Epoch, current.Sequence) } if comparison == 0 { - if current.ValidUntil.IsZero() || !current.ValidUntil.Equal(observation.ValidUntil) { - return fmt.Errorf("contribution %q changed the deadline of an existing fence", observation.ContributionUID) - } if current.GrantUID != "" && current.GrantUID != observation.GrantUID { return fmt.Errorf("contribution %q changed the grant of an existing fence", observation.ContributionUID) } @@ -1519,10 +1404,10 @@ func (a *Agent) validateContributionFences(plan model.PublicationPlan) error { func (a *Agent) commitContributionFences(plan model.PublicationPlan) { for _, observation := range plan.ObservationFences { - next := contributionFence{GrantUID: observation.GrantUID, Epoch: observation.WriterEpoch, Sequence: observation.Sequence, ValidUntil: observation.ValidUntil} + next := contributionFence{GrantUID: observation.GrantUID, Epoch: observation.WriterEpoch, Sequence: observation.Sequence} current := a.state.ContributionFences[observation.ContributionUID] comparison := model.Compare(next.Epoch, next.Sequence, current.Epoch, current.Sequence) - if comparison > 0 || (comparison == 0 && current.GrantUID == "" && !current.ValidUntil.IsZero() && current.ValidUntil.Equal(next.ValidUntil)) { + if comparison > 0 || (comparison == 0 && current.GrantUID == "") { a.state.ContributionFences[observation.ContributionUID] = next } } @@ -1535,61 +1420,26 @@ type watchdogLease struct { } func (a *Agent) watchdogPublicationError() error { - now := a.deps.Clock() for uid, p := range a.state.Publications { if p.Manifest.Tombstone { continue } - current := effectivePlanHash(p.Plan, now) + current := effectivePlanHash(p.Plan) if a.cfg.Mode == ModeCombined { - if len(a.publicationViewsAt(p, now)) == 0 { + if len(a.publicationViewsAt(p)) == 0 { continue } installed, ok := a.installedPublications[uid] if !ok || installed.Identity != publicationIdentity(p) || installed.EffectiveHash != current { - return fmt.Errorf("publication %s deadline changed after activation; refusing watchdog renewal", uid) + return fmt.Errorf("publication %s configuration differs from activation; refusing watchdog renewal", uid) } } else if p.EffectiveHash != current { - return fmt.Errorf("publication %s deadline changed after cache withdrawal; refusing watchdog renewal", uid) + return fmt.Errorf("publication %s configuration differs from the applied checkpoint; refusing watchdog renewal", uid) } } return nil } -// A deadline can cross during reload or DNS proof. Settle that transition -// locally before renewing readiness, under one budget across every retry. -func (a *Agent) renewSafeWatchdogLease(ctx context.Context) error { - timeout := a.cfg.PublicationDNSProbe.Timeout - if a.cfg.Mode == ModeResolver { - timeout = a.cfg.ResolverDNSProbe.Timeout - } - if timeout <= 0 { - timeout = 3 * time.Second - } - settleCtx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - for { - if err := settleCtx.Err(); err != nil { - return errors.Join(fmt.Errorf("settle local publication deadlines: %w", err), a.failClosed(ctx)) - } - var err error - if a.cfg.Mode == ModeCombined { - err = a.expireRecords(settleCtx) - } else { - err = a.expireResolverCache(settleCtx) - } - if err != nil { - return errors.Join(err, a.failClosed(ctx)) - } - if a.watchdogPublicationError() == nil { - return a.renewWatchdogLease(settleCtx) - } - if err := a.reconcile(settleCtx); err != nil { - return errors.Join(err, a.failClosed(ctx)) - } - } -} - func (a *Agent) renewWatchdogLease(ctx context.Context) error { if err := ctx.Err(); err != nil { return errors.Join(err, a.failClosed(ctx)) @@ -1610,3 +1460,11 @@ func (a *Agent) renewWatchdogLease(ctx context.Context) error { } return writeAtomicLease(a.cfg.WatchdogLeasePath, b, 0o600) } + +// Renew the local watchdog only when the activated publication matches its checkpoint. +func (a *Agent) renewSafeWatchdogLease(ctx context.Context) error { + if err := a.watchdogPublicationError(); err != nil { + return errors.Join(err, a.failClosed(ctx)) + } + return a.renewWatchdogLease(ctx) +} diff --git a/internal/internaldns/serving/agent_test.go b/internal/internaldns/serving/agent_test.go index 9db5bd59..a892d334 100644 --- a/internal/internaldns/serving/agent_test.go +++ b/internal/internaldns/serving/agent_test.go @@ -160,14 +160,13 @@ func envelope(t *testing.T, kind model.Kind, uid string, epoch, revision uint64, } func testSnapshot(now time.Time) model.ServingSnapshot { - b := model.Binding{BindingUID: "binding-a", ProjectUID: "project-a", ContextUID: "vpc-a", BindingGeneration: 1, ConfigurationRevision: 1, ConsumerAddress: "10.253.0.51", ClusterAddress: "10.253.0.41", Port: 53, Transports: []model.Transport{model.TransportUDP, model.TransportTCP}, Zones: []model.ZoneAttachment{{ZoneUID: "zone-a", Apex: "prod.internal", RequiredPublicationEpoch: 2, RequiredPublicationRevision: 4}}, Authorization: model.Authorization{IssuerEpoch: 1, Revision: 1, ValidUntil: now.Add(10 * time.Minute)}} + b := model.Binding{BindingUID: "binding-a", ProjectUID: "project-a", ContextUID: "vpc-a", BindingGeneration: 1, ConfigurationRevision: 1, ConsumerAddress: "10.253.0.51", ClusterAddress: "10.253.0.41", Port: 53, Transports: []model.Transport{model.TransportUDP, model.TransportTCP}, Zones: []model.ZoneAttachment{{ZoneUID: "zone-a", Apex: "prod.internal", RequiredPublicationEpoch: 2, RequiredPublicationRevision: 4}}, Authorization: model.Authorization{IssuerEpoch: 1, Revision: 1}} return model.ServingSnapshot{Region: "east", Shard: "s1", ConfigurationEpoch: 1, ConfigurationRevision: 1, GeneratedAt: now, Bindings: []model.Binding{b}, Members: []model.Member{{MemberID: "regional-0", ReplicaID: "regional-0", Role: "regional"}}} } func testPublication(t *testing.T, now time.Time) (model.PublicationChunk, model.PublicationManifest) { t.Helper() - validUntil := now.Add(time.Minute) - plan := model.PublicationPlan{ZoneUID: "zone-a", Apex: "prod.internal", ObservationFences: []model.ObservationFence{{ContributionUID: "contrib-a", GrantUID: "grant-a", WriterEpoch: 1, Sequence: 1, ValidUntil: validUntil}}, Owners: []string{"api.prod.internal"}, RRSets: []model.RRSet{{Name: "api.prod.internal", Type: "A", TTL: 30, Records: []model.RRRecord{{Content: "192.0.2.10", ContributionUID: "contrib-a", WriterEpoch: 1, Sequence: 1, Eligible: true, ValidUntil: validUntil}}}}} + plan := model.PublicationPlan{ZoneUID: "zone-a", Apex: "prod.internal", ObservationFences: []model.ObservationFence{{ContributionUID: "contrib-a", GrantUID: "grant-a", WriterEpoch: 1, Sequence: 1}}, Owners: []string{"api.prod.internal"}, RRSets: []model.RRSet{{Name: "api.prod.internal", Type: "A", TTL: 30, Records: []model.RRRecord{{Content: "192.0.2.10", ContributionUID: "contrib-a", WriterEpoch: 1, Sequence: 1, Eligible: true}}}}} payload, err := json.Marshal(plan) if err != nil { t.Fatal(err) @@ -179,23 +178,21 @@ func testPublication(t *testing.T, now time.Time) (model.PublicationChunk, model func TestWithdrawnObservationFencesOlderHealthyRecord(t *testing.T) { t.Parallel() - now := time.Now().UTC() agent := &Agent{state: newCheckpoint()} withdrawn := model.PublicationPlan{ ZoneUID: "zone-a", Apex: "prod.internal", - ObservationFences: []model.ObservationFence{{ContributionUID: "contrib-a", GrantUID: "grant-a", WriterEpoch: 7, Sequence: 33, ValidUntil: now.Add(-time.Minute)}}, + ObservationFences: []model.ObservationFence{{ContributionUID: "contrib-a", GrantUID: "grant-a", WriterEpoch: 7, Sequence: 33}}, RRSets: []model.RRSet{}, } if err := withdrawn.Validate(); err != nil { t.Fatal(err) } agent.commitContributionFences(withdrawn) - olderDeadline := now.Add(time.Minute) olderHealthy := model.PublicationPlan{ ZoneUID: "zone-a", Apex: "prod.internal", - ObservationFences: []model.ObservationFence{{ContributionUID: "contrib-a", GrantUID: "grant-a", WriterEpoch: 7, Sequence: 32, ValidUntil: olderDeadline}}, + ObservationFences: []model.ObservationFence{{ContributionUID: "contrib-a", GrantUID: "grant-a", WriterEpoch: 7, Sequence: 32}}, RRSets: []model.RRSet{{Name: "api.prod.internal", Type: "A", TTL: 30, Records: []model.RRRecord{{ - Content: "192.0.2.10", ContributionUID: "contrib-a", WriterEpoch: 7, Sequence: 32, Eligible: true, ValidUntil: olderDeadline, + Content: "192.0.2.10", ContributionUID: "contrib-a", WriterEpoch: 7, Sequence: 32, Eligible: true, }}}}, } if err := olderHealthy.Validate(); err != nil { @@ -206,17 +203,16 @@ func TestWithdrawnObservationFencesOlderHealthyRecord(t *testing.T) { } } -func TestObservationFenceReplayCannotExtendDeadline(t *testing.T) { +func TestObservationFenceReplayCannotChangeGrant(t *testing.T) { t.Parallel() - now := time.Now().UTC() agent := &Agent{state: newCheckpoint()} - original := model.PublicationPlan{ZoneUID: "zone-a", Apex: "prod.internal", ObservationFences: []model.ObservationFence{{ContributionUID: "contrib-a", GrantUID: "grant-a", WriterEpoch: 2, Sequence: 4, ValidUntil: now}}, RRSets: []model.RRSet{}} + original := model.PublicationPlan{ZoneUID: "zone-a", Apex: "prod.internal", ObservationFences: []model.ObservationFence{{ContributionUID: "contrib-a", GrantUID: "grant-a", WriterEpoch: 2, Sequence: 4}}, RRSets: []model.RRSet{}} agent.commitContributionFences(original) replay := original replay.ObservationFences = append([]model.ObservationFence(nil), original.ObservationFences...) - replay.ObservationFences[0].ValidUntil = now.Add(time.Hour) + replay.ObservationFences[0].GrantUID = "another-grant" if err := agent.validateContributionFences(replay); err == nil { - t.Fatal("equal observation fence extended its original deadline") + t.Fatal("equal observation fence changed its grant") } } @@ -277,7 +273,7 @@ func TestStartForceActivatesPersistedUnchangedConfigBeforeLease(t *testing.T) { config := Config{ Region: "east", Shard: "s1", MemberID: "node-0", StateDir: stateDir, Mode: ModeResolver, Render: renderConfig, UnsafeAllowNoDNSVerify: true, ExternalWatchdog: true, - WatchdogLeasePath: leasePath, WatchdogLease: 5 * time.Second, ExpiryInterval: time.Hour, + WatchdogLeasePath: leasePath, WatchdogLease: 5 * time.Second, ReconcileInterval: time.Hour, Transaction: TransactionConfig{ StageDir: filepath.Join(dir, "stage"), FailClosed: []Command{{Path: "gate"}}, @@ -347,13 +343,19 @@ func TestStepBoundsStalledAckPublishAndRenewsLocalLease(t *testing.T) { Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage")}, UnsafeAllowNoDNSVerify: true, ExternalWatchdog: true, WatchdogLeasePath: leasePath, WatchdogLease: 5 * time.Second, AckPublishTimeout: 30 * time.Millisecond, - }, Dependencies{AckPublisher: stalledPublisher{}, Runner: &recordingRunner{}, OnError: func(err error) { reported = append(reported, err) }}) + }, Dependencies{Runner: &recordingRunner{}, OnError: func(err error) { reported = append(reported, err) }}) if err != nil { t.Fatal(err) } agent.state.Snapshot = &snapshot agent.state.BindingFences, _ = advanceBindingFences(nil, snapshot) agent.state.ServingFence = fence{Epoch: 1, Revision: 1} + // Complete durable activation before measuring the ACK publication bound. + if err := agent.Step(context.Background()); err != nil { + t.Fatal(err) + } + agent.deps.AckPublisher = stalledPublisher{} + agent.lastServingAck = make(map[string]servingAckState) started := time.Now() if err := agent.Step(context.Background()); err != nil { t.Fatal(err) @@ -512,7 +514,6 @@ func TestStartContinuouslyRenewsResolverLeaseWhileBrokerDisconnected(t *testing. renderConfig.ClusterDNSDist = DNSDistRenderConfig{} renderConfig.ClusterBIND = BINDRenderConfig{} snapshot.Bindings[0].Zones = nil - snapshot.Bindings[0].Authorization.ValidUntil = time.Now().Add(10 * time.Second) leasePath := filepath.Join(dir, "lease.json") publishStarted := make(chan struct{}, 1) publishRelease := make(chan struct{}) @@ -522,8 +523,8 @@ func TestStartContinuouslyRenewsResolverLeaseWhileBrokerDisconnected(t *testing. Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage")}, UnsafeAllowNoFailClosed: true, UnsafeAllowNoDNSVerify: true, ExternalWatchdog: true, WatchdogLeasePath: leasePath, WatchdogLease: 500 * time.Millisecond, - ExpiryInterval: 20 * time.Millisecond, AckInterval: 50 * time.Millisecond, - AckPublishTimeout: 10 * time.Millisecond, LocalServingLease: time.Second, + ReconcileInterval: 20 * time.Millisecond, AckInterval: 50 * time.Millisecond, + AckPublishTimeout: 10 * time.Millisecond, }, Dependencies{AckPublisher: publisher, Runner: &recordingRunner{}}) if err != nil { t.Fatal(err) diff --git a/internal/internaldns/serving/bind_test.go b/internal/internaldns/serving/bind_test.go index a12b1fc7..d0dfb6b7 100644 --- a/internal/internaldns/serving/bind_test.go +++ b/internal/internaldns/serving/bind_test.go @@ -47,7 +47,7 @@ func installTestPublication(t *testing.T, a *Agent, now time.Time) { } func currentZone(t *testing.T, a *Agent) string { t.Helper() - files, err := renderPublications(a.cfg.Render, *a.state.Snapshot, map[string]bool{}, a.state.Publications, a.deps.Clock()) + files, err := renderPublications(a.cfg.Render, *a.state.Snapshot, map[string]bool{}, a.state.Publications) if err != nil { t.Fatal(err) } @@ -70,7 +70,7 @@ func TestBINDPublicationActivatesLocalPrivateAuthority(t *testing.T) { } installTestPublication(t, agent, now) data := currentZone(t, agent) - if !strings.Contains(data, "192.0.2.10") || !strings.Contains(data, publicationMailbox(agent.state.Publications["zone-a"], now)) { + if !strings.Contains(data, "192.0.2.10") || !strings.Contains(data, publicationMailbox(agent.state.Publications["zone-a"])) { t.Fatalf("bad zone: %s", data) } config, err := os.ReadFile(cfg.Render.ClusterBIND.Path) @@ -84,7 +84,7 @@ func TestBINDPublicationActivatesLocalPrivateAuthority(t *testing.T) { t.Fatal("publication not verified") } } -func TestBINDExpirySurvivesBrokerOutageAndRestart(t *testing.T) { +func TestBINDPublicationSurvivesBrokerOutageAndRestart(t *testing.T) { t.Parallel() now := time.Now().UTC() clock := now @@ -98,12 +98,12 @@ func TestBINDExpirySurvivesBrokerOutageAndRestart(t *testing.T) { installTestPublication(t, a, now) a.deps.AckPublisher = newBoundedAckPublisher(stalledPublisher{}) a.cfg.AckPublishTimeout = time.Millisecond - clock = now.Add(61 * time.Second) + clock = now.Add(24 * time.Hour) if err := a.Step(context.Background()); err != nil { t.Fatal(err) } - if data := currentZone(t, a); strings.Contains(data, "192.0.2.10") || !strings.Contains(data, "api.prod.internal. 5 IN "+model.OwnershipMarkerType) { - t.Fatalf("expiry did not preserve NODATA ownership: %s", data) + if data := currentZone(t, a); !strings.Contains(data, "192.0.2.10") { + t.Fatalf("outage removed the published record: %s", data) } local := a.state.Publications["zone-a"].LocalRevision restarted, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return clock }}) @@ -114,10 +114,10 @@ func TestBINDExpirySurvivesBrokerOutageAndRestart(t *testing.T) { t.Fatal(err) } if restarted.state.Publications["zone-a"].LocalRevision != local { - t.Fatal("restart renewed expired observation") + t.Fatal("restart changed the applied publication") } - if data := currentZone(t, restarted); strings.Contains(data, "192.0.2.10") { - t.Fatal("restart resurrected record") + if data := currentZone(t, restarted); !strings.Contains(data, "192.0.2.10") { + t.Fatal("restart lost the published record") } } func TestBINDActivationFailurePersistsObservationFence(t *testing.T) { @@ -145,7 +145,7 @@ func TestBINDActivationFailurePersistsObservationFence(t *testing.T) { if err != nil { t.Fatal(err) } - if f := saved.ContributionFences["contrib-a"]; f.Epoch != 1 || f.Sequence != 1 || !f.ValidUntil.Equal(now.Add(time.Minute)) { + if f := saved.ContributionFences["contrib-a"]; f.Epoch != 1 || f.Sequence != 1 { t.Fatalf("observation fence lost: %#v", f) } if saved.Publications["zone-a"].Verified { @@ -159,36 +159,6 @@ func TestBINDActivationFailurePersistsObservationFence(t *testing.T) { t.Fatalf("equal fenced retry failed: %v", err) } } -func TestBINDFailedExpiryDoesNotRenewWatchdog(t *testing.T) { - t.Parallel() - now := time.Now().UTC() - clock := now - cfg := regionalTestConfig(t) - runner := &recordingRunner{} - a, err := NewAgent(cfg, Dependencies{Runner: runner, Clock: func() time.Time { return clock }}) - if err != nil { - t.Fatal(err) - } - installTestPublication(t, a, now) - if err := os.Remove(cfg.WatchdogLeasePath); err != nil { - t.Fatal(err) - } - clock = now.Add(61 * time.Second) - runner.failAt = len(runner.calls) + 1 - if err := a.Step(context.Background()); err == nil { - t.Fatal("expiry validation failure accepted") - } - if _, err := os.Stat(cfg.WatchdogLeasePath); !errors.Is(err, os.ErrNotExist) { - t.Fatal("failed expiry renewed watchdog") - } - runner.failAt = 0 - if err := a.Step(context.Background()); err != nil { - t.Fatalf("pending expiry retry failed: %v", err) - } - if strings.Contains(currentZone(t, a), "192.0.2.10") { - t.Fatal("retry resurrected expired record") - } -} func TestBINDTombstoneFencesReplayAcrossRestart(t *testing.T) { t.Parallel() now := time.Now().UTC() @@ -214,7 +184,7 @@ func TestBINDTombstoneFencesReplayAcrossRestart(t *testing.T) { t.Fatal("old snapshot defeated tombstone") } } -func TestNodeRequiresDeadlinePlanAndExpiresCacheLocally(t *testing.T) { +func TestNodeRequiresPublicationAndRetainsCacheDuringOutage(t *testing.T) { t.Parallel() now := time.Now().UTC() clock := now @@ -234,9 +204,9 @@ func TestNodeRequiresDeadlinePlanAndExpiresCacheLocally(t *testing.T) { snapshot = testSnapshot(now) a.state.Snapshot = &snapshot a.state.BindingFences, _ = advanceBindingFences(nil, snapshot) - a.state.Acks["zone-a"] = map[string]replicaAck{"regional-0/regional-0": {Fence: fence{2, 4}, ValidUntil: now.Add(time.Minute)}} + a.state.Acks["zone-a"] = map[string]replicaAck{"regional-0/regional-0": {Fence: fence{2, 4}, ObservedAt: now, ValidUntil: now.Add(time.Minute)}} if a.publicationsReady(snapshot.Bindings[0]) { - t.Fatal("node enabled from ACK without deadline plan") + t.Fatal("node enabled from ACK without publication plan") } chunk, m := testPublication(t, now) if err := a.Handle(context.Background(), envelope(t, model.KindPublicationChunk, "zone-a", 2, 4, chunk)); err != nil { @@ -249,7 +219,7 @@ func TestNodeRequiresDeadlinePlanAndExpiresCacheLocally(t *testing.T) { t.Fatal("complete plan + current proof not eligible") } runner.calls = nil - clock = now.Add(61 * time.Second) + clock = now.Add(24 * time.Hour) if err := a.Step(context.Background()); err != nil { t.Fatal(err) } @@ -259,15 +229,15 @@ func TestNodeRequiresDeadlinePlanAndExpiresCacheLocally(t *testing.T) { flushed = true } } - if !flushed { - t.Fatal("node did not flush original-deadline record without broker") + if flushed { + t.Fatal("node flushed a valid record during the outage") } restarted, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return clock }}) if err != nil { t.Fatal(err) } - if restarted.state.Publications["zone-a"].Plan.ObservationFences[0].ValidUntil != now.Add(time.Minute) { - t.Fatal("restart changed original deadline") + if !restarted.publicationsReady(snapshot.Bindings[0]) { + t.Fatal("restart lost the installed publication proof") } } func TestBINDPublicationVersionProofIncludesFullContentFingerprint(t *testing.T) { @@ -279,9 +249,9 @@ func TestBINDPublicationVersionProofIncludesFullContentFingerprint(t *testing.T) t.Fatal(err) } p := publicationState{Fence: fence{m.WriterEpoch, m.Revision}, Plan: plan} - first := publicationMailbox(p, now) + first := publicationMailbox(p) p.Plan.RRSets[0].Records[0].Content = "192.0.2.11" - if first == publicationMailbox(p, now) { + if first == publicationMailbox(p) { t.Fatal("different installed data shares fingerprint") } if len(first) > 255 { @@ -323,7 +293,7 @@ func TestRegionalPoolRequiresOneMemberWithEveryCurrentZone(t *testing.T) { a.state.Publications[z.ZoneUID] = publicationState{Fence: fence{2, 4}, Verified: true} a.state.Acks[z.ZoneUID] = map[string]replicaAck{} } - proof := replicaAck{Fence: fence{2, 4}, ValidUntil: now.Add(time.Minute)} + proof := replicaAck{Fence: fence{2, 4}, ObservedAt: now, ValidUntil: now.Add(time.Minute)} a.state.Acks["zone-a"]["regional-0/"] = proof a.state.Acks["zone-b"]["regional-1/"] = proof if a.publicationsReady(b) { @@ -346,11 +316,11 @@ func TestRegionalPoolRequiresOneMemberWithEveryCurrentZone(t *testing.T) { if strings.Count(conf, "server:addPool(") != 1 { t.Fatalf("stale live member remains in context pool:\n%s", conf) } - // A newer ACK without the matching deadline plan gates even the formerly - // current member; a missed record stream cannot serve data with old leases. - a.state.Acks["zone-a"]["regional-1/"] = replicaAck{Fence: fence{2, 5}, ValidUntil: now.Add(time.Minute)} + // A newer ACK without the matching publication plan gates even the formerly + // current member; a resolver must hold the publication version confirmed by the member. + a.state.Acks["zone-a"]["regional-1/"] = replicaAck{Fence: fence{2, 5}, ObservedAt: now, ValidUntil: now.Add(time.Minute)} if a.publicationsReady(b) { - t.Fatal("ACK ahead of local deadline plan activated context") + t.Fatal("ACK ahead of local publication plan activated context") } } func TestMissingOrStalePrivatePublicationHasNoPublicFallback(t *testing.T) { @@ -358,7 +328,7 @@ func TestMissingOrStalePrivatePublicationHasNoPublicFallback(t *testing.T) { c, snapshot := renderFixture(t) c.Role = RenderRegionalBIND publications := map[string]publicationState{"zone-a": {Fence: fence{1, 1}, Plan: model.PublicationPlan{ZoneUID: "zone-a", Apex: "prod.internal"}}} - rendered, err := renderPublications(c, snapshot, map[string]bool{}, publications, time.Now()) + rendered, err := renderPublications(c, snapshot, map[string]bool{}, publications) if err != nil { t.Fatal(err) } @@ -393,7 +363,7 @@ func TestRejectedMemberProofRevokesRoutingAndIgnoresOlderReplay(t *testing.T) { if err := json.Unmarshal(chunk.Payload, &plan); err != nil { t.Fatal(err) } - a.state.Publications["zone-a"] = publicationState{Fence: fence{2, 4}, Plan: plan, Verified: true, EffectiveHash: effectivePlanHash(plan, now)} + a.state.Publications["zone-a"] = publicationState{Fence: fence{2, 4}, Plan: plan, Verified: true, EffectiveHash: effectivePlanHash(plan)} a.state.Acks["zone-a"] = map[string]replicaAck{"regional-0/regional-0": {Fence: fence{2, 4}, ObservedAt: now, ValidUntil: now.Add(time.Minute)}} ack := model.MemberAck{MemberID: "regional-0", ReplicaID: "regional-0", ResourceUID: "zone-a", Kind: model.KindPublicationManifest, Epoch: 2, Revision: 4, Phase: model.AckRejected, ObservedAt: now.Add(-time.Second), ValidUntil: now.Add(time.Minute)} if err := a.Handle(context.Background(), envelope(t, model.KindMemberAck, "zone-a", 2, 4, ack)); err != nil { @@ -506,47 +476,6 @@ func installedFileVerifier(t *testing.T, configPath string) PublicationVerifier }) } -func TestBINDProofPinsActivatedFilesWhenDeadlineCrossesReload(t *testing.T) { - t.Parallel() - now := time.Now().UTC() - clock := now - cfg := regionalTestConfig(t) - runner := &afterReloadRunner{} - a, err := NewAgent(cfg, Dependencies{Runner: runner, Clock: func() time.Time { return clock }, PublicationVerifier: installedFileVerifier(t, cfg.Render.ClusterBIND.Path)}) - if err != nil { - t.Fatal(err) - } - installTestPublication(t, a, now) - clock = now.Add(59 * time.Second) - a.needsActivation = true - runner.after = func() { clock = now.Add(61 * time.Second) } - if err := a.Step(context.Background()); err != nil { - t.Fatal(err) - } - if a.state.Publications["zone-a"].LocalRevision == 0 || strings.Contains(currentZone(t, a), "192.0.2.10") { - t.Fatal("expiry crossing reload was not materialized before watchdog renewal") - } - if _, err := os.Stat(cfg.WatchdogLeasePath); err != nil { - t.Fatalf("safe expired authority did not renew watchdog: %v", err) - } - // Restart must reconstruct file proofs through activation, never trust the - // earlier process's in-memory proof or its persisted rendered hash alone. - restarted, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return clock }, PublicationVerifier: installedFileVerifier(t, cfg.Render.ClusterBIND.Path)}) - if err != nil { - t.Fatal(err) - } - if len(restarted.installedPublications) != 0 { - t.Fatal("restart inherited unverified process file proofs") - } - restarted.needsActivation = true - if err := restarted.Step(context.Background()); err != nil { - t.Fatal(err) - } - if len(restarted.installedPublications["zone-a"].Views) == 0 { - t.Fatal("restart did not reconstruct exact activated zone proof") - } -} - func TestFailedBINDActivationRetainsPriorInstalledProof(t *testing.T) { t.Parallel() now := time.Now().UTC() @@ -592,7 +521,7 @@ func TestBINDDoesNotProbeAuthorityBelowRequiredPublicationFence(t *testing.T) { if err := json.Unmarshal(chunk.Payload, &plan); err != nil { t.Fatal(err) } - a.state.Publications["zone-a"] = publicationState{Fence: fence{2, 4}, Manifest: manifest, Plan: plan, Verified: true, EffectiveHash: effectivePlanHash(plan, now)} + a.state.Publications["zone-a"] = publicationState{Fence: fence{2, 4}, Manifest: manifest, Plan: plan, Verified: true, EffectiveHash: effectivePlanHash(plan)} if err := a.reconcile(context.Background()); err != nil { t.Fatal(err) } @@ -605,58 +534,6 @@ func TestBINDDoesNotProbeAuthorityBelowRequiredPublicationFence(t *testing.T) { } } -func TestBINDWatchdogCannotRenewExpiredInstalledSnapshot(t *testing.T) { - t.Parallel() - now := time.Now().UTC() - clock := now - cfg := regionalTestConfig(t) - a, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return clock }}) - if err != nil { - t.Fatal(err) - } - installTestPublication(t, a, now) - clock = now.Add(61 * time.Second) - if err := a.renewWatchdogLease(context.Background()); err == nil { - t.Fatal("expired installed snapshot renewed watchdog") - } - if _, err := os.Stat(cfg.WatchdogLeasePath); !errors.Is(err, os.ErrNotExist) || !a.needsActivation { - t.Fatalf("expired snapshot was not gated: %v", err) - } -} - -func TestBINDExpiryCrossingProofSettlesBeforeWatchdogRenewal(t *testing.T) { - t.Parallel() - now := time.Now().UTC() - clock := now - cfg := regionalTestConfig(t) - files := installedFileVerifier(t, cfg.Render.ClusterBIND.Path) - advance := false - verifier := publicationVerifierFunc(func(ctx context.Context, plan model.PublicationPlan, views []PublicationView) error { - err := files.VerifyPublication(ctx, plan, views) - if advance { - advance = false - clock = now.Add(61 * time.Second) - } - return err - }) - a, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return clock }, PublicationVerifier: verifier}) - if err != nil { - t.Fatal(err) - } - installTestPublication(t, a, now) - clock = now.Add(59 * time.Second) - advance = true - if err := a.Step(context.Background()); err != nil { - t.Fatal(err) - } - if strings.Contains(currentZone(t, a), "192.0.2.10") || a.state.Publications["zone-a"].LocalRevision == 0 { - t.Fatal("deadline crossing proof renewed stale authority") - } - if _, err := os.Stat(cfg.WatchdogLeasePath); err != nil { - t.Fatalf("settled expiry unnecessarily gated process: %v", err) - } -} - type partitioningAckPublisher struct { disconnected atomic.Bool verified atomic.Int64 @@ -676,7 +553,7 @@ func (p *partitioningAckPublisher) Publish(_ context.Context, _ string, env mode return transport.PublishAck{}, nil } -func TestBINDActivationRacingBrokerDisconnectKeepsSafeLeaseAndOriginalDeadline(t *testing.T) { +func TestBINDActivationRacingBrokerDisconnectKeepsServing(t *testing.T) { t.Parallel() now := time.Now().UTC() clock := now @@ -724,7 +601,7 @@ func TestBINDActivationRacingBrokerDisconnectKeepsSafeLeaseAndOriginalDeadline(t t.Fatalf("ACK partition removed safe local lease: %v", err) } // Identical manifest redelivery and snapshot delivery must share the same - // best-effort ACK boundary without extending observation deadlines. + // best-effort ACK boundary without changing the observation. if err := a.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 5, manifest)); err != nil { t.Fatal(err) } @@ -734,18 +611,15 @@ func TestBINDActivationRacingBrokerDisconnectKeepsSafeLeaseAndOriginalDeadline(t if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 2, snapshot)); err != nil { t.Fatal(err) } - clock = now.Add(61 * time.Second) + clock = now.Add(24 * time.Hour) if err := a.Step(context.Background()); err != nil { t.Fatal(err) } - if strings.Contains(currentZone(t, a), "192.0.2.11") { - t.Fatal("ACK partition left records past their original deadline") - } - if !a.state.Publications["zone-a"].Plan.ObservationFences[0].ValidUntil.Equal(now.Add(time.Minute)) { - t.Fatal("activation or ACK retry renewed the observation deadline") + if !strings.Contains(currentZone(t, a), "192.0.2.11") { + t.Fatal("ACK partition removed an applied record") } if _, err := os.Stat(cfg.WatchdogLeasePath); err != nil || a.needsActivation { - t.Fatalf("safe local expiry was gated by ACK outage: %v", err) + t.Fatalf("healthy local serving was gated by ACK outage: %v", err) } before := publisher.verified.Load() publisher.disconnected.Store(false) diff --git a/internal/internaldns/serving/binding_fences.go b/internal/internaldns/serving/binding_fences.go index 18bb83d6..ebb2867c 100644 --- a/internal/internaldns/serving/binding_fences.go +++ b/internal/internaldns/serving/binding_fences.go @@ -53,9 +53,6 @@ func advanceBindingFences(prior map[string]bindingFence, snapshot model.ServingS if cmp < 0 { return nil, fmt.Errorf("binding %s source authorization regressed", b.BindingUID) } - if cmp == 0 && !candidate.Authorization.ValidUntil.Equal(old.Authorization.ValidUntil) { - return nil, fmt.Errorf("binding %s changed the original deadline at the same source fence", b.BindingUID) - } if old.Tombstone && !candidate.Tombstone && cmp <= 0 { return nil, fmt.Errorf("binding %s reactivation requires a newer source authorization", b.BindingUID) } diff --git a/internal/internaldns/serving/binding_fences_test.go b/internal/internaldns/serving/binding_fences_test.go index bc065561..21a35e5d 100644 --- a/internal/internaldns/serving/binding_fences_test.go +++ b/internal/internaldns/serving/binding_fences_test.go @@ -23,7 +23,6 @@ func TestBindingWithdrawalPreservesSourceSequenceAndAllowsNormalRenewal(t *testi } active := testSnapshot(now) active.Bindings[0].Authorization.Revision = 2 - active.Bindings[0].Authorization.ValidUntil = now.Add(10 * time.Second) if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 1, active)); err != nil { t.Fatal(err) } @@ -43,18 +42,17 @@ func TestBindingWithdrawalPreservesSourceSequenceAndAllowsNormalRenewal(t *testi if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 2, withdrawn)); err != nil { t.Fatal(err) } - if f := a.state.BindingFences["binding-a"]; !f.Tombstone || f.Authorization.Revision != 2 || !f.Authorization.ValidUntil.Equal(active.Bindings[0].Authorization.ValidUntil) { + if f := a.state.BindingFences["binding-a"]; !f.Tombstone || f.Authorization.Revision != 2 { t.Fatal("withdrawal fabricated source authorization") } // Source2 -> Source3 is the next legitimate renewal; DNS must not have - // consumed sequence3 itself while withdrawing the expired authorization. + // consumed sequence3 itself while withdrawing the withdrawn authorization. renewed := withdrawn renewed.Bindings = append([]model.Binding(nil), withdrawn.Bindings...) renewed.ConfigurationRevision = 3 renewed.Bindings[0].ConfigurationRevision = 3 renewed.Bindings[0].Tombstone = false renewed.Bindings[0].Authorization.Revision = 3 - renewed.Bindings[0].Authorization.ValidUntil = now.Add(2 * time.Minute) if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 3, renewed)); err != nil { t.Fatalf("normal source2 -> source3 renewal rejected: %v", err) } @@ -66,9 +64,9 @@ func TestBindingWithdrawalPreservesSourceSequenceAndAllowsNormalRenewal(t *testi t.Fatal(err) } if a.state.Snapshot.Bindings[0].Tombstone || a.state.ServingFence.Revision != 3 { - t.Fatal("stale expiry replay withdrew current renewal") + t.Fatal("stale withdrawal replay removed current access") } - // A higher shard revision cannot legitimize an older source lease. + // A higher shard revision cannot legitimize an older source revision. stale := withdrawn stale.ConfigurationRevision = 4 if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 4, stale)); err == nil { @@ -112,9 +110,8 @@ func TestBindingOmissionFenceSurvivesRestartAndRequiresNewSourceAuthority(t *tes if err := restarted.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 3, candidate)); err == nil { t.Fatal("same source sequence resurrected omitted binding") } - candidate.Bindings[0].Authorization.ValidUntil = now.Add(20 * time.Minute) if err := restarted.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 3, candidate)); err == nil { - t.Fatal("same source sequence changed its original deadline") + t.Fatal("same source sequence changed its source revision") } candidate.Bindings[0].Authorization.Revision = 3 if err := restarted.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 3, candidate)); err != nil { diff --git a/internal/internaldns/serving/publication_hash.go b/internal/internaldns/serving/publication_hash.go index 73b74d95..11b72acf 100644 --- a/internal/internaldns/serving/publication_hash.go +++ b/internal/internaldns/serving/publication_hash.go @@ -5,16 +5,15 @@ package serving import ( "sort" "strings" - "time" "go.miloapis.com/dns-operator/internal/internaldns/model" ) -func effectivePlanHash(plan model.PublicationPlan, now time.Time) string { +func effectivePlanHash(plan model.PublicationPlan) string { var values []string for _, rr := range plan.RRSets { for _, r := range rr.Records { - if !r.Eligible || (r.ContributionUID != "" && !now.Before(r.ValidUntil)) { + if !r.Eligible { continue } values = append(values, model.AbsoluteName(rr.Name)+"\x00"+strings.ToUpper(rr.Type)+"\x00"+r.Content) diff --git a/internal/internaldns/serving/render.go b/internal/internaldns/serving/render.go index 2eab67c8..df6ddfe6 100644 --- a/internal/internaldns/serving/render.go +++ b/internal/internaldns/serving/render.go @@ -10,7 +10,6 @@ import ( "sort" "strconv" "strings" - "time" "go.miloapis.com/dns-operator/internal/internaldns/model" ) @@ -59,10 +58,10 @@ type RenderConfig struct { type Rendered struct{ Files map[string][]byte } func Render(c RenderConfig, snapshot model.ServingSnapshot, active map[string]bool) (Rendered, error) { - return renderPublications(c, snapshot, active, nil, time.Now()) + return renderPublications(c, snapshot, active, nil) } -func renderPublications(c RenderConfig, snapshot model.ServingSnapshot, active map[string]bool, publications map[string]publicationState, now time.Time) (Rendered, error) { +func renderPublications(c RenderConfig, snapshot model.ServingSnapshot, active map[string]bool, publications map[string]publicationState) (Rendered, error) { if err := snapshot.Validate(); err != nil { return Rendered{}, err } @@ -98,7 +97,7 @@ func renderPublications(c RenderConfig, snapshot model.ServingSnapshot, active m files[c.ClusterDNSDist.Path] = renderDNSDist(c.ClusterDNSDist, clusterBackends, bindings, active, true, c.ReadyMembers) } if role == RenderCluster || role == RenderBoth || role == RenderRegionalBIND { - cluster, err := renderClusterBIND(c.ClusterBIND, bindings, publications, now, files) + cluster, err := renderClusterBIND(c.ClusterBIND, bindings, publications, files) if err != nil { return Rendered{}, err } @@ -246,9 +245,9 @@ func renderDNSDist(c DNSDistRenderConfig, backends []model.Backend, bindings []m if cluster { addr = b.ClusterAddress } - fmt.Fprintf(&out, " [%q] = {context=%q, validUntil=%d, ready=%t},\n", addr, b.ViewName(), b.Authorization.ValidUntil.Unix(), !b.ConfigurationPending && active[b.BindingUID]) + fmt.Fprintf(&out, " [%q] = {context=%q, ready=%t},\n", addr, b.ViewName(), !b.ConfigurationPending && active[b.BindingUID]) } - fmt.Fprint(&out, "}\n\nfunction classifyContext(dq)\n local binding = bindings[dq.localaddr:toString()]\n if binding == nil or os.time() >= binding.validUntil then\n return DNSAction.Refused, \"\"\n end\n dq:setTag(\"dns-context\", binding.context)\n if not binding.ready then dq:setTag(\"context-unready\", \"true\") end\n return DNSAction.None, \"\"\nend\n\n") + fmt.Fprint(&out, "}\n\nfunction classifyContext(dq)\n local binding = bindings[dq.localaddr:toString()]\n if binding == nil then\n return DNSAction.Refused, \"\"\n end\n dq:setTag(\"dns-context\", binding.context)\n if not binding.ready then dq:setTag(\"context-unready\", \"true\") end\n return DNSAction.None, \"\"\nend\n\n") fmt.Fprint(&out, "addAction(AllRule(), LuaAction(classifyContext))\n") fmt.Fprint(&out, "addAction(TagRule(\"context-unready\", \"true\"), RCodeAction(DNSRCode.SERVFAIL))\n") if cluster { @@ -309,7 +308,7 @@ func renderNodeBIND(c BINDRenderConfig, bindings []model.Binding) []byte { return out.Bytes() } -func renderClusterBIND(c BINDRenderConfig, bindings []model.Binding, publications map[string]publicationState, now time.Time, files map[string][]byte) ([]byte, error) { +func renderClusterBIND(c BINDRenderConfig, bindings []model.Binding, publications map[string]publicationState, files map[string][]byte) ([]byte, error) { var out bytes.Buffer renderOptions(&out, c) for _, b := range bindings { @@ -322,7 +321,7 @@ func renderClusterBIND(c BINDRenderConfig, bindings []model.Binding, publication fmt.Fprintf(&out, " zone %q { type forward; forward only; forwarders { 127.0.0.1 port 9; }; };\n", strings.TrimSuffix(model.AbsoluteName(z.Apex), ".")) continue } - data, err := renderZone(p, now) + data, err := renderZone(p) if err != nil { return nil, err } diff --git a/internal/internaldns/serving/render_test.go b/internal/internaldns/serving/render_test.go index 0b2914ec..d252c513 100644 --- a/internal/internaldns/serving/render_test.go +++ b/internal/internaldns/serving/render_test.go @@ -11,7 +11,7 @@ import ( func renderFixture(t *testing.T) (RenderConfig, model.ServingSnapshot) { t.Helper() now := time.Now() - b := model.Binding{BindingUID: "bind-a", ProjectUID: "project-a", ContextUID: "vpc-a", BindingGeneration: 1, ConfigurationRevision: 3, ConsumerAddress: "10.253.0.51", ClusterAddress: "10.253.0.41", Port: 53, Transports: []model.Transport{model.TransportUDP, model.TransportTCP}, Zones: []model.ZoneAttachment{{ZoneUID: "zone-a", Apex: "prod.internal", RequiredPublicationEpoch: 1, RequiredPublicationRevision: 2}}, Authorization: model.Authorization{IssuerEpoch: 1, Revision: 2, ValidUntil: now.Add(time.Minute)}} + b := model.Binding{BindingUID: "bind-a", ProjectUID: "project-a", ContextUID: "vpc-a", BindingGeneration: 1, ConfigurationRevision: 3, ConsumerAddress: "10.253.0.51", ClusterAddress: "10.253.0.41", Port: 53, Transports: []model.Transport{model.TransportUDP, model.TransportTCP}, Zones: []model.ZoneAttachment{{ZoneUID: "zone-a", Apex: "prod.internal", RequiredPublicationEpoch: 1, RequiredPublicationRevision: 2}}, Authorization: model.Authorization{IssuerEpoch: 1, Revision: 2}} c := RenderConfig{NodeDNSDist: DNSDistRenderConfig{Path: "/tmp/node-dnsdist.conf", ListenAddress: "0.0.0.0:53", ACLs: []string{"10.253.0.0/24"}, PoolName: "node-shared", Backends: []model.Backend{{MemberID: "node-bind-0", Address: "10.253.0.30", Port: 5300}}}, ClusterDNSDist: DNSDistRenderConfig{Path: "/tmp/cluster-dnsdist.conf", ListenAddress: "0.0.0.0:53", ACLs: []string{"10.253.0.0/24"}, PoolName: "cluster-shared", Backends: []model.Backend{{MemberID: "cluster-bind-0", Address: "10.253.0.20", Port: 5300}}}, NodeBIND: BINDRenderConfig{Path: "/tmp/node-bind.conf", ListenAddress: "10.253.0.30", Port: 5300, ProxyPeers: []string{"10.253.0.50"}, DefaultCacheSize: "16M"}, ClusterBIND: BINDRenderConfig{Path: "/tmp/cluster-bind.conf", ListenAddress: "10.253.0.20", Port: 5300, ProxyPeers: []string{"10.253.0.40"}, DefaultCacheSize: "16M"}} s := model.ServingSnapshot{Region: "east", Shard: "s1", ConfigurationEpoch: 1, ConfigurationRevision: 3, GeneratedAt: now, Bindings: []model.Binding{b}} return c, s @@ -134,7 +134,7 @@ func TestRenderRoleOwnsOnlyItsTier(t *testing.T) { } } -func TestDNSDistIncludesDeadlineAndBackendHealthCheck(t *testing.T) { +func TestDNSDistUsesPersistentIdentityAndBackendHealthCheck(t *testing.T) { t.Parallel() c, s := renderFixture(t) r, err := Render(c, s, map[string]bool{"bind-a": true}) @@ -142,7 +142,10 @@ func TestDNSDistIncludesDeadlineAndBackendHealthCheck(t *testing.T) { t.Fatal(err) } conf := string(r.Files[c.NodeDNSDist.Path]) - for _, required := range []string{"validUntil=", "os.time() >= binding.validUntil", "checkName=\".\"", "useProxyProtocol=true"} { + if strings.Contains(conf, "validUntil") || strings.Contains(conf, "os.time()") { + t.Fatal("resolver access depends on a deadline") + } + for _, required := range []string{"checkName=\".\"", "useProxyProtocol=true"} { if !strings.Contains(conf, required) { t.Fatalf("dnsdist config missing %q:\n%s", required, conf) } diff --git a/internal/internaldns/serving/state.go b/internal/internaldns/serving/state.go index d2386829..bf432057 100644 --- a/internal/internaldns/serving/state.go +++ b/internal/internaldns/serving/state.go @@ -19,20 +19,21 @@ const legacyObservationGrantUID = "legacy-checkpoint" // contributionFence persists the complete immutable observation identity. // The optional revision field supplies the sequence when the decoded fence omits it. type contributionFence struct { - GrantUID string `json:"grantUID,omitempty"` - Epoch uint64 `json:"epoch"` - Sequence uint64 `json:"sequence,omitempty"` - ValidUntil time.Time `json:"validUntil,omitempty"` - LegacyRevision uint64 `json:"revision,omitempty"` + GrantUID string `json:"grantUID,omitempty"` + Epoch uint64 `json:"epoch"` + Sequence uint64 `json:"sequence,omitempty"` + + LegacyRevision uint64 `json:"revision,omitempty"` } type replicaAck struct { + Rejected bool `json:"rejected,omitempty"` ObservedAt time.Time `json:"observedAt"` Fence fence `json:"fence"` ValidUntil time.Time `json:"validUntil"` } -const checkpointFormatVersion = 4 +const checkpointFormatVersion = 5 type checkpoint struct { FormatVersion int `json:"formatVersion"` @@ -74,7 +75,7 @@ func (s stateStore) Load() (checkpoint, error) { return state, errors.New("checkpoint lacks required binding authority history") } for uid, f := range state.BindingFences { - if uid == "" || f.Identity == "" || f.ConfigurationGeneration == 0 || f.ConfigurationRevision == 0 || f.Authorization.IssuerEpoch == 0 || f.Authorization.Revision == 0 || f.Authorization.ValidUntil.IsZero() { + if uid == "" || f.Identity == "" || f.ConfigurationGeneration == 0 || f.ConfigurationRevision == 0 || f.Authorization.IssuerEpoch == 0 || f.Authorization.Revision == 0 { return state, fmt.Errorf("checkpoint binding %s has incomplete authority history", uid) } } @@ -132,8 +133,8 @@ func migrateLegacyObservationFences(plan model.PublicationPlan) (model.Publicati if record.ContributionUID == "" { continue } - next := model.ObservationFence{ContributionUID: record.ContributionUID, GrantUID: legacyObservationGrantUID, WriterEpoch: record.WriterEpoch, Sequence: record.Sequence, ValidUntil: record.ValidUntil} - if old, ok := byUID[record.ContributionUID]; ok && (old.WriterEpoch != next.WriterEpoch || old.Sequence != next.Sequence || !old.ValidUntil.Equal(next.ValidUntil)) { + next := model.ObservationFence{ContributionUID: record.ContributionUID, GrantUID: legacyObservationGrantUID, WriterEpoch: record.WriterEpoch, Sequence: record.Sequence} + if old, ok := byUID[record.ContributionUID]; ok && (old.WriterEpoch != next.WriterEpoch || old.Sequence != next.Sequence) { return plan, fmt.Errorf("legacy contribution %q has inconsistent record fences", record.ContributionUID) } byUID[record.ContributionUID] = next @@ -164,9 +165,6 @@ func migrateContributionHighWater(state *checkpoint) error { } current.LegacyRevision = 0 if proof, ok := proofs[uid]; ok && proof.WriterEpoch == current.Epoch && proof.Sequence == current.Sequence { - if current.ValidUntil.IsZero() { - current.ValidUntil = proof.ValidUntil - } if current.GrantUID == "" && proof.GrantUID != legacyObservationGrantUID { current.GrantUID = proof.GrantUID } diff --git a/internal/internaldns/serving/zone.go b/internal/internaldns/serving/zone.go index b995bfab..7a00f1a4 100644 --- a/internal/internaldns/serving/zone.go +++ b/internal/internaldns/serving/zone.go @@ -6,16 +6,14 @@ import ( "fmt" "strconv" "strings" - "time" "github.com/miekg/dns" "go.miloapis.com/dns-operator/internal/internaldns/model" ) -// publicationSerial proves the exact publication and locally enforced expiry -// content. It is unrelated to a wall clock and cannot renew contributor leases. -func publicationSerial(p publicationState, now time.Time) uint32 { - hash := publicationFingerprint(p, now) +// publicationSerial proves the exact publication and eligible record content. +func publicationSerial(p publicationState) uint32 { + hash := publicationFingerprint(p) serial, _ := strconv.ParseUint(hash[:8], 16, 32) if serial == 0 { serial = 1 @@ -23,28 +21,28 @@ func publicationSerial(p publicationState, now time.Time) uint32 { return uint32(serial) } -func publicationFingerprint(p publicationState, now time.Time) string { +func publicationFingerprint(p publicationState) string { data, _ := json.Marshal(p.Plan) - return model.Hash([]byte(fmt.Sprintf("%d/%d/%d/%s/%s", p.Fence.Epoch, p.Fence.Revision, p.LocalRevision, model.Hash(data), effectivePlanHash(p.Plan, now)))) + return model.Hash([]byte(fmt.Sprintf("%d/%d/%d/%s/%s", p.Fence.Epoch, p.Fence.Revision, p.LocalRevision, model.Hash(data), effectivePlanHash(p.Plan)))) } -func publicationMailbox(p publicationState, now time.Time) string { - hash := publicationFingerprint(p, now) +func publicationMailbox(p publicationState) string { + hash := publicationFingerprint(p) return hash[:32] + "." + hash[32:] + ".publication.internal." } -func renderZone(p publicationState, now time.Time) ([]byte, error) { +func renderZone(p publicationState) ([]byte, error) { if err := p.Plan.Validate(); err != nil { return nil, fmt.Errorf("invalid publication: %w", err) } var out bytes.Buffer apex := model.AbsoluteName(p.Plan.Apex) - fmt.Fprintf(&out, "$ORIGIN %s\n$TTL 5\n@ IN SOA ns1.internal-dns.invalid. %s %d 60 30 3600 5\n@ IN NS ns1.internal-dns.invalid.\n", apex, publicationMailbox(p, now), publicationSerial(p, now)) + fmt.Fprintf(&out, "$ORIGIN %s\n$TTL 5\n@ IN SOA ns1.internal-dns.invalid. %s %d 60 30 3600 5\n@ IN NS ns1.internal-dns.invalid.\n", apex, publicationMailbox(p), publicationSerial(p)) ownerHasRecord := map[string]bool{} for _, rrset := range p.Plan.RRSets { if strings.EqualFold(rrset.Type, "SOA") { return nil, fmt.Errorf("publication SOA is reserved for version proof") } for _, record := range rrset.Records { - if !record.Eligible || (record.ContributionUID != "" && !now.Before(record.ValidUntil)) { + if !record.Eligible { continue } line := fmt.Sprintf("%s %d IN %s %s", model.AbsoluteName(rrset.Name), rrset.TTL, strings.ToUpper(rrset.Type), record.Content) diff --git a/internal/internaldns/serving/zone_regression_test.go b/internal/internaldns/serving/zone_regression_test.go index d8ab91d0..b2abacdf 100644 --- a/internal/internaldns/serving/zone_regression_test.go +++ b/internal/internaldns/serving/zone_regression_test.go @@ -3,7 +3,6 @@ package serving import ( "strings" "testing" - "time" "github.com/miekg/dns" "go.miloapis.com/dns-operator/internal/internaldns/model" @@ -23,10 +22,9 @@ func parseRenderedZone(t *testing.T, zone []byte) []dns.RR { } func TestNODATAOwnershipDoesNotGrowNamesOrCollideWithTenantCNAME(t *testing.T) { - now := time.Now() owner := strings.Repeat("a", 63) + "." + strings.Repeat("b", 63) + "." + strings.Repeat("c", 63) + "." + strings.Repeat("d", 45) + ".x.internal." plan := model.PublicationPlan{ZoneUID: "zone", Apex: "x.internal", Owners: []string{owner, "api.x.internal"}, RRSets: []model.RRSet{{Name: "_dns-ownership.api.x.internal", Type: "CNAME", TTL: 5, Records: []model.RRRecord{{Content: "target.example.", Eligible: true}}}}} - data, err := renderZone(publicationState{Plan: plan}, now) + data, err := renderZone(publicationState{Plan: plan}) if err != nil { t.Fatal(err) } @@ -41,7 +39,7 @@ func TestNODATAOwnershipDoesNotGrowNamesOrCollideWithTenantCNAME(t *testing.T) { t.Fatalf("missing ownership metadata: %v", markers) } plan.RRSets = append(plan.RRSets, model.RRSet{Name: "api.x.internal", Type: "CNAME", TTL: 5, Records: []model.RRRecord{{Content: "healthy.example.", Eligible: true}}}) - data, err = renderZone(publicationState{Plan: plan}, now) + data, err = renderZone(publicationState{Plan: plan}) if err != nil { t.Fatal(err) } diff --git a/internal/internaldns/transport/jetstream.go b/internal/internaldns/transport/jetstream.go index 7aa00976..8f300b8a 100644 --- a/internal/internaldns/transport/jetstream.go +++ b/internal/internaldns/transport/jetstream.go @@ -92,7 +92,7 @@ func Connect(c Config) (*JetStream, error) { nats.Name(c.Name), nats.Timeout(defaultDuration(c.ConnectTimeout, 10*time.Second)), // A regional partition must not permanently close a long-running agent. - // Callers keep retrying while local expiry and the watchdog gate serving. + // Callers retry delivery while local health checks govern serving. nats.MaxReconnects(-1), } if c.CredentialsFile != "" { diff --git a/test/internaldns/README.md b/test/internaldns/README.md index 93e75daa..637aa9a7 100644 --- a/test/internaldns/README.md +++ b/test/internaldns/README.md @@ -83,8 +83,8 @@ The Go qualifier covers overlapping tenant names over UDP and TCP, A/AAAA and negative-cache isolation, multiple zones, managed name discovery, scoped publication and admission denial, record updates and deletion, health withdrawal and recovery, fixed fleet size, regional publication proof, -controller takeover, local record/access expiry while controllers and the -broker are stopped, and publication after broker/controller restoration using +controller takeover, continued tenant-isolated UDP/TCP resolution while controllers +and the broker are stopped, and publication after broker/controller restoration using the retained broker PVCs. Each assertion records its result; skipped scenarios do not count as passes. @@ -111,5 +111,5 @@ into the separate DNS service VPC. Kind cannot establish physical fabric, microVM attachment, host-loss durability, regional independence, or production capacity. -The suite requires internal wire version `v1alpha3`, fresh format 4 checkpoints, +The suite requires internal wire version `v1alpha4`, fresh format 5 checkpoints, and matching controller and fleet versions. diff --git a/test/internaldns/kubernetes/qualify/main.go b/test/internaldns/kubernetes/qualify/main.go index e84f7c9d..1f6a5431 100644 --- a/test/internaldns/kubernetes/qualify/main.go +++ b/test/internaldns/kubernetes/qualify/main.go @@ -34,7 +34,6 @@ import ( type project struct { Name string `json:"name"` ProjectUID string `json:"projectUID"` - SourceClusterUID string `json:"sourceClusterUID"` Namespace string `json:"namespace"` CustomerKubeconfig string `json:"customerKubeconfig"` ProductKubeconfig string `json:"productKubeconfig"` @@ -268,7 +267,7 @@ func (s *suite) run(path string) error { return fmt.Errorf("%s client: %w", p.Name, err) } } - if net.ParseIP(p.ConsumerAddress) == nil || p.ProjectUID == "" || p.SourceClusterUID == "" || p.PublisherSubject == "" { + if net.ParseIP(p.ConsumerAddress) == nil || p.ProjectUID == "" || p.PublisherSubject == "" { return fmt.Errorf("%s has incomplete identity or resolver address", p.Name) } } @@ -351,7 +350,7 @@ func (s *suite) prepare(p *project) error { p.access = dnsv1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "application-network-access", Namespace: p.Namespace}, Spec: dnsv1.DNSResolverAccessBindingSpec{ ContextRef: objectRef(&p.resolverContext), Region: s.env.Region, QueryIdentity: dnsv1.DNSResolverQueryIdentity{Type: dnsv1.DNSResolverQueryIdentityDestinationAddress, Value: p.ConsumerAddress}, Port: 53, Transports: []string{"UDP", "TCP"}, - Authorization: dnsv1.DNSResolverAccessAuthorization{WriterEpoch: p.resolverContext.Status.AccessWriterEpoch, Sequence: 1, ValidUntil: metav1.NewTime(time.Now().Add(5 * time.Minute).UTC())}, + Authorization: dnsv1.DNSResolverAccessAuthorization{WriterEpoch: p.resolverContext.Status.AccessWriterEpoch, Sequence: 1}, }} if err := p.integration.Create(s.ctx, &p.access); err != nil { return err @@ -395,7 +394,7 @@ func recordSets(name, address, ipv6 string) []dnsv1.DNSContributionRecordSet { return sets } -func (s *suite) publish(p *project, zone dnsv1.DNSObjectReference, prefix, name, address, ipv6 string, lifetime time.Duration) (*publication, error) { +func (s *suite) publish(p *project, zone dnsv1.DNSObjectReference, prefix, name, address, ipv6 string) (*publication, error) { pub := &publication{registration: dnsv1.DNSRegistration{ObjectMeta: metav1.ObjectMeta{Name: prefix + "-registration", Namespace: p.Namespace}, Spec: dnsv1.DNSRegistrationSpec{DNSZoneRef: zone, Name: name, RecordTypes: []dnsv1.RRType{dnsv1.RRTypeA}, PublicationPolicy: dnsv1.DNSPublicationPolicyEligibleContributions, TTLSeconds: 2}}} if ipv6 != "" { pub.registration.Spec.RecordTypes = append(pub.registration.Spec.RecordTypes, dnsv1.RRTypeAAAA) @@ -403,7 +402,7 @@ func (s *suite) publish(p *project, zone dnsv1.DNSObjectReference, prefix, name, if err := p.product.Create(s.ctx, &pub.registration); err != nil { return nil, err } - pub.grant = dnsv1.DNSContributionGrant{ObjectMeta: metav1.ObjectMeta{Name: prefix + "-compute", Namespace: p.Namespace}, Spec: dnsv1.DNSContributionGrantSpec{RegistrationRef: objectRef(&pub.registration), ProducerID: "compute", Principal: dnsv1.DNSProducerPrincipal{ClusterUID: p.SourceClusterUID, Subject: p.PublisherSubject}, RecordTypes: pub.registration.Spec.RecordTypes}} + pub.grant = dnsv1.DNSContributionGrant{ObjectMeta: metav1.ObjectMeta{Name: prefix + "-compute", Namespace: p.Namespace}, Spec: dnsv1.DNSContributionGrantSpec{RegistrationRef: objectRef(&pub.registration), ProducerID: "compute", Principal: dnsv1.DNSProducerPrincipal{Subject: p.PublisherSubject}, RecordTypes: pub.registration.Spec.RecordTypes}} if err := p.issuer.Create(s.ctx, &pub.grant); err != nil { return nil, err } @@ -423,13 +422,13 @@ func (s *suite) publish(p *project, zone dnsv1.DNSObjectReference, prefix, name, }); err != nil { return nil, err } - if err := s.observe(p, pub, true, lifetime); err != nil { + if err := s.observe(p, pub, true); err != nil { return nil, err } return pub, nil } -func (s *suite) observe(p *project, pub *publication, eligible bool, lifetime time.Duration) error { +func (s *suite) observe(p *project, pub *publication, eligible bool) error { if err := p.product.Get(s.ctx, client.ObjectKeyFromObject(&pub.contribution), &pub.contribution); err != nil { return err } @@ -442,8 +441,6 @@ func (s *suite) observe(p *project, pub *publication, eligible bool, lifetime ti if !eligible { pub.contribution.Status.Reason = "EndpointNotReady" } - deadline := metav1.NewTime(time.Now().Add(lifetime).UTC()) - pub.contribution.Status.ValidUntil = &deadline if err := p.product.Status().Patch(s.ctx, &pub.contribution, client.MergeFrom(before)); err != nil { return err } diff --git a/test/internaldns/kubernetes/qualify/scenarios.go b/test/internaldns/kubernetes/qualify/scenarios.go index fa7e3cd8..39801fd6 100644 --- a/test/internaldns/kubernetes/qualify/scenarios.go +++ b/test/internaldns/kubernetes/qualify/scenarios.go @@ -5,6 +5,7 @@ package main import ( "errors" "fmt" + "os/exec" "strings" "time" @@ -23,11 +24,11 @@ func (s *suite) scenarios(a, b *project) error { return err } } - pa, err := s.publish(a, objectRef(&a.zone), "common", "common", "10.42.0.10", "fd42::10", 90*time.Second) + pa, err := s.publish(a, objectRef(&a.zone), "common", "common", "10.42.0.10", "fd42::10") if err != nil { return err } - pb, err := s.publish(b, objectRef(&b.zone), "common", "common", "10.42.0.20", "fd42::20", 90*time.Second) + pb, err := s.publish(b, objectRef(&b.zone), "common", "common", "10.42.0.20", "fd42::20") if err != nil { return err } @@ -74,7 +75,7 @@ func (s *suite) scenarios(a, b *project) error { return err } s.pass("an unbound destination cannot resolve either tenant's private records", nil) - if _, err := s.publish(a, objectRef(&a.zone), "only-a", "only-a", "10.42.0.20", "", 90*time.Second); err != nil { + if _, err := s.publish(a, objectRef(&a.zone), "only-a", "only-a", "10.42.0.20", ""); err != nil { return err } if err := s.expect(a, "only-a.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.20"); err != nil { @@ -85,7 +86,7 @@ func (s *suite) scenarios(a, b *project) error { } s.pass("negative caches and overlapping endpoint address spaces stay isolated", nil) // Products discover their managed zone through context status. - if _, err := s.publish(a, a.resolverContext.Status.ManagedNamespace.DNSZoneRef, "managed", "web", "10.42.0.11", "", 90*time.Second); err != nil { + if _, err := s.publish(a, a.resolverContext.Status.ManagedNamespace.DNSZoneRef, "managed", "web", "10.42.0.11", ""); err != nil { return err } managedName := "web." + a.resolverContext.Status.ManagedNamespace.Suffix @@ -94,7 +95,7 @@ func (s *suite) scenarios(a, b *project) error { return err } } - if _, err := s.publish(b, b.resolverContext.Status.ManagedNamespace.DNSZoneRef, "managed", "web", "10.42.0.21", "", 90*time.Second); err != nil { + if _, err := s.publish(b, b.resolverContext.Status.ManagedNamespace.DNSZoneRef, "managed", "web", "10.42.0.21", ""); err != nil { return err } for _, tcp := range []bool{false, true} { @@ -113,14 +114,14 @@ func (s *suite) scenarios(a, b *project) error { if err := s.associate(a, apps, "apps"); err != nil { return err } - if _, err := s.publish(a, objectRef(apps), "apps", "api", "10.42.0.12", "", 90*time.Second); err != nil { + if _, err := s.publish(a, objectRef(apps), "apps", "api", "10.42.0.12", ""); err != nil { return err } if err := s.expect(a, "api.apps.internal", "A", true, dns.RcodeSuccess, "10.42.0.12"); err != nil { return err } s.pass("one resolver context serves its managed namespace and multiple custom zones", nil) - if err := s.observe(b, pb, true, 90*time.Second); err != nil { + if err := s.observe(b, pb, true); err != nil { return err } if err := s.update(a, pa, "10.42.0.30", "fd42::30"); err != nil { @@ -132,7 +133,7 @@ func (s *suite) scenarios(a, b *project) error { if err := s.expect(b, "common.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.20"); err != nil { return err } - if err := s.observe(a, pa, false, 90*time.Second); err != nil { + if err := s.observe(a, pa, false); err != nil { return err } for _, tcp := range []bool{false, true} { @@ -141,7 +142,7 @@ func (s *suite) scenarios(a, b *project) error { } } s.pass("an unhealthy endpoint stops producing DNS answers", nil) - if err := s.observe(a, pa, true, 90*time.Second); err != nil { + if err := s.observe(a, pa, true); err != nil { return err } if err := s.expect(a, "common.prod.internal", "A", true, dns.RcodeSuccess, "10.42.0.30"); err != nil { @@ -161,10 +162,10 @@ func (s *suite) scenarios(a, b *project) error { if err := s.takeover(a, pa); err != nil { return err } - if err := s.observe(b, pb, true, 90*time.Second); err != nil { + if err := s.observe(b, pb, true); err != nil { return err } - deleted, err := s.publish(a, objectRef(&a.zone), "deleted", "deleted", "10.42.0.14", "", 90*time.Second) + deleted, err := s.publish(a, objectRef(&a.zone), "deleted", "deleted", "10.42.0.14", "") if err != nil { return err } @@ -178,10 +179,10 @@ func (s *suite) scenarios(a, b *project) error { return err } s.pass("deleting a contribution withdraws its answer and preserves registration ownership", nil) - if err := s.offlineExpiry(a, b, pa, pb); err != nil { + if err := s.offlineServing(a, b); err != nil { return err } - if err := s.observe(b, pb, true, 90*time.Second); err != nil { + if err := s.observe(b, pb, true); err != nil { return err } if err := s.expect(b, "common.prod.internal", "A", true, dns.RcodeSuccess, "10.42.0.20"); err != nil { @@ -200,7 +201,7 @@ func (s *suite) update(p *project, pub *publication, address, ipv6 string) error if err := p.product.Patch(s.ctx, &pub.contribution, client.MergeFrom(before)); err != nil { return err } - return s.observe(p, pub, true, 90*time.Second) + return s.observe(p, pub, true) } func (s *suite) owner(zoneUID string) (dnsv1.DNSPublicationOwnership, error) { @@ -284,7 +285,7 @@ func (s *suite) takeover(p *project, pub *publication) (resultErr error) { }); err != nil { return err } - if err := s.observe(p, pub, true, 90*time.Second); err != nil { + if err := s.observe(p, pub, true); err != nil { return err } if err := s.expect(p, "common.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.30"); err != nil { @@ -322,49 +323,18 @@ func (s *suite) scaleBroker(replicas int32) error { }) } -func (s *suite) offlineExpiry(a, b *project, pa, pb *publication) (resultErr error) { - // Keep the healthy tenant's original record and access lease long enough to - // cover the outage. Only the expired tenant receives the short access lease. - // Finish access readiness before starting the record leases: serving ACKs - // follow the production 30-second interval and can consume that setup time. - if err := s.renewAccess(b, 5*time.Minute); err != nil { - return err - } - if err := s.observe(a, pa, true, 90*time.Second); err != nil { - return err - } - if err := s.observe(b, pb, true, 90*time.Second); err != nil { - return err - } - expiring, err := s.publish(a, objectRef(&a.zone), "expires", "expires", "10.42.0.15", "", 45*time.Second) - if err != nil { - return err - } - if err := s.expect(a, "expires.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.15"); err != nil { - return err - } - if err := a.integration.Get(s.ctx, client.ObjectKeyFromObject(&a.access), &a.access); err != nil { - return err - } - before := a.access.DeepCopy() - a.access.Spec.Authorization.Sequence++ - a.access.Spec.Authorization.ValidUntil = metav1.NewTime(time.Now().Add(50 * time.Second).UTC()) - if err := a.integration.Patch(s.ctx, &a.access, client.MergeFrom(before)); err != nil { - return err - } - if err := s.waitAccess(a); err != nil { - return err - } +// offlineServing checks uncached queries while control-plane delivery is unavailable. +func (s *suite) offlineServing(a, b *project) (resultErr error) { var broker appsv1.StatefulSet if err := s.platform.Get(s.ctx, client.ObjectKey{Namespace: s.env.Namespace, Name: s.env.BrokerStatefulSet}, &broker); err != nil { return err } - brokerReplicas := int32(1) + replicas := int32(3) if broker.Spec.Replicas != nil { - brokerReplicas = *broker.Spec.Replicas + replicas = *broker.Spec.Replicas } defer func() { - if err := s.scaleBroker(brokerReplicas); err != nil { + if err := s.scaleBroker(replicas); err != nil { resultErr = errors.Join(resultErr, fmt.Errorf("broker restoration: %w", err)) } for _, name := range s.env.ControllerDeployments { @@ -381,83 +351,74 @@ func (s *suite) offlineExpiry(a, b *project, pa, pb *publication) (resultErr err if err := s.scaleBroker(0); err != nil { return err } - if expiring.contribution.Status.ValidUntil == nil || !expiring.contribution.Status.ValidUntil.After(time.Now().Add(3*time.Second)) { - return fmt.Errorf("record expired before the outage; no offline expiry proof") - } - if !a.access.Spec.Authorization.ValidUntil.After(time.Now().Add(10 * time.Second)) { - return fmt.Errorf("access expired before the outage; no offline authorization proof") - } - if !a.access.Spec.Authorization.ValidUntil.After(expiring.contribution.Status.ValidUntil.Add(5 * time.Second)) { - return fmt.Errorf("record and access deadlines overlap; no independent expiry proof") - } - if pb.contribution.Status.ValidUntil == nil || - !pb.contribution.Status.ValidUntil.After(a.access.Spec.Authorization.ValidUntil.Add(15*time.Second)) { - return fmt.Errorf("healthy record lease cannot cover access expiry; no outage positive control") - } - if err := s.expect(a, "expires.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.15"); err != nil { - return err - } - s.pass("Kubernetes controllers and broker stop without stopping the serving pods", nil) - if deadline := expiring.contribution.Status.ValidUntil; deadline != nil { - if remaining := time.Until(deadline.Add(3 * time.Second)); remaining > 0 { - time.Sleep(remaining) + // The outage spans multiple DNS cache and fleet acknowledgement intervals. + until := time.Now().Add(2 * time.Minute) + for time.Now().Before(until) { + for _, tcp := range []bool{false, true} { + for _, tenant := range []struct { + project *project + address string + }{{a, "10.42.0.30"}, {b, "10.42.0.20"}} { + answer, err := s.query(tenant.project, "common.prod.internal", "A", tcp) + if err != nil { + return err + } + if answer.Error != "" || answer.Rcode != dns.RcodeSuccess || len(answer.Addresses) != 1 || answer.Addresses[0] != tenant.address { + return fmt.Errorf("outage query for %s TCP=%t: %#v", tenant.project.Name, tcp, answer) + } + } + } + select { + case <-s.ctx.Done(): + return s.ctx.Err() + case <-time.After(time.Second): } } - if err := s.expect(a, "expires.prod.internal", "A", false, dns.RcodeSuccess); err != nil { - return err - } - if err := s.expect(b, "common.prod.internal", "A", true, dns.RcodeSuccess, "10.42.0.20"); err != nil { + s.pass("both tenants keep resolving over UDP and TCP with controllers and messaging stopped", nil) + var pods corev1.PodList + if err := s.platform.List(s.ctx, &pods, client.InNamespace(s.env.Namespace), client.MatchingLabels{"internal-dns-role": "node"}); err != nil { return err } - s.pass("local record expiry preserves the original deadline during a broker and controller outage", expiring.contribution.Status.ValidUntil) - if remaining := time.Until(a.access.Spec.Authorization.ValidUntil.Add(-3 * time.Second)); remaining > 0 { - time.Sleep(remaining) + if len(pods.Items) != 1 { + return fmt.Errorf("expected one node resolver, got %d", len(pods.Items)) } - answer, err := s.query(a, "expires.prod.internal", "A", false) - if err != nil { - return err + pod := pods.Items[0] + var restarts int32 + for _, status := range pod.Status.ContainerStatuses { + if status.Name == "agent" { + restarts = status.RestartCount + } } - if answer.Error != "" || answer.Rcode != dns.RcodeSuccess || len(answer.Answers) != 0 || - !a.access.Spec.Authorization.ValidUntil.After(time.Now()) { - return fmt.Errorf("access was not usable just before its original deadline: %+v", answer) + cmd := exec.CommandContext(s.ctx, "kubectl", "--kubeconfig", s.env.PlatformKubeconfig, "-n", s.env.Namespace, "exec", pod.Name, "-c", "agent", "--", "pkill", "-TERM", "-f", "^/internal-dns --role=agent") + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("restart resolver agent: %w: %s", err, output) } - s.pass("unexpired access remains usable after record expiry", map[string]any{ - "validUntil": a.access.Spec.Authorization.ValidUntil, "answer": answer, - }) - if remaining := time.Until(a.access.Spec.Authorization.ValidUntil.Add(3 * time.Second)); remaining > 0 { - time.Sleep(remaining) + if err := s.wait("resolver agent restart without messaging", time.Minute, func() (bool, error) { + var current corev1.Pod + if err := s.platform.Get(s.ctx, client.ObjectKeyFromObject(&pod), ¤t); err != nil { + return false, err + } + for _, status := range current.Status.ContainerStatuses { + if status.Name == "agent" && status.RestartCount > restarts && status.State.Running != nil { + return true, nil + } + } + return false, nil + }); err != nil { + return err } for _, tcp := range []bool{false, true} { - if err := s.wait("expired access rejected", 15*time.Second, func() (bool, error) { - answer, err := s.query(a, "common.prod.internal", "A", tcp) - if err != nil { - return false, err - } - return answer.Error != "" || answer.Rcode == dns.RcodeRefused || answer.Rcode == dns.RcodeServerFailure, nil - }); err != nil { + if err := s.expect(a, "common.prod.internal", "A", tcp, dns.RcodeSuccess, "10.42.0.30"); err != nil { + return err + } + if err := s.expect(b, "common.prod.internal", "A", tcp, dns.RcodeSuccess, "10.42.0.20"); err != nil { return err } } - if err := s.expect(b, "common.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.20"); err != nil { - return err - } - s.pass("expired access fails closed while another tenant keeps resolving", a.access.Spec.Authorization.ValidUntil) + s.pass("resolver agent restores both tenant contexts from persistent state without messaging", nil) return nil } -func (s *suite) renewAccess(p *project, lifetime time.Duration) error { - if err := p.integration.Get(s.ctx, client.ObjectKeyFromObject(&p.access), &p.access); err != nil { - return err - } - before := p.access.DeepCopy() - p.access.Spec.Authorization.Sequence++ - p.access.Spec.Authorization.ValidUntil = metav1.NewTime(time.Now().Add(lifetime).UTC()) - if err := p.integration.Patch(s.ctx, &p.access, client.MergeFrom(before)); err != nil { - return err - } - return s.waitAccess(p) -} - func readyPods(pods []corev1.Pod, replicas int32) bool { if len(pods) != int(replicas) { return false diff --git a/test/internaldns/kubernetes/render/main.go b/test/internaldns/kubernetes/render/main.go index dcd7c18b..6d030b00 100644 --- a/test/internaldns/kubernetes/render/main.go +++ b/test/internaldns/kubernetes/render/main.go @@ -384,7 +384,7 @@ func configure() { tokenConfig(admin, "", project, "dns-publisher", "compute-system", "product") tokenConfig(admin, "", project, "dns-grant-issuer", "compute-system", "issuer") tokenConfig(admin, "", project, "dns-vpc-integration", "network-system", "integration") - projectRows = append(projectRows, object{"name": project, "projectUID": project + "-uid", "sourceClusterUID": []string{"source-cluster-a-uid", "source-cluster-b-uid"}[i], "namespace": projectNS, "customerKubeconfig": filepath.Join(out, project+"-customer.kubeconfig"), "productKubeconfig": filepath.Join(out, project+"-product.kubeconfig"), "issuerKubeconfig": filepath.Join(out, project+"-issuer.kubeconfig"), "integrationKubeconfig": filepath.Join(out, project+"-integration.kubeconfig"), "publisherSubject": "system:serviceaccount:compute-system:dns-publisher", "consumerAddress": []string{"fd53::a", "fd53::b"}[i]}) + projectRows = append(projectRows, object{"name": project, "projectUID": project + "-uid", "namespace": projectNS, "customerKubeconfig": filepath.Join(out, project+"-customer.kubeconfig"), "productKubeconfig": filepath.Join(out, project+"-product.kubeconfig"), "issuerKubeconfig": filepath.Join(out, project+"-issuer.kubeconfig"), "integrationKubeconfig": filepath.Join(out, project+"-integration.kubeconfig"), "publisherSubject": "system:serviceaccount:compute-system:dns-publisher", "consumerAddress": []string{"fd53::a", "fd53::b"}[i]}) } sources["stringData"] = sourceData items := make([]object, 0, 8) @@ -392,10 +392,9 @@ func configure() { cfg := exampleConfigs("control-plane.example.yaml")["internal-dns-control-plane-config"] cfg.ClusterPrefix = "fd54::/64" cfg.OwnershipLeaseSeconds = 10 - cfg.Projects = []runtime.ProjectConfig{{Name: "project-a", ProjectUID: "project-a-uid", SourceClusterUID: "source-cluster-a-uid", Namespace: projectNS, Kubeconfig: "/sources/project-a.json"}, {Name: "project-b", ProjectUID: "project-b-uid", SourceClusterUID: "source-cluster-b-uid", Namespace: projectNS, Kubeconfig: "/sources/project-b.json"}} + cfg.Projects = []runtime.ProjectConfig{{Name: "project-a", ProjectUID: "project-a-uid", Namespace: projectNS, Kubeconfig: "/sources/project-a.json"}, {Name: "project-b", ProjectUID: "project-b-uid", Namespace: projectNS, Kubeconfig: "/sources/project-b.json"}} cfg.NodeBackends = []model.Backend{{MemberID: "node-bind-0", Address: "127.0.0.1", Port: 5300}} cfg.ClusterBackends = []model.Backend{{MemberID: memberNames[2], Address: bindIP[0], Port: 5300}, {MemberID: memberNames[3], Address: bindIP[1], Port: 5300}} - cfg.Admission.MaxAccessLeaseSeconds = 600 for _, name := range []string{"control-a", "control-b"} { c := cfg c.NATS.Name = name diff --git a/test/internaldns/production-parity.md b/test/internaldns/production-parity.md index 84dfb42b..7126f71f 100644 --- a/test/internaldns/production-parity.md +++ b/test/internaldns/production-parity.md @@ -33,7 +33,7 @@ cluster health or that the proposed private DNS fleet is deployed. - [NATS infrastructure](https://github.com/datum-cloud/infra/blob/5cc5caba5b7f886ca00e7c3ed90726834dac7fac/infrastructure/databases/nats/base/nats-hr.yaml) declares three broker replicas, persistent JetStream storage, and client and route TLS. The Kubernetes lab uses that three-replica storage and TLS pattern. - Its full broker outage test proves local expiry and replica restart using + Its full broker outage test proves local serving continuity and replica restart using retained PVCs. Quorum loss, disk/host-loss recovery, and regional transport still need dedicated scenarios. - [Public edge DNS](https://github.com/datum-cloud/infra/tree/5cc5caba5b7f886ca00e7c3ed90726834dac7fac/apps/dns-operator/downstream/edge) @@ -137,7 +137,7 @@ intended for staging. Do not mount the Docker socket into those workloads. 2. **Kubernetes deployment (implemented).** Reuse pinned test-infra, serving images and command helpers, TLS admission, scoped worker/product credentials, two controllers, three TLS brokers, and persistent serving state. Run - publication, isolation, takeover, and expiry checks against real pods. ACK + publication, isolation, takeover, and outage checks against real pods. ACK timing is 30/90 seconds; ownership lease and termination grace are shortened to 10 and 2 seconds. CI runs the public regression chain in the same job. Publication resumes after broker/controller restoration using retained @@ -172,11 +172,14 @@ Required deployment and consumer-path scenarios: service route cannot return another VPC's records. Product credentials cannot issue grants or authorize network access. - Endpoint health withdrawal and recovery, deletion, source API loss, and - stale-observation replay obey the original deadlines and configured TTLs. + stale-observation replay preserve current source fences and configured TTLs. + Records remain available during control-plane outages until an explicit update + withdraws them. - Either controller can take ownership without stale publication; a healthy project continues publishing while another project's API is unavailable. -- Broker leader loss retains publication with quorum; quorum loss, agent - restart, member failure, and node failure preserve expiry and fail closed. +- Broker leader loss retains publication with quorum. Agent restart during + quorum loss restores verified checkpoints without broker access. Member and node + failures stop unsafe local serving through independent health checks. - Certificate rotation, webhook unavailability, rolling member replacement, and PVC reuse do not bypass admission or regress checkpoint authority. - Adding contexts leaves the workload count fixed, and public authoritative