From 8d6de60acfe1852fcd5dee95d6f59e7081da918c Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 12:09:13 -0500 Subject: [PATCH 1/7] feat: Manage private DNS contexts and resolver access --- .../access_recovery_envtest_test.go | 290 ++++++++ .../controlplane/context_reconciler.go | 680 ++++++++++++++++++ .../controlplane/context_reconciler_test.go | 304 ++++++++ internal/internaldns/controlplane/naming.go | 96 +++ .../internaldns/controlplane/naming_test.go | 28 + .../internaldns/controlplane/reconciler.go | 161 +++++ .../internaldns/controlplane/references.go | 18 + .../controlplane/references_test.go | 24 + internal/internaldns/controlplane/values.go | 5 + internal/internaldns/platform/allocator.go | 174 +++++ .../internaldns/platform/allocator_test.go | 78 ++ internal/internaldns/projectstatus/status.go | 44 ++ .../internaldns/projectstatus/status_test.go | 26 + 13 files changed, 1928 insertions(+) create mode 100644 internal/internaldns/controlplane/access_recovery_envtest_test.go create mode 100644 internal/internaldns/controlplane/context_reconciler.go create mode 100644 internal/internaldns/controlplane/context_reconciler_test.go create mode 100644 internal/internaldns/controlplane/naming.go create mode 100644 internal/internaldns/controlplane/naming_test.go create mode 100644 internal/internaldns/controlplane/reconciler.go create mode 100644 internal/internaldns/controlplane/references.go create mode 100644 internal/internaldns/controlplane/references_test.go create mode 100644 internal/internaldns/controlplane/values.go create mode 100644 internal/internaldns/platform/allocator.go create mode 100644 internal/internaldns/platform/allocator_test.go create mode 100644 internal/internaldns/projectstatus/status.go create mode 100644 internal/internaldns/projectstatus/status_test.go diff --git a/internal/internaldns/controlplane/access_recovery_envtest_test.go b/internal/internaldns/controlplane/access_recovery_envtest_test.go new file mode 100644 index 00000000..55b066c1 --- /dev/null +++ b/internal/internaldns/controlplane/access_recovery_envtest_test.go @@ -0,0 +1,290 @@ +package controlplane + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" + "go.miloapis.com/dns-operator/internal/internaldns/platform" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +// TestAccessRecoveryPreservesSourceFencesWithCEL exercises the actual CRD +// transition rules together with controller withdrawal, renewal, and UID pins. +func TestAccessRecoveryPreservesSourceFencesWithCEL(t *testing.T) { + assets := os.Getenv("KUBEBUILDER_ASSETS") + if assets == "" { + t.Skip("set KUBEBUILDER_ASSETS to run access recovery against the API server") + } + environment := &envtest.Environment{BinaryAssetsDirectory: assets, CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, ErrorIfCRDPathMissing: true} + cfg, err := environment.Start() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := environment.Stop(); err != nil { + t.Error(err) + } + }) + scheme := runtime.NewScheme() + requireAccessTestSuccess(t, dnsv1alpha1.AddToScheme(scheme)) + requireAccessTestSuccess(t, corev1.AddToScheme(scheme)) + cl, err := client.New(cfg, client.Options{Scheme: scheme}) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + for _, ns := range []string{"project", "platform"} { + requireAccessTestSuccess(t, cl.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}})) + } + now := time.Now().UTC().Truncate(time.Second) + resolverContext := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "context", Namespace: "project"}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "opaque-consumer"}} + readyContext := func(c *dnsv1alpha1.DNSResolverContext) { + t.Helper() + requireAccessTestSuccess(t, cl.Create(ctx, c)) + c.Status.AccessWriterEpoch = 1 + apimeta.SetStatusCondition(&c.Status.Conditions, metav1.Condition{Type: "Ready", Status: metav1.ConditionTrue, ObservedGeneration: c.Generation, Reason: "Accepted", LastTransitionTime: metav1.NewTime(now)}) + requireAccessTestSuccess(t, cl.Status().Update(ctx, c)) + } + readyContext(resolverContext) + deadline := metav1.NewTime(now.Add(time.Minute + 22357123*time.Nanosecond)) + access := &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "access", Namespace: "project"}, Spec: dnsv1alpha1.DNSResolverAccessBindingSpec{ContextRef: dnsv1alpha1.DNSObjectReference{Name: resolverContext.Name, UID: resolverContext.UID}, Region: "central", QueryIdentity: dnsv1alpha1.DNSResolverQueryIdentity{Type: dnsv1alpha1.DNSResolverQueryIdentityDestinationAddress, Value: "fd70:100::10"}, Port: 53, Transports: []string{"UDP", "TCP"}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 1, Sequence: 2, ValidUntil: deadline}}} + access.APIVersion = dnsv1alpha1.GroupVersion.String() + access.Kind = "DNSResolverAccessBinding" + raw, err := json.Marshal(access) + if err != nil { + t.Fatal(err) + } + // Submit unstructured JSON, as kubectl does, so the API initially stores a + // fractional deadline independently of the typed client encoder. + rawAccess := &unstructured.Unstructured{} + requireAccessTestSuccess(t, json.Unmarshal(raw, &rawAccess.Object)) + rawAccess.Object["spec"].(map[string]interface{})["authorization"].(map[string]interface{})["validUntil"] = deadline.UTC().Format(time.RFC3339Nano) + requireAccessTestSuccess(t, cl.Create(ctx, rawAccess)) + requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(access), access)) + r := &Reconciler{Client: cl, Scheme: scheme, Options: ReconcilerOptions{ProjectUID: "project-uid", PlatformNamespace: "platform", Region: "central", Shard: "shared", MaxAccessLease: 5 * time.Minute, AddressAllocator: &platform.Allocator{Client: cl, Namespace: "platform"}}} + reconcile := func(at time.Time) { + t.Helper() + requireAccessTestSuccess(t, r.reconcileContextBindings(ctx, "project", nil, map[string]*dnsv1alpha1.DNSZone{}, at)) + } + reconcile(now) + var bindings dnsv1alpha1.DNSResolverBindingList + requireAccessTestSuccess(t, cl.List(ctx, &bindings, client.InNamespace("platform"))) + if len(bindings.Items) != 1 { + t.Fatalf("got %d bindings", len(bindings.Items)) + } + binding := bindings.Items[0] + if !binding.Spec.Authorization.ValidUntil.Equal(&deadline) { + t.Fatalf("binding projection changed fractional deadline: %v != %v", binding.Spec.Authorization.ValidUntil, deadline) + } + bindingKey := client.ObjectKeyFromObject(&binding) + readBinding := func() { + t.Helper() + requireAccessTestSuccess(t, cl.Get(ctx, bindingKey, &binding)) + } + reconcile(now) + readBinding() + if binding.Spec.Tombstone || !binding.Spec.Authorization.ValidUntil.Equal(&deadline) { + t.Fatal("unchanged fractional authorization was withdrawn on second reconcile") + } + requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(access), access)) + if accepted := apimeta.FindStatusCondition(access.Status.Conditions, "Accepted"); accepted == nil || accepted.Status != metav1.ConditionTrue { + t.Fatal("fractional source authorization lost Accepted status") + } + subsecondMutation := binding.DeepCopy() + subsecondMutation.Spec.Authorization.ValidUntil = metav1.NewTime(deadline.Add(time.Nanosecond)) + if err := cl.Update(ctx, subsecondMutation); !apierrors.IsInvalid(err) { + t.Fatalf("same-fence nanosecond deadline mutation accepted: %v", err) + } + // False is omitted from the wire. An ordinary active update must still + // evaluate the optional tombstone transition rules without a CEL error. + active := binding.DeepCopy() + active.Labels["validation"] = "active-update" + if err := cl.Update(ctx, active); err != nil { + t.Fatalf("active update with omitted false tombstone failed: %v", err) + } + readBinding() + for _, change := range []struct { + name string + mutate func(*dnsv1alpha1.DNSResolverBinding) + }{ + {"context UID", func(b *dnsv1alpha1.DNSResolverBinding) { b.Spec.Source.ResolverContextRef.UID = "replacement-context" }}, + {"access UID", func(b *dnsv1alpha1.DNSResolverBinding) { b.Spec.Source.AccessBindingRef.UID = "replacement-access" }}, + {"placement", func(b *dnsv1alpha1.DNSResolverBinding) { b.Spec.Placement.Shard = "replacement-shard" }}, + {"listener", func(b *dnsv1alpha1.DNSResolverBinding) { b.Spec.Configuration.Listeners.Node.Address = "fd70:100::11" }}, + } { + mutated := binding.DeepCopy() + change.mutate(mutated) + mutated.Spec.Configuration.Revision++ + if err := cl.Update(ctx, mutated); !apierrors.IsInvalid(err) { + t.Fatalf("immutable %s change accepted: %v", change.name, err) + } + } + // Expiry advances the DNS-owned configuration fence, preserving the exact + // integration-owned authorization. Source sequence 3 remains available. + expiredAt := now.Add(61 * time.Second) + reconcile(expiredAt) + readBinding() + if !binding.Spec.Tombstone || binding.Spec.Configuration.Revision != 2 || binding.Spec.Authorization.Sequence != 2 || !binding.Spec.Authorization.ValidUntil.Equal(&deadline) { + t.Fatalf("withdrawal consumed source authority: %#v", binding.Spec) + } + requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(access), access)) + staleExpiredAccess := *access.DeepCopy() + // A direct same-fence replay cannot reopen the retired configuration. + replay := binding.DeepCopy() + replay.Spec.Tombstone = false + replay.Spec.Configuration.Revision++ + if err := cl.Update(ctx, replay); !apierrors.IsInvalid(err) { + t.Fatalf("same-fence reactivation accepted: %v", err) + } + mutated := binding.DeepCopy() + mutated.Spec.Authorization.ValidUntil = metav1.NewTime(expiredAt.Add(time.Minute + 987654321*time.Nanosecond)) + if err := cl.Update(ctx, mutated); !apierrors.IsInvalid(err) { + t.Fatalf("same-fence deadline mutation accepted: %v", err) + } + // Even if the source API has no admission webhook, the controller compares + // against its retained authorization fence and rejects a deadline replay. + requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(access), access)) + baseAccess := access.DeepCopy() + renewedDeadline := metav1.NewTime(expiredAt.Add(time.Minute + 987654321*time.Nanosecond)) + access.Spec.Authorization.ValidUntil = renewedDeadline + requireAccessTestSuccess(t, cl.Patch(ctx, access, client.MergeFromWithOptions(baseAccess, client.MergeFromWithOptimisticLock{}))) + requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(access), access)) + if !access.Spec.Authorization.ValidUntil.Equal(&renewedDeadline) { + t.Fatal("typed merge patch lost source deadline precision") + } + reconcile(expiredAt) + readBinding() + if !binding.Spec.Tombstone { + t.Fatal("same source fence reopened withdrawn access") + } + // Ordinary next-sequence renewal is independent of private DNS counters. + requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(access), access)) + access.Spec.Authorization.Sequence = 3 + requireAccessTestSuccess(t, cl.Update(ctx, access)) + reconcile(expiredAt) + readBinding() + if binding.Spec.Tombstone || binding.Spec.Authorization.Sequence != 3 || binding.Spec.Configuration.Revision != 3 || !binding.Spec.Authorization.ValidUntil.Equal(&access.Spec.Authorization.ValidUntil) { + t.Fatalf("next source sequence did not recover access: %#v", binding.Spec) + } + // Another controller can List expired source sequence 2 before this renewed + // platform binding is visible. Cleanup must freshly read sequence 3 instead + // of converting an unchanged stale rejection status into revocation. + staleController := &Reconciler{Client: &accessListSnapshotClient{Client: cl, accesses: []dnsv1alpha1.DNSResolverAccessBinding{staleExpiredAccess}}, Scheme: scheme, Options: r.Options} + requireAccessTestSuccess(t, staleController.reconcileContextBindings(ctx, "project", nil, map[string]*dnsv1alpha1.DNSZone{}, expiredAt)) + readBinding() + if binding.Spec.Tombstone || binding.Spec.Authorization.Sequence != 3 || binding.Spec.Configuration.Revision != 3 { + t.Fatalf("stale source List withdrew renewed binding: %#v", binding.Spec) + } + staleController.Client = &accessListSnapshotClient{Client: cl} + requireAccessTestSuccess(t, staleController.reconcileContextBindings(ctx, "project", nil, map[string]*dnsv1alpha1.DNSZone{}, expiredAt)) + readBinding() + if binding.Spec.Tombstone || binding.Spec.Configuration.Revision != 3 { + t.Fatal("omitted source List withdrew renewed binding") + } + // Deleting the context revokes access before its source deadline. Recreating + // the same name cannot satisfy the old UID-pinned integration reference. + oldContextUID := resolverContext.UID + requireAccessTestSuccess(t, cl.Delete(ctx, resolverContext)) + reconcile(expiredAt) + readBinding() + if !binding.Spec.Tombstone || binding.Spec.Authorization.Sequence != 3 || !binding.Spec.Authorization.ValidUntil.Equal(&access.Spec.Authorization.ValidUntil) { + t.Fatalf("context withdrawal rewrote source authority: %#v", binding.Spec) + } + freshContext := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "context", Namespace: "project"}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "opaque-consumer"}} + readyContext(freshContext) + if freshContext.UID == oldContextUID { + t.Fatal("context UID did not rotate") + } + reconcile(expiredAt) + readBinding() + if !binding.Spec.Tombstone { + t.Fatal("recreated context name restored old access") + } + // A new access UID cannot reuse a quarantined destination. A fresh protected + // destination creates a separate binding and leaves the retired one closed. + requireAccessTestSuccess(t, cl.Delete(ctx, access)) + freshAccess := &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "access", Namespace: "project"}, Spec: access.Spec} + freshAccess.Spec.ContextRef = dnsv1alpha1.DNSObjectReference{Name: freshContext.Name, UID: freshContext.UID} + freshAccess.Spec.Authorization.Sequence = 1 + requireAccessTestSuccess(t, cl.Create(ctx, freshAccess)) + requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(freshAccess), freshAccess)) + if !freshAccess.Spec.Authorization.ValidUntil.Equal(&renewedDeadline) { + t.Fatal("typed create lost source deadline precision") + } + reconcile(expiredAt) + requireAccessTestSuccess(t, cl.List(ctx, &bindings, client.InNamespace("platform"))) + if len(bindings.Items) != 1 { + t.Fatal("new access lifetime reused a protected destination") + } + requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(freshAccess), freshAccess)) + freshAccess.Spec.QueryIdentity.Value = "fd70:100::11" + freshAccess.Spec.Authorization.Sequence = 2 + requireAccessTestSuccess(t, cl.Update(ctx, freshAccess)) + reconcile(expiredAt) + requireAccessTestSuccess(t, cl.List(ctx, &bindings, client.InNamespace("platform"))) + if len(bindings.Items) != 2 { + t.Fatalf("fresh protected identity created %d bindings", len(bindings.Items)) + } + readBinding() + if !binding.Spec.Tombstone { + t.Fatal("old binding reopened after access UID recreation") + } + // A renewal committed after the source confirmation but before destructive + // platform Patch must win through the platform resourceVersion fence. + var freshBinding dnsv1alpha1.DNSResolverBinding + for _, candidate := range bindings.Items { + if candidate.Spec.Source.AccessBindingRef.UID == freshAccess.UID { + freshBinding = candidate + } + } + if freshBinding.UID == "" { + t.Fatal("fresh access binding missing") + } + requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(freshAccess), freshAccess)) + freshAccess.Spec.Authorization.Sequence = 3 + freshAccess.Spec.Authorization.ValidUntil = metav1.NewTime(expiredAt.Add(-time.Second)) + requireAccessTestSuccess(t, cl.Update(ctx, freshAccess)) + raceClient := &bindingPatchRaceClient{Client: cl, beforeWithdrawal: func(ctx context.Context) error { + if err := cl.Get(ctx, client.ObjectKeyFromObject(freshAccess), freshAccess); err != nil { + return err + } + freshAccess.Spec.Authorization.Sequence = 4 + freshAccess.Spec.Authorization.ValidUntil = renewedDeadline + if err := cl.Update(ctx, freshAccess); err != nil { + return err + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(&freshBinding), &freshBinding); err != nil { + return err + } + freshBinding.Spec.Authorization = freshAccess.Spec.Authorization + freshBinding.Spec.Configuration.Revision++ + return cl.Update(ctx, &freshBinding) + }} + raceController := &Reconciler{Client: &accessListSnapshotClient{Client: raceClient}, Scheme: scheme, Options: r.Options} + if err := raceController.reconcileContextBindings(ctx, "project", nil, nil, expiredAt); !apierrors.IsConflict(err) { + t.Fatalf("concurrent platform renewal did not fence withdrawal: %v", err) + } + requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(&freshBinding), &freshBinding)) + if freshBinding.Spec.Tombstone || freshBinding.Spec.Authorization.Sequence != 4 || !freshBinding.Spec.Authorization.ValidUntil.Equal(&renewedDeadline) { + t.Fatal("destructive cleanup overwrote concurrent platform renewal") + } +} + +func requireAccessTestSuccess(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } +} diff --git a/internal/internaldns/controlplane/context_reconciler.go b/internal/internaldns/controlplane/context_reconciler.go new file mode 100644 index 00000000..24de7a80 --- /dev/null +++ b/internal/internaldns/controlplane/context_reconciler.go @@ -0,0 +1,680 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controlplane + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "net/netip" + "reflect" + "sort" + "time" + + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" + "go.miloapis.com/dns-operator/internal/internaldns/model" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +func jsonEqual(a, b any) bool { return reflect.DeepEqual(a, b) } + +func (r *Reconciler) reconcileNamingPolicies(ctx context.Context, items []dnsv1alpha1.DNSNamingPolicy, assocs []dnsv1alpha1.DNSZoneAssociation, zones map[string]*dnsv1alpha1.DNSZone, now time.Time) error { + for i := range items { + p := &items[i] + vpcUID, contextUID, contextName := types.UID(""), types.UID(""), "" + valid := false + + var resolverContext dnsv1alpha1.DNSResolverContext + err := r.Get(ctx, client.ObjectKey{Namespace: p.Namespace, Name: p.Spec.ResolverContextRef.Name}, &resolverContext) + ready := apimeta.FindStatusCondition(resolverContext.Status.Conditions, "Ready") + valid = err == nil && p.Spec.ResolverContextRef.UID != "" && p.Spec.ResolverContextRef.UID == resolverContext.UID && resolverContext.DeletionTimestamp.IsZero() && ready != nil && ready.Status == metav1.ConditionTrue && ready.ObservedGeneration == resolverContext.Generation + if valid { + vpcUID = types.UID(resolverContext.Spec.ConsumerID) + contextUID = resolverContext.UID + contextName = resolverContext.Name + } + + resolved := []dnsv1alpha1.DNSResolvedAdditionalNameRule{} + seen := map[string]bool{} + for _, rule := range p.Spec.AdditionalNames { + z := zones[rule.DNSZoneRef.Name] + associated := associatedWithContext(assocs, zUID(z), contextUID) + + if z == nil || z.Spec.Visibility != dnsv1alpha1.DNSZoneVisibilityPrivate || !refMatches(rule.DNSZoneRef, z.Name, z.UID, z.Generation) || !associated { + valid = false + continue + } + key := string(rule.RegistrationClass) + "\x00" + canonicalName(rule.NamePrefix) + if rule.NamePrefix == "" || seen[key] { + valid = false + continue + } + seen[key] = true + resolved = append(resolved, dnsv1alpha1.DNSResolvedAdditionalNameRule{RegistrationClass: rule.RegistrationClass, DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: z.Name, UID: z.UID, Generation: z.Generation}, NamePrefix: canonicalName(rule.NamePrefix)}) + } + base := p.DeepCopy() + p.Status.ResolvedVPCRef = dnsv1alpha1.DNSObjectReference{Name: p.Spec.VPCRef.Name, UID: vpcUID} + + p.Status.ResolvedVPCRef.Name = contextName + p.Status.ResolvedResolverContextRef = dnsv1alpha1.DNSObjectReference{Name: contextName, UID: contextUID} + + p.Status.ResolvedAdditionalNames = resolved + status := metav1.ConditionTrue + reason, msg := dnsValueAccepted, "naming policy is authorized" + if !valid { + status = metav1.ConditionFalse + reason = "InvalidPolicy" + msg = "all rules require an associated private zone and unique class/prefix" + } + if setCondition(&p.Status.Conditions, dnsValueAccepted, status, reason, msg, p.Generation, now) { + if err := r.Status().Patch(ctx, p, client.MergeFrom(base)); err != nil { + return err + } + } + } + return nil +} + +func zUID(z *dnsv1alpha1.DNSZone) types.UID { + if z == nil { + return "" + } + return z.UID +} + +func associatedWithContext(assocs []dnsv1alpha1.DNSZoneAssociation, zoneUID, contextUID types.UID) bool { + for _, a := range assocs { + c := apimeta.FindStatusCondition(a.Status.Conditions, dnsValueAccepted) + if c != nil && c.Status == metav1.ConditionTrue && a.Status.ResolvedDNSZoneRef.UID == zoneUID && a.Status.ResolvedResolverContextRef.UID == contextUID { + return true + } + } + return false +} + +func (r *Reconciler) reconcileManagedNamespaces(ctx context.Context, ns string, now time.Time) error { + + var list dnsv1alpha1.DNSManagedNamespaceList + if err := r.List(ctx, &list, client.InNamespace(ns)); err != nil { + return err + } + for i := range list.Items { + m := &list.Items[i] + if m.Spec.ProjectUID != r.Options.ProjectUID || m.Spec.VPCRef.UID == "" { + if err := r.patchCondition(ctx, m, dnsValueAccepted, metav1.ConditionFalse, "InvalidIdentity", "project and pinned VPC UID are required", now); err != nil { + return err + } + continue + } + consumerToken := model.OpaqueToken(string(m.Spec.VPCRef.UID)) + lifetimeToken := model.OpaqueToken(m.Labels["internal-dns.miloapis.com/context-uid"]) + + suffix := canonicalName(consumerToken + "." + m.Spec.DomainSuffix) + zoneName := "managed-" + lifetimeToken[:12] + zone := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: zoneName, Namespace: ns, Labels: map[string]string{"dns.networking.miloapis.com/managed": "true"}}, Spec: dnsv1alpha1.DNSZoneSpec{DomainName: suffix, DNSZoneClassName: m.Spec.DNSZoneClassName, Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}} + if r.Scheme != nil { + if err := controllerutil.SetControllerReference(m, zone, r.Scheme); err != nil { + return err + } + } + if err := r.createIfAbsent(ctx, zone); err != nil { + return err + } + if err := r.Get(ctx, client.ObjectKeyFromObject(zone), zone); err != nil { + return err + } + assocName := zoneName + "-vpc" + assocSpec := dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name, UID: zone.UID}, VPCRef: m.Spec.VPCRef, Managed: true} + + assocSpec.VPCRef = dnsv1alpha1.DNSObjectReference{} + var contextObject dnsv1alpha1.DNSResolverContext + if err := r.Get(ctx, client.ObjectKey{Namespace: ns, Name: m.Spec.VPCRef.Name}, &contextObject); err != nil { + return err + } + assocSpec.ResolverContextRef = dnsv1alpha1.DNSObjectReference{Name: contextObject.Name, UID: contextObject.UID} + + assoc := &dnsv1alpha1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: assocName, Namespace: ns, Labels: map[string]string{"dns.networking.miloapis.com/managed": "true"}}, Spec: assocSpec} + if r.Scheme != nil { + if err := controllerutil.SetControllerReference(m, assoc, r.Scheme); err != nil { + return err + } + } + if err := r.createIfAbsent(ctx, assoc); err != nil { + return err + } + if err := r.Get(ctx, client.ObjectKeyFromObject(assoc), assoc); err != nil { + return err + } + base := m.DeepCopy() + accepted := apimeta.FindStatusCondition(assoc.Status.Conditions, dnsValueAccepted) + ready := accepted != nil && accepted.Status == metav1.ConditionTrue && assoc.Status.ResolvedDNSZoneRef.UID == zone.UID + if ready { + m.Status.DNSZoneRef = dnsv1alpha1.DNSObjectReference{Name: zone.Name, UID: zone.UID} + m.Status.AssociationRef = dnsv1alpha1.DNSObjectReference{Name: assoc.Name, UID: assoc.UID} + m.Status.CanonicalSuffix = suffix + setCondition(&m.Status.Conditions, dnsValueAccepted, metav1.ConditionTrue, "Ready", "managed namespace is allocated", m.Generation, now) + } else { + m.Status.DNSZoneRef = dnsv1alpha1.DNSObjectReference{} + m.Status.AssociationRef = dnsv1alpha1.DNSObjectReference{} + m.Status.CanonicalSuffix = "" + setCondition(&m.Status.Conditions, dnsValueAccepted, metav1.ConditionFalse, "PendingAssociation", "managed namespace association is not accepted", m.Generation, now) + } + if err := r.Status().Patch(ctx, m, client.MergeFrom(base)); err != nil && !apierrors.IsNotFound(err) { + return err + } + } + return nil +} + +func (r *Reconciler) reconcileResolverContexts(ctx context.Context, ns string, now time.Time) error { + var contexts dnsv1alpha1.DNSResolverContextList + if err := r.List(ctx, &contexts, client.InNamespace(ns)); err != nil { + return err + } + sort.Slice(contexts.Items, func(i, j int) bool { return contexts.Items[i].Name < contexts.Items[j].Name }) + consumerOwners := map[string]types.UID{} + for i := range contexts.Items { + c := &contexts.Items[i] + base := c.DeepCopy() + valid := c.Spec.ConsumerID != "" && c.DeletionTimestamp.IsZero() + duplicate := consumerOwners[c.Spec.ConsumerID] != "" && consumerOwners[c.Spec.ConsumerID] != c.UID + if valid && !duplicate { + consumerOwners[c.Spec.ConsumerID] = c.UID + } + valid = valid && !duplicate + previousReady := apimeta.FindStatusCondition(c.Status.Conditions, "Ready") + if !valid && c.Status.AccessWriterEpoch > 0 && previousReady != nil && previousReady.Status == metav1.ConditionTrue { + c.Status.AccessWriterEpoch++ + } + if valid && c.Status.AccessWriterEpoch == 0 { + c.Status.AccessWriterEpoch = 1 + } + c.Status.ServingTargets = nil + if valid { + for _, target := range r.Options.servingRegions() { + c.Status.ServingTargets = append(c.Status.ServingTargets, dnsv1alpha1.DNSResolverContextServingTarget{Region: target.Region, Shard: target.Shard}) + } + } + ready := valid && !c.Spec.ManagedNamespace.Enabled + if c.Spec.ManagedNamespace.Enabled && valid { + name := "context-" + model.OpaqueToken(string(c.UID))[:20] + m := &dnsv1alpha1.DNSManagedNamespace{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: map[string]string{"internal-dns.miloapis.com/platform-owned": "true", "internal-dns.miloapis.com/context-uid": string(c.UID)}}, Spec: dnsv1alpha1.DNSManagedNamespaceSpec{ProjectUID: r.Options.ProjectUID, VPCRef: dnsv1alpha1.DNSObjectReference{Name: c.Name, UID: types.UID(c.Spec.ConsumerID)}, DomainSuffix: r.Options.ManagedDomainSuffix, DNSZoneClassName: r.Options.PrivateZoneClassName}} + if r.Scheme != nil { + if err := controllerutil.SetControllerReference(c, m, r.Scheme); err != nil { + return err + } + } + if err := r.createIfAbsent(ctx, m); err != nil { + return err + } + if err := r.Get(ctx, client.ObjectKeyFromObject(m), m); err != nil { + return err + } + c.Status.ManagedNamespace.DNSZoneRef = m.Status.DNSZoneRef + c.Status.ManagedNamespace.Suffix = m.Status.CanonicalSuffix + if condition := apimeta.FindStatusCondition(m.Status.Conditions, dnsValueAccepted); condition != nil && condition.Status == metav1.ConditionTrue && m.Status.DNSZoneRef.UID != "" { + ready = true + } + } + status, reason, msg := metav1.ConditionTrue, dnsValueAccepted, "resolver context is active" + if !valid { + status, reason, msg = metav1.ConditionFalse, "InvalidIdentity", "consumerID is required and context must be live" + if duplicate { + reason, msg = "DuplicateConsumerID", "another live resolver context owns this consumerID" + } + } + if valid && !ready { + status, reason, msg = metav1.ConditionFalse, "Preparing", "managed namespace is not ready" + } + setCondition(&c.Status.Conditions, "Ready", status, reason, msg, c.Generation, now) + if !jsonEqual(base.Status, c.Status) { + if err := r.Status().Patch(ctx, c, client.MergeFrom(base)); err != nil { + return err + } + } + } + return nil +} + +func (r *Reconciler) reconcileAssociations(ctx context.Context, items []dnsv1alpha1.DNSZoneAssociation, zones map[string]*dnsv1alpha1.DNSZone, now time.Time) error { + type apexKey struct { + vpc types.UID + apex string + } + owners := map[apexKey]types.UID{} + sort.Slice(items, func(i, j int) bool { return items[i].Name < items[j].Name }) + for i := range items { + a := &items[i] + z := zones[a.Spec.DNSZoneRef.Name] + status := metav1.ConditionTrue + reason, msg := dnsValueAccepted, "association accepted" + vpcUID, vpcErr := types.UID(""), error(nil) + + var resolverContext dnsv1alpha1.DNSResolverContext + vpcErr = r.Get(ctx, client.ObjectKey{Namespace: a.Namespace, Name: a.Spec.ResolverContextRef.Name}, &resolverContext) + if vpcErr == nil && (a.Spec.ResolverContextRef.UID == "" || a.Spec.ResolverContextRef.UID != resolverContext.UID || !resolverContext.DeletionTimestamp.IsZero()) { + vpcErr = errors.New("resolver context lifetime is not current") + } + if vpcErr == nil { + vpcUID = resolverContext.UID + } + + if !a.DeletionTimestamp.IsZero() || z == nil || z.Spec.Visibility != dnsv1alpha1.DNSZoneVisibilityPrivate || vpcErr != nil || !refMatches(a.Spec.DNSZoneRef, z.Name, z.UID, z.Generation) { + status = metav1.ConditionFalse + reason = "InvalidReference" + msg = "private zone and pinned VPC/zone UIDs are required" + } else if old := owners[apexKey{vpcUID, canonicalName(z.Spec.DomainName)}]; old != "" && old != z.UID { + status = metav1.ConditionFalse + reason = "ApexConflict" + msg = "another zone owns this apex in the VPC" + } else { + owners[apexKey{vpcUID, canonicalName(z.Spec.DomainName)}] = z.UID + } + base := a.DeepCopy() + a.Status.ResolvedVPCRef = dnsv1alpha1.DNSObjectReference{Name: a.Spec.VPCRef.Name, UID: vpcUID} + + a.Status.ResolvedVPCRef = dnsv1alpha1.DNSObjectReference{} + a.Status.ResolvedResolverContextRef = dnsv1alpha1.DNSObjectReference{Name: a.Spec.ResolverContextRef.Name, UID: vpcUID} + + if z != nil { + a.Status.ResolvedDNSZoneRef = dnsv1alpha1.DNSObjectReference{Name: z.Name, UID: z.UID, Generation: z.Generation} + } + if setCondition(&a.Status.Conditions, dnsValueAccepted, status, reason, msg, a.Generation, now) { + if err := r.Status().Patch(ctx, a, client.MergeFrom(base)); err != nil { + return err + } + } + } + return nil +} + +func (r *Reconciler) reconcileBindings(ctx context.Context, ns string, assocs []dnsv1alpha1.DNSZoneAssociation, zones map[string]*dnsv1alpha1.DNSZone, now time.Time) error { + return r.reconcileContextBindings(ctx, ns, assocs, zones, now) +} + +func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, assocs []dnsv1alpha1.DNSZoneAssociation, zones map[string]*dnsv1alpha1.DNSZone, now time.Time) error { + var accesses dnsv1alpha1.DNSResolverAccessBindingList + var contexts dnsv1alpha1.DNSResolverContextList + var existing dnsv1alpha1.DNSResolverBindingList + if err := r.List(ctx, &accesses, client.InNamespace(ns)); err != nil { + return err + } + if err := r.List(ctx, &contexts, client.InNamespace(ns)); err != nil { + return err + } + if err := r.List(ctx, &existing, client.InNamespace(r.Options.PlatformNamespace)); err != nil { + return err + } + bindingsByName := map[string]*dnsv1alpha1.DNSResolverBinding{} + for i := range existing.Items { + b := &existing.Items[i] + if b.Spec.Source.ProjectUID == r.Options.ProjectUID { + bindingsByName[b.Name] = b + } + } + contextByUID := map[types.UID]*dnsv1alpha1.DNSResolverContext{} + for i := range contexts.Items { + contextByUID[contexts.Items[i].UID] = &contexts.Items[i] + } + zonesByContext := map[types.UID][]types.UID{} + for _, a := range assocs { + z := zones[a.Spec.DNSZoneRef.Name] + accepted := apimeta.FindStatusCondition(a.Status.Conditions, dnsValueAccepted) + if z != nil && accepted != nil && accepted.Status == metav1.ConditionTrue { + zonesByContext[a.Status.ResolvedResolverContextRef.UID] = append(zonesByContext[a.Status.ResolvedResolverContextRef.UID], z.UID) + } + } + sort.Slice(accesses.Items, func(i, j int) bool { return accesses.Items[i].Name < accesses.Items[j].Name }) + destinations := map[string]types.UID{} + desired := map[string]bool{} + for i := range accesses.Items { + a := &accesses.Items[i] + base := a.DeepCopy() + spec := a.Spec + contextObject := contextByUID[spec.ContextRef.UID] + contextReady := resolverContextReady(contextObject) + valid := a.DeletionTimestamp.IsZero() && contextObject != nil && contextObject.DeletionTimestamp.IsZero() && contextObject.Name == spec.ContextRef.Name && contextReady && contextObject.Status.AccessWriterEpoch > 0 && spec.Authorization.WriterEpoch == contextObject.Status.AccessWriterEpoch && spec.Authorization.Sequence >= a.Status.ObservedSequence && spec.Authorization.ValidUntil.After(now) && !spec.Authorization.ValidUntil.After(now.Add(r.Options.MaxAccessLease)) + bindingName := "binding-" + model.OpaqueToken(string(r.Options.ProjectUID) + "/" + string(a.UID))[:20] + if prior := bindingsByName[bindingName]; prior != nil { + compare := model.Compare(uint64(spec.Authorization.WriterEpoch), uint64(spec.Authorization.Sequence), uint64(prior.Spec.Authorization.WriterEpoch), uint64(prior.Spec.Authorization.Sequence)) + if compare < 0 || (compare == 0 && (!spec.Authorization.ValidUntil.Equal(&prior.Spec.Authorization.ValidUntil) || prior.Spec.Tombstone)) { + valid = false + } + } + if _, err := netip.ParseAddr(spec.QueryIdentity.Value); err != nil || spec.QueryIdentity.Type != dnsv1alpha1.DNSResolverQueryIdentityDestinationAddress || spec.Port < 1 { + valid = false + } + if _, ok := r.targetForRegion(spec.Region); !ok || spec.Port != 53 || !validAccessTransports(spec.Transports) { + valid = false + } + key := spec.Region + "\x00" + spec.QueryIdentity.Value + "\x00" + fmt.Sprint(spec.Port) + if owner := destinations[key]; owner != "" && owner != a.UID { + valid = false + } else if valid { + if r.Options.AddressAllocator == nil || r.Options.AddressAllocator.ClaimDestination(ctx, r.Options.ProjectUID, a.UID, spec.Region, netip.MustParseAddr(spec.QueryIdentity.Value).String(), spec.Port) != nil { + valid = false + } else { + destinations[key] = a.UID + } + } + status, reason, msg := metav1.ConditionTrue, dnsValueAccepted, "access binding is current" + if !valid { + status, reason, msg = metav1.ConditionFalse, "InvalidOrStaleAccess", "context lifetime, epoch, sequence, destination, and bounded deadline must be current" + } + setCondition(&a.Status.Conditions, dnsValueAccepted, status, reason, msg, a.Generation, now) + if !valid { + a.Status.BindingRef = nil + setCondition(&a.Status.Conditions, "Ready", metav1.ConditionFalse, "AccessRejected", "access binding is not accepted", a.Generation, now) + if !jsonEqual(base.Status, a.Status) { + if err := r.Status().Patch(ctx, a, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { + return err + } + } + continue + } + zids := append([]types.UID(nil), zonesByContext[contextObject.UID]...) + sort.Slice(zids, func(i, j int) bool { return zids[i] < zids[j] }) + name := "binding-" + model.OpaqueToken(string(r.Options.ProjectUID) + "/" + string(a.UID))[:20] + desired[name] = true + var old dnsv1alpha1.DNSResolverBinding + err := r.Get(ctx, client.ObjectKey{Namespace: r.Options.PlatformNamespace, Name: name}, &old) + exists := err == nil + if err != nil && !apierrors.IsNotFound(err) { + return err + } + cluster := "" + generation, revision := int64(1), int64(1) + if exists { + cluster = old.Spec.Configuration.Listeners.Regional.Address + generation = old.Spec.Configuration.Generation + revision = old.Spec.Configuration.Revision + if !zoneUIDsEqual(old.Spec.Configuration.ZoneRefs, zids) { + generation++ + revision++ + } else if old.Spec.Tombstone || old.Spec.Authorization.WriterEpoch != spec.Authorization.WriterEpoch || old.Spec.Authorization.Sequence != spec.Authorization.Sequence { + revision++ + } + } else { + if r.Options.AddressAllocator == nil { + return errors.New("address allocator is required for context binding") + } + _, cluster, err = r.Options.AddressAllocator.Allocate(ctx, r.Options.ProjectUID, contextObject.UID) + if err != nil { + return err + } + } + binding := &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: r.Options.PlatformNamespace, Labels: map[string]string{"internal-dns.miloapis.com/context-uid": string(contextObject.UID), "internal-dns.miloapis.com/access-uid": string(a.UID), "internal-dns.miloapis.com/region": model.SafeToken(spec.Region), "internal-dns.miloapis.com/shard": model.SafeToken(r.shardForRegion(spec.Region))}}, Spec: dnsv1alpha1.DNSResolverBindingSpec{Source: dnsv1alpha1.DNSResolverBindingSource{ProjectUID: r.Options.ProjectUID, ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: contextObject.Name, UID: contextObject.UID}, AccessBindingRef: dnsv1alpha1.DNSObjectReference{Name: a.Name, UID: a.UID}}, Placement: dnsv1alpha1.DNSResolverBindingPlacement{Region: spec.Region, Shard: r.shardForRegion(spec.Region)}, Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Generation: generation, Revision: revision, Listeners: dnsv1alpha1.DNSResolverBindingListeners{Node: dnsv1alpha1.DNSResolverBindingListener{Address: spec.QueryIdentity.Value, Port: spec.Port, Transports: append([]string(nil), spec.Transports...)}, Regional: dnsv1alpha1.DNSResolverBindingListener{Address: cluster, Port: spec.Port, Transports: append([]string(nil), spec.Transports...)}}, ZoneRefs: zoneReferences(zids, zones)}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: spec.Authorization.WriterEpoch, Sequence: spec.Authorization.Sequence, ValidUntil: spec.Authorization.ValidUntil}}} + if !exists { + if err := r.Create(ctx, binding); err != nil { + return err + } + if err := r.Get(ctx, client.ObjectKeyFromObject(binding), binding); err != nil { + return err + } + } else if !bindingEqual(old.Spec, binding.Spec) { + baseOld := old.DeepCopy() + old.Spec = binding.Spec + old.Labels = binding.Labels + if err := r.Patch(ctx, &old, client.MergeFromWithOptions(baseOld, client.MergeFromWithOptimisticLock{})); err != nil { + return err + } + binding = &old + } else { + binding = &old + } + a.Status.ObservedSequence = spec.Authorization.Sequence + a.Status.BindingRef = &dnsv1alpha1.DNSObjectReference{Name: binding.Name, UID: binding.UID} + setCondition(&a.Status.Conditions, "Ready", metav1.ConditionFalse, "PendingReplicaVerification", "serving fleet acknowledgement is pending", a.Generation, now) + if c := apimeta.FindStatusCondition(binding.Status.Conditions, "Serving"); c != nil && c.Status == metav1.ConditionTrue && binding.Status.ObservedConfigurationRevision == binding.Spec.Configuration.Revision && bindingAckCurrent(binding, now) { + setCondition(&a.Status.Conditions, "Ready", metav1.ConditionTrue, "Ready", "required serving replicas applied this access generation", a.Generation, now) + } + if !jsonEqual(base.Status, a.Status) { + if err := r.Status().Patch(ctx, a, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { + return err + } + } + } + for i := range existing.Items { + b := &existing.Items[i] + if b.Spec.Source.ProjectUID != r.Options.ProjectUID || desired[b.Name] || b.Spec.Tombstone { + continue + } + if b.Labels["internal-dns.miloapis.com/context-uid"] == "" { + continue + } + withdraw, err := r.confirmAccessWithdrawal(ctx, ns, b, now) + if err != nil { + return err + } + if !withdraw { + continue + } + base := b.DeepCopy() + b.Spec.Tombstone = true + b.Spec.Configuration.Revision++ + // Authorization belongs to the trusted integration. Preserve its fence and + // deadline; withdrawal is ordered independently by configuration revision. + if err := r.Patch(ctx, b, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { + return err + } + } + return nil +} + +// confirmAccessWithdrawal prevents a stale or incomplete source List from +// withdrawing a binding renewed by another controller. Read uncertainty leaves +// the existing authorization intact; serving agents still enforce its deadline. +func (r *Reconciler) confirmAccessWithdrawal(ctx context.Context, namespace string, binding *dnsv1alpha1.DNSResolverBinding, now time.Time) (bool, error) { + var access dnsv1alpha1.DNSResolverAccessBinding + if err := r.Get(ctx, client.ObjectKey{Namespace: namespace, Name: binding.Spec.Source.AccessBindingRef.Name}, &access); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, err + } + if access.UID != binding.Spec.Source.AccessBindingRef.UID || !access.DeletionTimestamp.IsZero() { + return true, nil + } + authorization := access.Spec.Authorization + prior := binding.Spec.Authorization + compare := model.Compare(uint64(authorization.WriterEpoch), uint64(authorization.Sequence), uint64(prior.WriterEpoch), uint64(prior.Sequence)) + if compare < 0 || (compare == 0 && !authorization.ValidUntil.Equal(&prior.ValidUntil)) { + return false, nil + } + if access.Spec.ContextRef.Name != binding.Spec.Source.ResolverContextRef.Name || access.Spec.ContextRef.UID != binding.Spec.Source.ResolverContextRef.UID { + return true, nil + } + var resolverContext dnsv1alpha1.DNSResolverContext + if err := r.Get(ctx, client.ObjectKey{Namespace: namespace, Name: binding.Spec.Source.ResolverContextRef.Name}, &resolverContext); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, err + } + if resolverContext.UID != binding.Spec.Source.ResolverContextRef.UID || !resolverContext.DeletionTimestamp.IsZero() { + return true, nil + } + ready := apimeta.FindStatusCondition(resolverContext.Status.Conditions, "Ready") + if !authorization.ValidUntil.After(now) || resolverContext.Status.AccessWriterEpoch > authorization.WriterEpoch { + return true, nil + } + // Readiness or epoch propagation can lag a healthy authorization. Such + // uncertainty is not retirement evidence and must not consume its lease. + if ready == nil || ready.Status != metav1.ConditionTrue || ready.ObservedGeneration != resolverContext.Generation || resolverContext.Status.AccessWriterEpoch < authorization.WriterEpoch { + return false, nil + } + valid := access.Spec.Region == binding.Spec.Placement.Region && access.Spec.QueryIdentity.Type == dnsv1alpha1.DNSResolverQueryIdentityDestinationAddress && access.Spec.QueryIdentity.Value == binding.Spec.Configuration.Listeners.Node.Address && access.Spec.Port == binding.Spec.Configuration.Listeners.Node.Port && validAccessTransports(access.Spec.Transports) + return !valid, nil +} + +func (r *Reconciler) shardForRegion(region string) string { + t, _ := r.targetForRegion(region) + return t.Shard +} + +func (r *Reconciler) targetForRegion(region string) (ServingRegion, bool) { + for _, s := range r.Options.servingRegions() { + if s.Region == region { + return s, true + } + } + return ServingRegion{}, false +} + +func validAccessTransports(values []string) bool { + if len(values) != 2 { + return false + } + seen := map[string]bool{} + for _, v := range values { + if (v != "UDP" && v != "TCP") || seen[v] { + return false + } + seen[v] = true + } + return seen["UDP"] && seen["TCP"] +} + +func bindingAckCurrent(binding *dnsv1alpha1.DNSResolverBinding, now time.Time) bool { + for _, ack := range binding.Status.MemberAcknowledgements { + // Serving=True is the aggregate full-member gate computed by AckSink. + // This row check additionally prevents a stale condition from surviving + // past the exact configuration/authorization generation or its local ACK + // lease. An ACK lease may be shorter than the access deadline, but never + // longer because AckSink caps it before persisting the row. + if ack.Phase == string(model.AckVerified) && ack.Revision == binding.Spec.Configuration.Revision && ack.AuthorizationIssuerEpoch == binding.Spec.Authorization.WriterEpoch && ack.AuthorizationRevision == binding.Spec.Authorization.Sequence && ack.ValidUntil != nil && ack.ValidUntil.After(now) && !ack.ValidUntil.After(binding.Spec.Authorization.ValidUntil.Time) { + return true + } + } + return false +} + +// selectServingTargets assigns one shared shard in each region. An existing +// live assignment wins while its shard remains configured, so adding capacity +// does not silently move established VPC listeners. Explicit migration is a +// separate operation; only a missing/removed assignment invokes rendezvous. +func (r *Reconciler) selectServingTargets(vpc types.UID, existing []dnsv1alpha1.DNSResolverBinding) []ServingRegion { + byRegion := map[string][]ServingRegion{} + for _, target := range r.Options.servingRegions() { + byRegion[target.Region] = append(byRegion[target.Region], target) + } + regions := make([]string, 0, len(byRegion)) + for region := range byRegion { + regions = append(regions, region) + } + sort.Strings(regions) + var selected []ServingRegion + for _, region := range regions { + candidates := byRegion[region] + sort.Slice(candidates, func(i, j int) bool { return candidates[i].Shard < candidates[j].Shard }) + var preserved *ServingRegion + for _, binding := range existing { + if binding.Spec.Source.ProjectUID != r.Options.ProjectUID || binding.Spec.Source.ResolverContextRef.UID != vpc || binding.Spec.Placement.Region != region || binding.Spec.Tombstone || !binding.DeletionTimestamp.IsZero() { + continue + } + for _, candidate := range candidates { + if binding.Spec.Placement.Shard == candidate.Shard { + c := candidate + if preserved == nil || c.Shard < preserved.Shard { + preserved = &c + } + } + } + } + if preserved != nil { + selected = append(selected, *preserved) + continue + } + best := candidates[0] + bestScore := "" + for _, candidate := range candidates { + sum := sha256.Sum256([]byte(string(r.Options.ProjectUID) + "/" + string(vpc) + "/" + region + "/" + candidate.Shard)) + score := string(sum[:]) + if bestScore == "" || score > bestScore { + best, bestScore = candidate, score + } + } + selected = append(selected, best) + } + return selected +} + +func (r *Reconciler) createIfAbsent(ctx context.Context, o client.Object) error { + err := r.Create(ctx, o) + if apierrors.IsAlreadyExists(err) { + return nil + } + return err +} + +func (r *Reconciler) patchCondition(ctx context.Context, m *dnsv1alpha1.DNSManagedNamespace, t string, status metav1.ConditionStatus, reason, msg string, now time.Time) error { + base := m.DeepCopy() + if !setCondition(&m.Status.Conditions, t, status, reason, msg, m.Generation, now) { + return nil + } + return r.Status().Patch(ctx, m, client.MergeFrom(base)) +} + +func setCondition(cs *[]metav1.Condition, t string, status metav1.ConditionStatus, reason, msg string, generation int64, now time.Time) bool { + return apimeta.SetStatusCondition(cs, metav1.Condition{Type: t, Status: status, Reason: reason, Message: msg, ObservedGeneration: generation, LastTransitionTime: metav1.NewTime(now)}) +} + +// BindingAddressCandidates supplies stable IPv6 candidates to a durable global +// allocator. Callers must collision-check, persist, and quarantine leases. +func BindingAddressCandidates(uid types.UID) (string, string) { + sum := sha256.Sum256([]byte(uid)) + a := binary.BigEndian.Uint64(sum[:8]) + b := binary.BigEndian.Uint64(sum[8:16]) + return netip.AddrFrom16([16]byte{0xfd, 0x53, byte(a >> 56), byte(a >> 48), byte(a >> 40), byte(a >> 32), byte(a >> 24), byte(a >> 16), byte(a >> 8), byte(a), byte(b >> 40), byte(b >> 32), byte(b >> 24), byte(b >> 16), byte(b >> 8), byte(b)}).String(), netip.AddrFrom16([16]byte{0xfd, 0x54, byte(a >> 56), byte(a >> 48), byte(a >> 40), byte(a >> 32), byte(a >> 24), byte(a >> 16), byte(a >> 8), byte(a), byte(b >> 40), byte(b >> 32), byte(b >> 24), byte(b >> 16), byte(b >> 8), byte(b)}).String() +} + +func bindingEqual(a, b dnsv1alpha1.DNSResolverBindingSpec) bool { + x := a + y := b + x.Authorization.ValidUntil = metav1.Time{} + y.Authorization.ValidUntil = metav1.Time{} + xb, _ := json.Marshal(x) + yb, _ := json.Marshal(y) + return string(xb) == string(yb) +} + +func zoneReferences(uids []types.UID, zones map[string]*dnsv1alpha1.DNSZone) []dnsv1alpha1.DNSObjectReference { + refs := make([]dnsv1alpha1.DNSObjectReference, 0, len(uids)) + for _, uid := range uids { + for _, zone := range zones { + if zone.UID == uid { + refs = append(refs, dnsv1alpha1.DNSObjectReference{Name: zone.Name, UID: uid}) + break + } + } + } + return refs +} + +func zoneUIDsEqual(refs []dnsv1alpha1.DNSObjectReference, uids []types.UID) bool { + if len(refs) != len(uids) { + return false + } + for i := range refs { + if refs[i].UID != uids[i] { + return false + } + } + return true +} + +func resolverContextReady(contextObject *dnsv1alpha1.DNSResolverContext) bool { + if contextObject == nil { + return false + } + ready := apimeta.FindStatusCondition(contextObject.Status.Conditions, "Ready") + return ready != nil && ready.Status == metav1.ConditionTrue && ready.ObservedGeneration == contextObject.Generation +} diff --git a/internal/internaldns/controlplane/context_reconciler_test.go b/internal/internaldns/controlplane/context_reconciler_test.go new file mode 100644 index 00000000..e86b0c01 --- /dev/null +++ b/internal/internaldns/controlplane/context_reconciler_test.go @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controlplane + +import ( + "context" + "fmt" + "testing" + "time" + + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" + "go.miloapis.com/dns-operator/internal/internaldns/model" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +type testAllocator struct{} + +// accessListSnapshotClient models a source List taken before another +// controller commits a renewed platform binding. Gets remain authoritative. +type accessListSnapshotClient struct { + client.Client + accesses []dnsv1alpha1.DNSResolverAccessBinding + getErrorFor string +} + +type bindingPatchRaceClient struct { + client.Client + beforeWithdrawal func(context.Context) error +} + +func (c *bindingPatchRaceClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if binding, ok := obj.(*dnsv1alpha1.DNSResolverBinding); ok && binding.Spec.Tombstone && c.beforeWithdrawal != nil { + before := c.beforeWithdrawal + c.beforeWithdrawal = nil + if err := before(ctx); err != nil { + return err + } + } + return c.Client.Patch(ctx, obj, patch, opts...) +} + +func (c *accessListSnapshotClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + _, access := obj.(*dnsv1alpha1.DNSResolverAccessBinding) + _, resolverContext := obj.(*dnsv1alpha1.DNSResolverContext) + if (access && c.getErrorFor == "access") || (resolverContext && c.getErrorFor == "context") { + return fmt.Errorf("injected %s read uncertainty", c.getErrorFor) + } + return c.Client.Get(ctx, key, obj, opts...) +} + +func (c *accessListSnapshotClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + if accesses, ok := list.(*dnsv1alpha1.DNSResolverAccessBindingList); ok { + accesses.Items = append([]dnsv1alpha1.DNSResolverAccessBinding(nil), c.accesses...) + return nil + } + return c.Client.List(ctx, list, opts...) +} + +func (testAllocator) Allocate(context.Context, types.UID, types.UID) (string, string, error) { + return "fd53::10", "fd54::10", nil +} + +func (testAllocator) ClaimDestination(context.Context, types.UID, types.UID, string, string, int32) error { + return nil +} + +func TestContextAccessProjectsExactAuthorizationAndExpiresClosed(t *testing.T) { + ctx := context.Background() + now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) + s := runtime.NewScheme() + if err := dnsv1alpha1.AddToScheme(s); err != nil { + t.Fatal(err) + } + contextObject := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "ctx", Namespace: "project", UID: "context-uid", Generation: 1}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "consumer-uid"}, Status: dnsv1alpha1.DNSResolverContextStatus{AccessWriterEpoch: 4}} + setCondition(&contextObject.Status.Conditions, "Ready", metav1.ConditionTrue, "Ready", "", 1, now) + zone := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "zone", Namespace: "project", UID: "zone-uid", Generation: 1}, Spec: dnsv1alpha1.DNSZoneSpec{Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}} + association := &dnsv1alpha1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: "assoc", Namespace: "project", UID: "assoc-uid", Generation: 1}, Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: "zone", UID: "zone-uid"}, ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: "ctx", UID: "context-uid"}}, Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: "zone", UID: "zone-uid", Generation: 1}, ResolvedResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: "ctx", UID: "context-uid"}}} + setCondition(&association.Status.Conditions, "Accepted", metav1.ConditionTrue, "Accepted", "", 1, now) + until := metav1.NewTime(now.Add(2 * time.Minute)) + access := &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "access", Namespace: "project", UID: "access-uid", Generation: 1}, Spec: dnsv1alpha1.DNSResolverAccessBindingSpec{ContextRef: dnsv1alpha1.DNSObjectReference{Name: "ctx", UID: "context-uid"}, Region: "central", QueryIdentity: dnsv1alpha1.DNSResolverQueryIdentity{Type: dnsv1alpha1.DNSResolverQueryIdentityDestinationAddress, Value: "fd70:100::10"}, Port: 53, Transports: []string{"UDP", "TCP"}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 4, Sequence: 12, ValidUntil: until}}} + c := fake.NewClientBuilder().WithScheme(s).WithObjects(contextObject, zone, association, access).WithStatusSubresource(&dnsv1alpha1.DNSResolverContext{}, &dnsv1alpha1.DNSResolverAccessBinding{}, &dnsv1alpha1.DNSResolverAccessBinding{}, &dnsv1alpha1.DNSResolverBinding{}).Build() + r := &Reconciler{Client: c, Scheme: s, Options: ReconcilerOptions{ProjectUID: "project-uid", PlatformNamespace: "platform", Region: "central", Shard: "shared", MaxAccessLease: 5 * time.Minute, AddressAllocator: testAllocator{}}} + if err := r.reconcileContextBindings(ctx, "project", []dnsv1alpha1.DNSZoneAssociation{*association}, map[string]*dnsv1alpha1.DNSZone{"zone": zone}, now); err != nil { + t.Fatal(err) + } + var bindings dnsv1alpha1.DNSResolverBindingList + if err := c.List(ctx, &bindings, client.InNamespace("platform")); err != nil { + t.Fatal(err) + } + if len(bindings.Items) != 1 { + t.Fatalf("got %d bindings", len(bindings.Items)) + } + b := bindings.Items[0] + if b.Spec.Configuration.Listeners.Node.Address != "fd70:100::10" || b.Spec.Source.ResolverContextRef.UID != "context-uid" || b.Spec.Authorization.WriterEpoch != 4 || b.Spec.Authorization.Sequence != 12 || !b.Spec.Authorization.ValidUntil.Equal(&until) { + t.Fatalf("access projection drifted: %#v", b.Spec) + } + if b.Spec.Source.ProjectUID != "project-uid" || b.Spec.Source.ResolverContextRef.Name != "ctx" || b.Spec.Source.AccessBindingRef.Name != "access" || b.Spec.Source.AccessBindingRef.UID != "access-uid" || b.Spec.Configuration.ZoneRefs[0].Name != "zone" || b.Spec.Configuration.ZoneRefs[0].UID != "zone-uid" || b.Spec.Placement.Region != "central" { + t.Fatalf("source identity or zone references drifted: %#v", b.Spec) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(access), access); err != nil { + t.Fatal(err) + } + if access.Status.ObservedSequence != 12 { + t.Fatalf("sequence fence not persisted: %#v", access.Status) + } + extraZone := zone.DeepCopy() + extraZone.Name = "extra" + extraZone.UID = "extra-uid" + extraAssociation := association.DeepCopy() + extraAssociation.Name = "extra-assoc" + extraAssociation.Spec.DNSZoneRef = dnsv1alpha1.DNSObjectReference{Name: extraZone.Name, UID: extraZone.UID} + extraAssociation.Status.ResolvedDNSZoneRef = extraAssociation.Spec.DNSZoneRef + if err := r.reconcileContextBindings(ctx, "project", []dnsv1alpha1.DNSZoneAssociation{*association, *extraAssociation}, map[string]*dnsv1alpha1.DNSZone{"zone": zone, "extra": extraZone}, now); err != nil { + t.Fatal(err) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(&b), &b); err != nil { + t.Fatal(err) + } + if b.Spec.Configuration.Generation != 2 || b.Spec.Configuration.Revision != 2 { + t.Fatalf("zone membership change reused configuration fences: %#v", b.Spec.Configuration) + } + + if err := r.reconcileContextBindings(ctx, "project", []dnsv1alpha1.DNSZoneAssociation{*association}, map[string]*dnsv1alpha1.DNSZone{"zone": zone}, now.Add(3*time.Minute)); err != nil { + t.Fatal(err) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(&b), &b); err != nil { + t.Fatal(err) + } + if !b.Spec.Tombstone || b.Spec.Authorization.Sequence != 12 || !b.Spec.Authorization.ValidUntil.Equal(&until) || b.Spec.Configuration.Revision != 3 { + t.Fatalf("withdrawal changed integration authority or failed to advance configuration: %#v", b.Spec) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(access), access); err != nil { + t.Fatal(err) + } + ready := apimeta.FindStatusCondition(access.Status.Conditions, "Ready") + if ready == nil || ready.Status != metav1.ConditionFalse { + t.Fatalf("invalid access retained Ready: %#v", access.Status.Conditions) + } + staleExpiredAccess := *access.DeepCopy() + access.Spec.Authorization.ValidUntil = metav1.NewTime(now.Add(4 * time.Minute)) + if err := c.Update(ctx, access); err != nil { + t.Fatal(err) + } + if err := r.reconcileContextBindings(ctx, "project", []dnsv1alpha1.DNSZoneAssociation{*association}, map[string]*dnsv1alpha1.DNSZone{"zone": zone}, now.Add(3*time.Minute)); err != nil { + t.Fatal(err) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(&b), &b); err != nil { + t.Fatal(err) + } + if !b.Spec.Tombstone { + t.Fatal("same-fence source deadline replay reopened access") + } + if err := c.Get(ctx, client.ObjectKeyFromObject(access), access); err != nil { + t.Fatal(err) + } + access.Spec.Authorization.Sequence++ + if err := c.Update(ctx, access); err != nil { + t.Fatal(err) + } + if err := r.reconcileContextBindings(ctx, "project", []dnsv1alpha1.DNSZoneAssociation{*association}, map[string]*dnsv1alpha1.DNSZone{"zone": zone}, now.Add(3*time.Minute)); err != nil { + t.Fatal(err) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(&b), &b); err != nil { + t.Fatal(err) + } + if b.Spec.Tombstone || b.Spec.Authorization.Sequence != 13 || !b.Spec.Authorization.ValidUntil.Equal(&access.Spec.Authorization.ValidUntil) || b.Spec.Configuration.Revision != 4 { + t.Fatalf("ordinary next source sequence failed to renew access: %#v", b.Spec) + } + for _, snapshot := range [][]dnsv1alpha1.DNSResolverAccessBinding{{staleExpiredAccess}, nil} { + staleController := &Reconciler{Client: &accessListSnapshotClient{Client: c, accesses: snapshot}, Scheme: s, Options: r.Options} + if err := staleController.reconcileContextBindings(ctx, "project", nil, nil, now.Add(3*time.Minute)); err != nil { + t.Fatal(err) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(&b), &b); err != nil { + t.Fatal(err) + } + if b.Spec.Tombstone || b.Spec.Configuration.Revision != 4 || b.Spec.Authorization.Sequence != 13 { + t.Fatalf("stale source snapshot withdrew renewed binding: %#v", b.Spec) + } + } + for _, kind := range []string{"access", "context"} { + uncertainController := &Reconciler{Client: &accessListSnapshotClient{Client: c, getErrorFor: kind}, Scheme: s, Options: r.Options} + if err := uncertainController.reconcileContextBindings(ctx, "project", nil, nil, now.Add(3*time.Minute)); err == nil { + t.Fatalf("%s read uncertainty was not surfaced", kind) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(&b), &b); err != nil { + t.Fatal(err) + } + if b.Spec.Tombstone || b.Spec.Configuration.Revision != 4 { + t.Fatalf("%s read uncertainty revoked authorization", kind) + } + } + if err := c.Get(ctx, client.ObjectKeyFromObject(contextObject), contextObject); err != nil { + t.Fatal(err) + } + setCondition(&contextObject.Status.Conditions, "Ready", metav1.ConditionFalse, "Reconciling", "", contextObject.Generation, now.Add(3*time.Minute)) + if err := c.Status().Update(ctx, contextObject); err != nil { + t.Fatal(err) + } + uncertainController := &Reconciler{Client: &accessListSnapshotClient{Client: c}, Scheme: s, Options: r.Options} + if err := uncertainController.reconcileContextBindings(ctx, "project", nil, nil, now.Add(3*time.Minute)); err != nil { + t.Fatal(err) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(&b), &b); err != nil { + t.Fatal(err) + } + if b.Spec.Tombstone || b.Spec.Configuration.Revision != 4 { + t.Fatal("transient context readiness revoked a live authorization") + } +} + +func TestBindingAckCurrentRequiresVerifiedExactAuthorization(t *testing.T) { + now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) + deadline := metav1.NewTime(now.Add(time.Minute)) + ackDeadline := metav1.NewTime(now.Add(30 * time.Second)) + b := &dnsv1alpha1.DNSResolverBinding{Spec: dnsv1alpha1.DNSResolverBindingSpec{Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Revision: 7}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 3, Sequence: 11, ValidUntil: deadline}}, Status: dnsv1alpha1.DNSResolverBindingStatus{MemberAcknowledgements: []dnsv1alpha1.DNSApplyAcknowledgement{{Phase: string(model.AckVerified), Revision: 7, AuthorizationIssuerEpoch: 3, AuthorizationRevision: 11, ValidUntil: &ackDeadline}}}} + if !bindingAckCurrent(b, now) { + t.Fatal("exact verified ACK was rejected") + } + b.Status.MemberAcknowledgements[0].Phase = string(model.AckRejected) + if bindingAckCurrent(b, now) { + t.Fatal("rejected ACK was accepted") + } + b.Status.MemberAcknowledgements[0].Phase = string(model.AckVerified) + b.Status.MemberAcknowledgements[0].AuthorizationRevision = 10 + if bindingAckCurrent(b, now) { + t.Fatal("stale authorization ACK was accepted") + } + b.Status.MemberAcknowledgements[0].AuthorizationRevision = 11 + overlong := metav1.NewTime(now.Add(2 * time.Minute)) + b.Status.MemberAcknowledgements[0].ValidUntil = &overlong + if bindingAckCurrent(b, now) { + t.Fatal("ACK beyond access deadline was accepted") + } +} + +func TestContextNamingPolicyResolvesMultipleZonesWithoutNetworking(t *testing.T) { + ctx := context.Background() + now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) + s := runtime.NewScheme() + if err := dnsv1alpha1.AddToScheme(s); err != nil { + t.Fatal(err) + } + resolverContext := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "ctx", Namespace: "project", UID: "ctx-uid", Generation: 2}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "consumer-uid"}} + setCondition(&resolverContext.Status.Conditions, "Ready", metav1.ConditionTrue, "Ready", "", 2, now) + z1 := dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "managed", Namespace: "project", UID: "zone-1", Generation: 1}, Spec: dnsv1alpha1.DNSZoneSpec{Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}} + z2 := dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "services", Namespace: "project", UID: "zone-2", Generation: 3}, Spec: dnsv1alpha1.DNSZoneSpec{Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}} + associations := []dnsv1alpha1.DNSZoneAssociation{{Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSObjectReference{UID: z1.UID}, ResolvedResolverContextRef: dnsv1alpha1.DNSObjectReference{UID: resolverContext.UID}}}, {Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSObjectReference{UID: z2.UID}, ResolvedResolverContextRef: dnsv1alpha1.DNSObjectReference{UID: resolverContext.UID}}}} + for i := range associations { + setCondition(&associations[i].Status.Conditions, "Accepted", metav1.ConditionTrue, "Accepted", "", 1, now) + } + policy := &dnsv1alpha1.DNSNamingPolicy{ObjectMeta: metav1.ObjectMeta{Name: "names", Namespace: "project", UID: "policy-uid", Generation: 1}, Spec: dnsv1alpha1.DNSNamingPolicySpec{ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: resolverContext.Name, UID: resolverContext.UID}, AdditionalNames: []dnsv1alpha1.DNSAdditionalNameRule{{RegistrationClass: dnsv1alpha1.DNSRegistrationClassInstanceIdentity, DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: z1.Name, UID: z1.UID, Generation: z1.Generation}, NamePrefix: "instances"}, {RegistrationClass: dnsv1alpha1.DNSRegistrationClassServiceDiscovery, DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: z2.Name, UID: z2.UID, Generation: z2.Generation}, NamePrefix: "services"}}}} + c := fake.NewClientBuilder().WithScheme(s).WithObjects(resolverContext, policy).WithStatusSubresource(&dnsv1alpha1.DNSResolverContext{}, &dnsv1alpha1.DNSResolverAccessBinding{}, &dnsv1alpha1.DNSNamingPolicy{}).Build() + r := &Reconciler{Client: c, Options: ReconcilerOptions{}} + if err := r.reconcileNamingPolicies(ctx, []dnsv1alpha1.DNSNamingPolicy{*policy}, associations, map[string]*dnsv1alpha1.DNSZone{z1.Name: &z1, z2.Name: &z2}, now); err != nil { + t.Fatal(err) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(policy), policy); err != nil { + t.Fatal(err) + } + accepted := apimeta.FindStatusCondition(policy.Status.Conditions, "Accepted") + if accepted == nil || accepted.Status != metav1.ConditionTrue || len(policy.Status.ResolvedAdditionalNames) != 2 { + t.Fatalf("context naming policy not resolved: %#v", policy.Status) + } + if policy.Status.ResolvedResolverContextRef.UID != resolverContext.UID || policy.Status.ResolvedVPCRef.UID != "consumer-uid" { + t.Fatalf("identity projection incorrect: %#v", policy.Status) + } +} + +func TestStableOneShardPerRegionAssignment(t *testing.T) { + r := &Reconciler{Options: ReconcilerOptions{ProjectUID: "project", ServingRegions: []ServingRegion{{Region: "central", Shard: "a"}, {Region: "central", Shard: "b"}, {Region: "west", Shard: "w"}}}} + first := r.selectServingTargets("vpc-1", nil) + if len(first) != 2 { + t.Fatalf("got %d targets, want one per region", len(first)) + } + central := first[0] + if central.Region != "central" { + t.Fatalf("targets not sorted: %#v", first) + } + // Find another immutable VPC identity that rendezvous assigns to the other + // shard, proving the pool is shared rather than duplicated per VPC. + different := false + for i := 2; i < 200; i++ { + candidate := r.selectServingTargets(types.UID(fmt.Sprintf("vpc-%d", i)), nil) + if candidate[0].Shard != central.Shard { + different = true + break + } + } + if !different { + t.Fatal("rendezvous assignment never used the other shard") + } + existing := []dnsv1alpha1.DNSResolverBinding{{Spec: dnsv1alpha1.DNSResolverBindingSpec{Source: dnsv1alpha1.DNSResolverBindingSource{ProjectUID: "project", ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: string("vpc-1"), UID: "vpc-1"}}, Placement: dnsv1alpha1.DNSResolverBindingPlacement{Region: "central", Shard: central.Shard}}}} + r.Options.ServingRegions = append(r.Options.ServingRegions, ServingRegion{Region: "central", Shard: "c"}) + after := r.selectServingTargets("vpc-1", existing) + if after[0].Shard != central.Shard { + t.Fatalf("adding a shard moved live VPC: before=%s after=%s", central.Shard, after[0].Shard) + } +} diff --git a/internal/internaldns/controlplane/naming.go b/internal/internaldns/controlplane/naming.go new file mode 100644 index 00000000..f5499ef0 --- /dev/null +++ b/internal/internaldns/controlplane/naming.go @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controlplane + +import ( + "context" + "fmt" + "sort" + "strings" + + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type ResolvedName struct { + DNSZoneRef dnsv1alpha1.DNSObjectReference + Name string + Canonical bool +} + +// ResolveNaming returns the stable platform namespace first and authorized +// additive policy names after it. It never selects an arbitrary associated +// customer zone. +func ResolveNaming(ctx context.Context, c client.Client, namespace string, projectUID, vpcUID types.UID, class dnsv1alpha1.DNSRegistrationClass, allocatedName string) ([]ResolvedName, error) { + label := dnsLabel(allocatedName) + if label == "" { + return nil, fmt.Errorf("allocated name %q has no DNS-safe label", allocatedName) + } + var managed dnsv1alpha1.DNSManagedNamespaceList + if err := c.List(ctx, &managed, client.InNamespace(namespace)); err != nil { + return nil, err + } + var out []ResolvedName + for i := range managed.Items { + m := &managed.Items[i] + accepted := apimeta.FindStatusCondition(m.Status.Conditions, dnsValueAccepted) + if m.Spec.ProjectUID == projectUID && m.Spec.VPCRef.UID == vpcUID && accepted != nil && accepted.Status == metav1.ConditionTrue { + prefix := classPrefix(class) + out = append(out, ResolvedName{DNSZoneRef: m.Status.DNSZoneRef, Name: label + "." + prefix, Canonical: true}) + break + } + } + if len(out) == 0 { + return nil, fmt.Errorf("managed DNS namespace for VPC %s is not ready", vpcUID) + } + var policies dnsv1alpha1.DNSNamingPolicyList + if err := c.List(ctx, &policies, client.InNamespace(namespace)); err != nil { + return nil, err + } + sort.Slice(policies.Items, func(i, j int) bool { + if policies.Items[i].Spec.Priority == policies.Items[j].Spec.Priority { + return policies.Items[i].Name < policies.Items[j].Name + } + return policies.Items[i].Spec.Priority > policies.Items[j].Spec.Priority + }) + for i := range policies.Items { + p := &policies.Items[i] + accepted := apimeta.FindStatusCondition(p.Status.Conditions, dnsValueAccepted) + if p.Status.ResolvedVPCRef.UID != vpcUID || accepted == nil || accepted.Status != metav1.ConditionTrue { + continue + } + for _, rule := range p.Status.ResolvedAdditionalNames { + if rule.RegistrationClass == class { + out = append(out, ResolvedName{DNSZoneRef: rule.DNSZoneRef, Name: label + "." + canonicalName(rule.NamePrefix)}) + } + } + } + return out, nil +} + +func classPrefix(class dnsv1alpha1.DNSRegistrationClass) string { + switch class { + case dnsv1alpha1.DNSRegistrationClassInstanceIdentity: + return "instances" + default: + return "services" + } +} +func dnsLabel(s string) string { + s = strings.ToLower(strings.TrimSpace(s)) + var b strings.Builder + dash := false + for _, r := range s { + if r >= 'a' && r <= 'z' || r >= '0' && r <= '9' { + b.WriteRune(r) + dash = false + } else if !dash && b.Len() > 0 { + b.WriteByte('-') + dash = true + } + } + return strings.Trim(b.String(), "-") +} diff --git a/internal/internaldns/controlplane/naming_test.go b/internal/internaldns/controlplane/naming_test.go new file mode 100644 index 00000000..53062cb3 --- /dev/null +++ b/internal/internaldns/controlplane/naming_test.go @@ -0,0 +1,28 @@ +package controlplane + +import ( + "context" + "testing" + + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestResolveNamingReturnsCanonicalThenAuthorizedAliases(t *testing.T) { + s := runtime.NewScheme() + if err := dnsv1alpha1.AddToScheme(s); err != nil { + t.Fatal(err) + } + managed := &dnsv1alpha1.DNSManagedNamespace{ObjectMeta: metav1.ObjectMeta{Name: "managed", Namespace: "project"}, Spec: dnsv1alpha1.DNSManagedNamespaceSpec{ProjectUID: "project-a", VPCRef: dnsv1alpha1.DNSObjectReference{Name: "prod", UID: "vpc-a"}}, Status: dnsv1alpha1.DNSManagedNamespaceStatus{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: "canonical", UID: "zone-c"}, Conditions: []metav1.Condition{{Type: "Accepted", Status: metav1.ConditionTrue}}}} + policy := &dnsv1alpha1.DNSNamingPolicy{ObjectMeta: metav1.ObjectMeta{Name: "custom", Namespace: "project"}, Spec: dnsv1alpha1.DNSNamingPolicySpec{Priority: 10}, Status: dnsv1alpha1.DNSNamingPolicyStatus{ResolvedVPCRef: dnsv1alpha1.DNSObjectReference{Name: "prod", UID: "vpc-a"}, ResolvedAdditionalNames: []dnsv1alpha1.DNSResolvedAdditionalNameRule{{RegistrationClass: dnsv1alpha1.DNSRegistrationClassInstanceIdentity, DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: "custom", UID: "zone-x"}, NamePrefix: "instances"}}, Conditions: []metav1.Condition{{Type: "Accepted", Status: metav1.ConditionTrue}}}} + c := fake.NewClientBuilder().WithScheme(s).WithObjects(managed, policy).Build() + names, err := ResolveNaming(context.Background(), c, "project", "project-a", "vpc-a", dnsv1alpha1.DNSRegistrationClassInstanceIdentity, "Web 01") + if err != nil { + t.Fatal(err) + } + if len(names) != 2 || names[0].Name != "web-01.instances" || !names[0].Canonical || names[1].Name != "web-01.instances" || names[1].Canonical { + t.Fatalf("unexpected names: %#v", names) + } +} diff --git a/internal/internaldns/controlplane/reconciler.go b/internal/internaldns/controlplane/reconciler.go new file mode 100644 index 00000000..57c90cd3 --- /dev/null +++ b/internal/internaldns/controlplane/reconciler.go @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controlplane + +import ( + "context" + "errors" + "time" + + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" + "go.miloapis.com/dns-operator/internal/internaldns/model" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +type ReconcilerOptions struct { + ProjectUID types.UID + SourceClusterUID string + ProjectNamespace string + PlatformNamespace string + Region string + Shard string + Identity string + ChunkSize int + LeaseDuration time.Duration + Now func() time.Time + AddressAllocator AddressAllocator + ManagedDomainSuffix string + PrivateZoneClassName string + ServingRegions []ServingRegion + MaxAccessLease time.Duration +} + +type ServingRegion struct { + Region string + Shard string +} + +// AddressAllocator durably leases globally unique listener destinations. Its +// implementation must retain quarantine state after release; a pure hash is +// only a candidate and is never sufficient proof of uniqueness. +type AddressAllocator interface { + Allocate(context.Context, types.UID, types.UID) (consumer, cluster string, err error) + ClaimDestination(context.Context, types.UID, types.UID, string, string, int32) error +} + +func (o *ReconcilerOptions) defaults() { + if o.PlatformNamespace == "" { + o.PlatformNamespace = "internal-dns-system" + } + if o.Region == "" { + o.Region = "default" + } + if o.Shard == "" { + o.Shard = "private-dns-01" + } + if o.Identity == "" { + o.Identity = "internal-dns-controller" + } + if o.ChunkSize <= 0 { + o.ChunkSize = 256 * 1024 + } + if o.LeaseDuration <= 0 { + o.LeaseDuration = 30 * time.Second + } + + if o.MaxAccessLease <= 0 { + o.MaxAccessLease = 5 * time.Minute + } + if o.Now == nil { + o.Now = func() time.Time { return time.Now().UTC() } + } +} + +func (o ReconcilerOptions) servingRegions() []ServingRegion { + if len(o.ServingRegions) > 0 { + return o.ServingRegions + } + return []ServingRegion{{Region: o.Region, Shard: o.Shard}} +} + +type Options = ReconcilerOptions + +// Reconciler compiles one project's desired DNS state. ProjectUID and +// SourceClusterUID are trusted configuration supplied by the replication +// bridge; Kubernetes namespace identity is never treated as project identity. +type Reconciler struct { + client.Client + Scheme *runtime.Scheme + Options ReconcilerOptions +} + +func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + r.Options.defaults() + if r.Options.ProjectUID == "" || r.Options.SourceClusterUID == "" { + return ctrl.Result{}, errors.New("projectUID and sourceClusterUID are required") + } + ns := r.Options.ProjectNamespace + if ns == "" { + ns = req.Namespace + } + if ns == "" || ns == r.Options.PlatformNamespace { + return ctrl.Result{}, nil + } + return r.reconcileProject(ctx, ns) +} + +func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { + r.Options.defaults() + enqueue := handler.EnqueueRequestsFromMapFunc(func(_ context.Context, o client.Object) []reconcile.Request { + return []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: o.GetNamespace(), Name: "project"}}} + }) + controllerName := "internal-dns-control-plane-" + model.OpaqueToken(string(r.Options.ProjectUID))[:12] + b := ctrl.NewControllerManagedBy(mgr).Named(controllerName).For(&dnsv1alpha1.DNSResolverContext{}, builder.WithPredicates()). + Watches(&dnsv1alpha1.DNSZone{}, enqueue).Watches(&dnsv1alpha1.DNSZoneAssociation{}, enqueue). + Watches(&dnsv1alpha1.DNSNamingPolicy{}, enqueue). + Watches(&dnsv1alpha1.DNSManagedNamespace{}, enqueue) + + return b.Watches(&dnsv1alpha1.DNSResolverAccessBinding{}, enqueue).Complete(r) + +} + +func (r *Reconciler) reconcileProject(ctx context.Context, ns string) (ctrl.Result, error) { + now := r.Options.Now().UTC() + + if err := r.reconcileResolverContexts(ctx, ns, now); err != nil { + return ctrl.Result{}, err + } + + if err := r.reconcileManagedNamespaces(ctx, ns, now); err != nil { + return ctrl.Result{}, err + } + var zones dnsv1alpha1.DNSZoneList + var assocs dnsv1alpha1.DNSZoneAssociationList + var policies dnsv1alpha1.DNSNamingPolicyList + for _, item := range []client.ObjectList{&zones, &assocs, &policies} { + if err := r.List(ctx, item, client.InNamespace(ns)); err != nil { + return ctrl.Result{}, err + } + } + zoneByName := map[string]*dnsv1alpha1.DNSZone{} + for i := range zones.Items { + z := &zones.Items[i] + zoneByName[z.Name] = z + } + if err := r.reconcileAssociations(ctx, assocs.Items, zoneByName, now); err != nil { + return ctrl.Result{}, err + } + if err := r.reconcileNamingPolicies(ctx, policies.Items, assocs.Items, zoneByName, now); err != nil { + return ctrl.Result{}, err + } + if err := r.reconcileBindings(ctx, ns, assocs.Items, zoneByName, now); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: r.Options.LeaseDuration / 2}, nil +} diff --git a/internal/internaldns/controlplane/references.go b/internal/internaldns/controlplane/references.go new file mode 100644 index 00000000..bb0410d7 --- /dev/null +++ b/internal/internaldns/controlplane/references.go @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controlplane + +import ( + "strings" + + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" + "k8s.io/apimachinery/pkg/types" +) + +func refMatches(ref dnsv1alpha1.DNSObjectReference, name string, uid types.UID, generation int64) bool { + return ref.Name == name && (ref.UID != "" && ref.UID == uid) && (ref.Generation == 0 || ref.Generation == generation) +} + +func canonicalName(s string) string { + return strings.ToLower(strings.TrimSuffix(strings.TrimSpace(s), ".")) +} diff --git a/internal/internaldns/controlplane/references_test.go b/internal/internaldns/controlplane/references_test.go new file mode 100644 index 00000000..32c10734 --- /dev/null +++ b/internal/internaldns/controlplane/references_test.go @@ -0,0 +1,24 @@ +package controlplane + +import ( + "testing" + + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" +) + +func TestReferenceRequiresExactLifetime(t *testing.T) { + for _, tt := range []struct { + ref dnsv1alpha1.DNSObjectReference + matches bool + }{ + {dnsv1alpha1.DNSObjectReference{Name: "zone"}, false}, + {dnsv1alpha1.DNSObjectReference{Name: "zone", UID: "old"}, false}, + {dnsv1alpha1.DNSObjectReference{Name: "zone", UID: "current"}, true}, + {dnsv1alpha1.DNSObjectReference{Name: "zone", UID: "current", Generation: 1}, false}, + {dnsv1alpha1.DNSObjectReference{Name: "zone", UID: "current", Generation: 2}, true}, + } { + if got := refMatches(tt.ref, "zone", "current", 2); got != tt.matches { + t.Errorf("%#v matches=%v", tt.ref, got) + } + } +} diff --git a/internal/internaldns/controlplane/values.go b/internal/internaldns/controlplane/values.go new file mode 100644 index 00000000..011b19dc --- /dev/null +++ b/internal/internaldns/controlplane/values.go @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controlplane + +const dnsValueAccepted = "Accepted" diff --git a/internal/internaldns/platform/allocator.go b/internal/internaldns/platform/allocator.go new file mode 100644 index 00000000..3a169eb5 --- /dev/null +++ b/internal/internaldns/platform/allocator.go @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package platform + +import ( + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "net/netip" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const leaseLabel = "internal-dns.miloapis.com/address-lease" + +type AddressLease struct { + Owner string `json:"owner"` + ProjectUID string `json:"projectUID"` + VPCUID string `json:"vpcUID"` + ConsumerAddress string `json:"consumerAddress"` + ClusterAddress string `json:"clusterAddress"` + CreatedAt time.Time `json:"createdAt"` +} + +// Allocator reserves all generated addresses through atomic, globally named Kubernetes address claims. +// Each owner has its own lease, avoiding a fleet-wide size limit or write lock. Leases are retained after revocation: addresses are never reused by +// another VPC lifetime. Explicit garbage collection needs a quarantine policy. +type Allocator struct { + Client client.Client + Namespace string + ConsumerPrefix string + ClusterPrefix string +} + +func (a *Allocator) Allocate(ctx context.Context, projectUID, vpcUID types.UID) (string, string, error) { + lease, err := a.reserve(ctx, string(projectUID), string(vpcUID)) + return lease.ConsumerAddress, lease.ClusterAddress, err +} + +// ClaimDestination atomically reserves an integration-selected service +// destination across every project sharing this platform control plane. Claims +// are retained after revocation so a stale route cannot become valid for a new +// context lifetime. +func (a *Allocator) ClaimDestination(ctx context.Context, projectUID, accessUID types.UID, region, address string, port int32) error { + parsed, err := netip.ParseAddr(address) + if err != nil || parsed.IsUnspecified() || projectUID == "" || accessUID == "" || region == "" || port != 53 { + return fmt.Errorf("destination claim identity, region, IP, and DNS port are required") + } + address = parsed.String() + owner := string(projectUID) + "/" + string(accessUID) + claim := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "dns-destination-claim-" + model.OpaqueToken(region+"/"+address+"/"+fmt.Sprint(port)), Namespace: a.Namespace, Labels: map[string]string{leaseLabel: "true"}}, Data: map[string]string{"owner": owner, "region": region, "address": address, "port": fmt.Sprint(port)}} + if err := a.Client.Create(ctx, claim); apierrors.IsAlreadyExists(err) { + var existing corev1.ConfigMap + if err := a.Client.Get(ctx, client.ObjectKeyFromObject(claim), &existing); err != nil { + return err + } + if existing.Data["owner"] != owner || existing.Data["region"] != region || existing.Data["address"] != address || existing.Data["port"] != fmt.Sprint(port) { + return fmt.Errorf("service destination is already reserved") + } + return nil + } else { + return err + } +} + +func (a *Allocator) reserve(ctx context.Context, projectUID, vpcUID string) (AddressLease, error) { + if a.Client == nil || a.Namespace == "" || projectUID == "" || vpcUID == "" { + return AddressLease{}, fmt.Errorf("allocator requires a client, namespace, and immutable project/VPC identities") + } + owner := projectUID + "/" + vpcUID + + prefixes := []string{a.ConsumerPrefix, a.ClusterPrefix} + defaults := []string{"fd53::/64", "fd54::/64"} + parsed := make([]netip.Prefix, 2) + for i, p := range prefixes { + if p == "" { + p = defaults[i] + } + v, err := netip.ParsePrefix(p) + if err != nil || !v.Addr().Is6() || v.Bits() > 112 { + return AddressLease{}, fmt.Errorf("allocator prefix must be IPv6 with at least 16 host bits: %q", p) + } + parsed[i] = v.Masked() + } + key := client.ObjectKey{Namespace: a.Namespace, Name: "dns-address-owner-" + model.OpaqueToken(owner)} + var stored corev1.ConfigMap + if err := a.Client.Get(ctx, key, &stored); err == nil { + return decodeLease(stored, owner) + } else if !apierrors.IsNotFound(err) { + return AddressLease{}, err + } + for salt := 0; salt < 1024; salt++ { + candidate := AddressLease{Owner: owner, ProjectUID: projectUID, VPCUID: vpcUID, CreatedAt: time.Now().UTC()} + + candidate.ConsumerAddress = addressInPrefix(parsed[0], owner+"/consumer", salt) + candidate.ClusterAddress = addressInPrefix(parsed[1], owner+"/cluster", salt) + collision := false + within := map[string]bool{} + for _, addr := range []string{candidate.ConsumerAddress, candidate.ClusterAddress} { + if addr == "" { + continue + } + if within[addr] { + collision = true + break + } + within[addr] = true + claim := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "dns-address-claim-" + model.OpaqueToken(addr), Namespace: a.Namespace, Labels: map[string]string{leaseLabel: "true"}}, Data: map[string]string{"owner": owner, "address": addr}} + if err := a.Client.Create(ctx, claim); apierrors.IsAlreadyExists(err) { + var existing corev1.ConfigMap + if err := a.Client.Get(ctx, client.ObjectKeyFromObject(claim), &existing); err != nil { + return AddressLease{}, err + } + if existing.Data["owner"] != owner || existing.Data["address"] != addr { + collision = true + break + } + } else if err != nil { + return AddressLease{}, err + } + } + if collision { + continue + } + b, err := json.Marshal(candidate) + if err != nil { + return AddressLease{}, err + } + lease := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: key.Name, Namespace: key.Namespace, Labels: map[string]string{leaseLabel: "true"}}, Data: map[string]string{"lease": string(b)}} + if err := a.Client.Create(ctx, lease); apierrors.IsAlreadyExists(err) { + if err := a.Client.Get(ctx, key, &stored); err != nil { + return AddressLease{}, err + } + return decodeLease(stored, owner) + } else if err != nil { + return AddressLease{}, err + } + return candidate, nil + } + return AddressLease{}, fmt.Errorf("address collision budget exhausted") +} + +func decodeLease(stored corev1.ConfigMap, owner string) (AddressLease, error) { + var lease AddressLease + if err := json.Unmarshal([]byte(stored.Data["lease"]), &lease); err != nil { + return lease, fmt.Errorf("corrupt address lease: %w", err) + } + if lease.Owner != owner || lease.ProjectUID == "" || lease.VPCUID == "" { + return lease, fmt.Errorf("address lease identity mismatch") + } + return lease, nil +} + +func addressInPrefix(prefix netip.Prefix, owner string, salt int) string { + base := prefix.Addr().As16() + hash := sha256.Sum256([]byte(fmt.Sprintf("%s/%d", owner, salt))) + for bit := prefix.Bits(); bit < 128; bit++ { + i, mask := bit/8, byte(1< 0 { + t.Fatal(failures) + } + if len(addresses) != 80 { + t.Fatalf("got %d address claims", len(addresses)) + } + a, b, err := allocator.Allocate(ctx, "project", "vpc-1") + if err != nil { + t.Fatal(err) + } + a2, b2, err := (&Allocator{Client: cl, Namespace: "platform"}).Allocate(ctx, "project", "vpc-1") + if err != nil || a != a2 || b != b2 { + t.Fatal("allocator restart changed an existing lease") + } + replacement, _, err := allocator.Allocate(ctx, "project", "new-vpc-lifetime") + if err != nil || replacement == a { + t.Fatal("new resource lifetime reused the old address") + } +} diff --git a/internal/internaldns/projectstatus/status.go b/internal/internaldns/projectstatus/status.go new file mode 100644 index 00000000..a6da7001 --- /dev/null +++ b/internal/internaldns/projectstatus/status.go @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +// Package projectstatus defines the single project-facing DNS publication +// projection used by both the source reconciler and the acknowledgement sink. +package projectstatus + +import ( + "time" + + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func Published(conditions *[]metav1.Condition, complete bool, generation int64, now time.Time) bool { + reason, message := "PendingReplicaVerification", "publication awaits required replica verification" + if complete { + reason, message = "Published", "required serving replicas verified the publication" + } + return set(conditions, "Published", complete, reason, message, generation, now) +} + +func Programmed(conditions *[]metav1.Condition, applied bool, generation int64, now time.Time) bool { + reason, message := "PendingReplicaApplication", "publication awaits serving replica application" + if applied { + reason, message = "Programmed", "at least one serving replica applied the publication" + } + return set(conditions, "Programmed", applied, reason, message, generation, now) +} + +func Available(conditions *[]metav1.Condition, available bool, generation int64, now time.Time) bool { + reason, message := "NoEligibleEndpoints", "publication contains no eligible endpoint" + if available { + reason, message = "Available", "publication contains an eligible endpoint" + } + return set(conditions, "Available", available, reason, message, generation, now) +} + +func set(conditions *[]metav1.Condition, conditionType string, value bool, reason, message string, generation int64, now time.Time) bool { + status := metav1.ConditionFalse + if value { + status = metav1.ConditionTrue + } + return apimeta.SetStatusCondition(conditions, metav1.Condition{Type: conditionType, Status: status, Reason: reason, Message: message, ObservedGeneration: generation, LastTransitionTime: metav1.NewTime(now)}) +} diff --git a/internal/internaldns/projectstatus/status_test.go b/internal/internaldns/projectstatus/status_test.go new file mode 100644 index 00000000..2033ad8b --- /dev/null +++ b/internal/internaldns/projectstatus/status_test.go @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package projectstatus + +import ( + "reflect" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestProjectionIsQuiescentAfterConvergence(t *testing.T) { + now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) + var conditions []metav1.Condition + if !Published(&conditions, true, 7, now) || !Programmed(&conditions, true, 7, now) || !Available(&conditions, false, 7, now) { + t.Fatal("initial projection did not report changes") + } + converged := append([]metav1.Condition(nil), conditions...) + if Published(&conditions, true, 7, now.Add(time.Minute)) || Programmed(&conditions, true, 7, now.Add(time.Minute)) || Available(&conditions, false, 7, now.Add(time.Minute)) { + t.Fatal("unchanged projection reported a status mutation") + } + if !reflect.DeepEqual(conditions, converged) { + t.Fatalf("unchanged projection churned conditions:\n%#v\n%#v", converged, conditions) + } +} From 10a428b3f74a7480d6d3c2f2927e7851bbcf22e1 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 16:15:58 -0500 Subject: [PATCH 2/7] Exercise recovery with API-enforced access validation --- .../access_recovery_envtest_test.go | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/internal/internaldns/controlplane/access_recovery_envtest_test.go b/internal/internaldns/controlplane/access_recovery_envtest_test.go index 55b066c1..dff35d3b 100644 --- a/internal/internaldns/controlplane/access_recovery_envtest_test.go +++ b/internal/internaldns/controlplane/access_recovery_envtest_test.go @@ -153,16 +153,17 @@ func TestAccessRecoveryPreservesSourceFencesWithCEL(t *testing.T) { if err := cl.Update(ctx, mutated); !apierrors.IsInvalid(err) { t.Fatalf("same-fence deadline mutation accepted: %v", err) } - // Even if the source API has no admission webhook, the controller compares - // against its retained authorization fence and rejects a deadline replay. + // Source CEL rejects deadline changes without an increasing sequence. requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(access), access)) baseAccess := access.DeepCopy() renewedDeadline := metav1.NewTime(expiredAt.Add(time.Minute + 987654321*time.Nanosecond)) access.Spec.Authorization.ValidUntil = renewedDeadline - requireAccessTestSuccess(t, cl.Patch(ctx, access, client.MergeFromWithOptions(baseAccess, client.MergeFromWithOptimisticLock{}))) + if err := cl.Patch(ctx, access, client.MergeFromWithOptions(baseAccess, client.MergeFromWithOptimisticLock{})); !apierrors.IsInvalid(err) { + t.Fatalf("same-sequence source deadline mutation accepted: %v", err) + } requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(access), access)) - if !access.Spec.Authorization.ValidUntil.Equal(&renewedDeadline) { - t.Fatal("typed merge patch lost source deadline precision") + if !access.Spec.Authorization.ValidUntil.Equal(&deadline) { + t.Fatal("rejected source patch changed its retained deadline") } reconcile(expiredAt) readBinding() @@ -172,7 +173,12 @@ func TestAccessRecoveryPreservesSourceFencesWithCEL(t *testing.T) { // Ordinary next-sequence renewal is independent of private DNS counters. requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(access), access)) access.Spec.Authorization.Sequence = 3 + access.Spec.Authorization.ValidUntil = renewedDeadline requireAccessTestSuccess(t, cl.Update(ctx, access)) + requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(access), access)) + if !access.Spec.Authorization.ValidUntil.Equal(&renewedDeadline) { + t.Fatal("source renewal lost deadline precision") + } reconcile(expiredAt) readBinding() if binding.Spec.Tombstone || binding.Spec.Authorization.Sequence != 3 || binding.Spec.Configuration.Revision != 3 || !binding.Spec.Authorization.ValidUntil.Equal(&access.Spec.Authorization.ValidUntil) { @@ -231,7 +237,14 @@ func TestAccessRecoveryPreservesSourceFencesWithCEL(t *testing.T) { requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(freshAccess), freshAccess)) freshAccess.Spec.QueryIdentity.Value = "fd70:100::11" freshAccess.Spec.Authorization.Sequence = 2 - requireAccessTestSuccess(t, cl.Update(ctx, freshAccess)) + if err := cl.Update(ctx, freshAccess); !apierrors.IsInvalid(err) { + t.Fatalf("access identity changed in place: %v", err) + } + freshSpec := freshAccess.Spec + requireAccessTestSuccess(t, cl.Delete(ctx, freshAccess)) + freshAccess = &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "access", Namespace: "project"}, Spec: freshSpec} + requireAccessTestSuccess(t, cl.Create(ctx, freshAccess)) + requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(freshAccess), freshAccess)) reconcile(expiredAt) requireAccessTestSuccess(t, cl.List(ctx, &bindings, client.InNamespace("platform"))) if len(bindings.Items) != 2 { From cde282e330f8c91c2426c2f30e06417f5607e5aa Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 11:14:46 -0500 Subject: [PATCH 3/7] fix: Remove unused recovery test import --- .../internaldns/controlplane/access_recovery_envtest_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/internaldns/controlplane/access_recovery_envtest_test.go b/internal/internaldns/controlplane/access_recovery_envtest_test.go index 3e03273a..68aa271f 100644 --- a/internal/internaldns/controlplane/access_recovery_envtest_test.go +++ b/internal/internaldns/controlplane/access_recovery_envtest_test.go @@ -13,7 +13,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" From 1b483f0377684390cc632e6717f331e76e323bf1 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 21 Oct 2026 21:17:00 -0500 Subject: [PATCH 4/7] fix: protect resolver address and consumer ownership --- .../controlplane/context_reconciler.go | 8 ++++- .../controlplane/context_reconciler_test.go | 33 +++++++++++++++++++ internal/internaldns/platform/allocator.go | 33 ++++++++++++++----- .../internaldns/platform/allocator_test.go | 22 +++++++++++++ 4 files changed, 86 insertions(+), 10 deletions(-) diff --git a/internal/internaldns/controlplane/context_reconciler.go b/internal/internaldns/controlplane/context_reconciler.go index c09efda6..0ba06c59 100644 --- a/internal/internaldns/controlplane/context_reconciler.go +++ b/internal/internaldns/controlplane/context_reconciler.go @@ -111,7 +111,13 @@ func (r *Reconciler) reconcileResolverContexts(ctx context.Context, ns string, n if err := r.List(ctx, &contexts, client.InNamespace(ns)); err != nil { return err } - sort.Slice(contexts.Items, func(i, j int) bool { return contexts.Items[i].Name < contexts.Items[j].Name }) + sort.Slice(contexts.Items, func(i, j int) bool { + left, right := contexts.Items[i], contexts.Items[j] + if !left.CreationTimestamp.Equal(&right.CreationTimestamp) { + return left.CreationTimestamp.Before(&right.CreationTimestamp) + } + return left.UID < right.UID + }) consumerOwners := map[string]types.UID{} for i := range contexts.Items { c := &contexts.Items[i] diff --git a/internal/internaldns/controlplane/context_reconciler_test.go b/internal/internaldns/controlplane/context_reconciler_test.go index ceced401..1f948b7e 100644 --- a/internal/internaldns/controlplane/context_reconciler_test.go +++ b/internal/internaldns/controlplane/context_reconciler_test.go @@ -285,3 +285,36 @@ func TestStableOneShardPerRegionAssignment(t *testing.T) { t.Fatalf("adding a shard moved live VPC: before=%s after=%s", central.Shard, after[0].Shard) } } + +func TestNewContextCannotDisplaceExistingConsumerOwner(t *testing.T) { + ctx := context.Background() + now := time.Now().UTC() + s := runtime.NewScheme() + if err := dnsv1alpha1.AddToScheme(s); err != nil { + t.Fatal(err) + } + older := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "z-established", Namespace: "project", UID: "old", CreationTimestamp: metav1.NewTime(now.Add(-time.Hour)), Generation: 1}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "consumer"}} + c := fake.NewClientBuilder().WithScheme(s).WithObjects(older).WithStatusSubresource(&dnsv1alpha1.DNSResolverContext{}).Build() + r := &Reconciler{Client: c, Scheme: s, Options: ReconcilerOptions{PlatformNamespace: "platform"}} + if err := r.reconcileResolverContexts(ctx, "project", now); err != nil { + t.Fatal(err) + } + newer := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "a-new", Namespace: "project", UID: "new", CreationTimestamp: metav1.NewTime(now), Generation: 1}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "consumer"}} + if err := c.Create(ctx, newer); err != nil { + t.Fatal(err) + } + if err := r.reconcileResolverContexts(ctx, "project", now); err != nil { + t.Fatal(err) + } + for _, object := range []*dnsv1alpha1.DNSResolverContext{older, newer} { + if err := c.Get(ctx, client.ObjectKeyFromObject(object), object); err != nil { + t.Fatal(err) + } + } + if condition := apimeta.FindStatusCondition(older.Status.Conditions, "Ready"); condition == nil || condition.Status != metav1.ConditionTrue { + t.Fatal("new context displaced established consumer ownership") + } + if condition := apimeta.FindStatusCondition(newer.Status.Conditions, "Ready"); condition == nil || condition.Reason != "DuplicateConsumerID" || condition.Status != metav1.ConditionFalse { + t.Fatal("duplicate consumer was not rejected") + } +} diff --git a/internal/internaldns/platform/allocator.go b/internal/internaldns/platform/allocator.go index b9ebbaf4..ae0c3eee 100644 --- a/internal/internaldns/platform/allocator.go +++ b/internal/internaldns/platform/allocator.go @@ -48,9 +48,17 @@ func (a *Allocator) Allocate(ctx context.Context, projectUID, contextUID types.U // context lifetime. func (a *Allocator) ClaimDestination(ctx context.Context, projectUID, accessUID types.UID, region, address string, port int32) error { parsed, err := netip.ParseAddr(address) - if err != nil || parsed.IsUnspecified() || projectUID == "" || accessUID == "" || region == "" || port != 53 { + if err != nil || !parsed.IsGlobalUnicast() || projectUID == "" || accessUID == "" || region == "" || port != 53 { return fmt.Errorf("destination claim identity, region, IP, and DNS port are required") } + parsed = parsed.Unmap() + prefix, err := a.regionalPrefix() + if err != nil { + return err + } + if prefix.Contains(parsed) { + return fmt.Errorf("service destination must be outside the regional resolver address pool") + } address = parsed.String() owner := string(projectUID) + "/" + string(accessUID) claim := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "dns-destination-claim-" + model.OpaqueToken(region+"/"+address+"/"+fmt.Sprint(port)), Namespace: a.Namespace, Labels: map[string]string{leaseLabel: "true"}}, Data: map[string]string{"owner": owner, "region": region, "address": address, "port": fmt.Sprint(port)}} @@ -74,15 +82,10 @@ func (a *Allocator) reserve(ctx context.Context, projectUID, contextUID string) } owner := projectUID + "/" + contextUID - prefix := a.ClusterPrefix - if prefix == "" { - prefix = "fd54::/64" - } - parsed, err := netip.ParsePrefix(prefix) - if err != nil || !parsed.Addr().Is6() || parsed.Bits() > 112 { - return AddressLease{}, fmt.Errorf("allocator prefix must be IPv6 with at least 16 host bits: %q", prefix) + parsed, err := a.regionalPrefix() + if err != nil { + return AddressLease{}, err } - parsed = parsed.Masked() key := client.ObjectKey{Namespace: a.Namespace, Name: "dns-address-owner-" + model.OpaqueToken(owner)} var stored corev1.ConfigMap if err := a.Client.Get(ctx, key, &stored); err == nil { @@ -149,3 +152,15 @@ func addressInPrefix(prefix netip.Prefix, owner string, salt int) string { base[15] |= 1 // Never allocate the subnet's all-zero host address. return netip.AddrFrom16(base).String() } + +func (a *Allocator) regionalPrefix() (netip.Prefix, error) { + prefix := a.ClusterPrefix + if prefix == "" { + prefix = "fd54::/64" + } + parsed, err := netip.ParsePrefix(prefix) + if err != nil || !parsed.Addr().Is6() || parsed.Bits() > 112 { + return netip.Prefix{}, fmt.Errorf("allocator prefix must be IPv6 with at least 16 host bits: %q", prefix) + } + return parsed.Masked(), nil +} diff --git a/internal/internaldns/platform/allocator_test.go b/internal/internaldns/platform/allocator_test.go index bb0c5e16..f94814c9 100644 --- a/internal/internaldns/platform/allocator_test.go +++ b/internal/internaldns/platform/allocator_test.go @@ -76,3 +76,25 @@ func TestAddressClaimsAreConcurrentStableAndLifetimeScoped(t *testing.T) { t.Fatal("new resource lifetime reused the old address") } } + +func TestServiceDestinationsCannotOverlapRegionalAddressPool(t *testing.T) { + ctx := context.Background() + for _, prefix := range []string{"", "fd80::/96"} { + allocator := Allocator{Client: testClient(t), Namespace: "platform", ClusterPrefix: prefix} + address, err := allocator.Allocate(ctx, "project-a", "context-a") + if err != nil { + t.Fatal(err) + } + if err := allocator.ClaimDestination(ctx, "project-b", "context-b", "region", address, 53); err == nil { + t.Fatal("service destination reused another context's regional address") + } + for _, address := range []string{"127.0.0.1", "::1", "fe80::1", "169.254.1.1", "ff02::1", "224.0.0.1", "::"} { + if err := allocator.ClaimDestination(ctx, "project-a", "context-a", "region", address, 53); err == nil { + t.Fatalf("non-unicast service destination accepted: %s", address) + } + } + if err := allocator.ClaimDestination(ctx, "project-a", "context-a", "region", "fd70::1", 53); err != nil { + t.Fatal(err) + } + } +} From ae2a3d6fd225ba0179c8c57ab4296541c5d9651e Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 21 Oct 2026 21:19:01 -0500 Subject: [PATCH 5/7] fix: protect retained regional DNS address claims --- internal/internaldns/platform/allocator.go | 7 +++++++ internal/internaldns/platform/allocator_test.go | 13 +++++++++++++ 2 files changed, 20 insertions(+) diff --git a/internal/internaldns/platform/allocator.go b/internal/internaldns/platform/allocator.go index ae0c3eee..144ac3e7 100644 --- a/internal/internaldns/platform/allocator.go +++ b/internal/internaldns/platform/allocator.go @@ -60,6 +60,13 @@ func (a *Allocator) ClaimDestination(ctx context.Context, projectUID, accessUID return fmt.Errorf("service destination must be outside the regional resolver address pool") } address = parsed.String() + var regionalClaim corev1.ConfigMap + claimKey := client.ObjectKey{Namespace: a.Namespace, Name: "dns-address-claim-" + model.OpaqueToken(address)} + if err := a.Client.Get(ctx, claimKey, ®ionalClaim); err == nil { + return fmt.Errorf("service destination is reserved for regional resolution") + } else if !apierrors.IsNotFound(err) { + return err + } owner := string(projectUID) + "/" + string(accessUID) claim := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "dns-destination-claim-" + model.OpaqueToken(region+"/"+address+"/"+fmt.Sprint(port)), Namespace: a.Namespace, Labels: map[string]string{leaseLabel: "true"}}, Data: map[string]string{"owner": owner, "region": region, "address": address, "port": fmt.Sprint(port)}} if err := a.Client.Create(ctx, claim); apierrors.IsAlreadyExists(err) { diff --git a/internal/internaldns/platform/allocator_test.go b/internal/internaldns/platform/allocator_test.go index f94814c9..6b7d3260 100644 --- a/internal/internaldns/platform/allocator_test.go +++ b/internal/internaldns/platform/allocator_test.go @@ -98,3 +98,16 @@ func TestServiceDestinationsCannotOverlapRegionalAddressPool(t *testing.T) { } } } + +func TestServiceDestinationCannotReuseRetainedRegionalClaim(t *testing.T) { + ctx := context.Background() + allocator := Allocator{Client: testClient(t), Namespace: "platform"} + address, err := allocator.Allocate(ctx, "project-a", "context-a") + if err != nil { + t.Fatal(err) + } + allocator.ClusterPrefix = "fd80::/96" + if err := allocator.ClaimDestination(ctx, "project-b", "context-b", "region", address, 53); err == nil { + t.Fatal("service destination reused a retained regional allocation") + } +} From 25135580bd0afa8f531fee97989d5ceabd2c4181 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 22 Oct 2026 09:31:46 -0500 Subject: [PATCH 6/7] fix: Stop setting the removed DNSZoneAssociation managed flag The parent branch removed `DNSZoneAssociationSpec.Managed`. The context reconciler already marks the managed association with its controller owner reference and the managed label, so it no longer sets the flag. Co-Authored-By: Claude Opus 5.5 --- internal/internaldns/controlplane/context_reconciler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/internaldns/controlplane/context_reconciler.go b/internal/internaldns/controlplane/context_reconciler.go index 0ba06c59..795cf948 100644 --- a/internal/internaldns/controlplane/context_reconciler.go +++ b/internal/internaldns/controlplane/context_reconciler.go @@ -604,7 +604,7 @@ func (r *Reconciler) reconcileManagedZone(ctx context.Context, c *dnsv1alpha1.DN if err := r.Get(ctx, client.ObjectKeyFromObject(zone), zone); err != nil { return false, err } - association := &dnsv1alpha1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: zoneName + "-context", Namespace: c.Namespace, Labels: map[string]string{"dns.networking.miloapis.com/managed": "true"}}, Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name, UID: zone.UID}, ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: c.Name, UID: c.UID}, Managed: true}} + association := &dnsv1alpha1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: zoneName + "-context", Namespace: c.Namespace, Labels: map[string]string{"dns.networking.miloapis.com/managed": "true"}}, Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name, UID: zone.UID}, ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: c.Name, UID: c.UID}}} if r.Scheme != nil { if err := controllerutil.SetControllerReference(c, association, r.Scheme, controllerutil.WithBlockOwnerDeletion(false)); err != nil { return false, err From a489e2d054f1aa376465f868e14c3096185115db Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 22 Oct 2026 14:42:05 -0500 Subject: [PATCH 7/7] fix: report invalid managed domains and retry transient destination claims A resolver context rejected for an invalid managed domain suffix now reports `InvalidManagedDomain` with a message saying the suffix is not a valid DNS name, instead of `InvalidIdentity` and "consumerID is required". The access binding withdrawal path already treated that reason as terminal. The allocator now wraps every merit-based refusal of a service destination in `model.ErrDestinationRefused`. The access reconciler rejects the binding only for that error and returns any other error, so a transient API failure is retried instead of flipping an accepted binding to `AccessRejected` until the next requeue. `ResolveNaming` now requires the context's Ready condition to match its current generation and its managed namespace to carry a zone UID, so a context without a managed namespace, or with a stale one, yields no canonical name. Co-Authored-By: Claude Opus 5.5 --- .../controlplane/context_reconciler.go | 11 +++- .../controlplane/context_reconciler_test.go | 63 +++++++++++++++++++ internal/internaldns/controlplane/naming.go | 4 +- .../internaldns/controlplane/naming_test.go | 22 +++++++ internal/internaldns/model/model.go | 5 ++ internal/internaldns/platform/allocator.go | 8 +-- 6 files changed, 107 insertions(+), 6 deletions(-) diff --git a/internal/internaldns/controlplane/context_reconciler.go b/internal/internaldns/controlplane/context_reconciler.go index 795cf948..422f092e 100644 --- a/internal/internaldns/controlplane/context_reconciler.go +++ b/internal/internaldns/controlplane/context_reconciler.go @@ -8,6 +8,7 @@ import ( "encoding/binary" "encoding/json" "errors" + "fmt" "net/netip" "reflect" "sort" @@ -149,6 +150,8 @@ func (r *Reconciler) reconcileResolverContexts(ctx context.Context, ns string, n status, reason, msg = metav1.ConditionFalse, "InvalidIdentity", "consumerID is required and context must be live" if duplicate { reason, msg = "DuplicateConsumerID", "another live resolver context owns this consumerID" + } else if invalidDomain && c.Spec.ConsumerID != "" && c.DeletionTimestamp.IsZero() { + reason, msg = "InvalidManagedDomain", "the managed namespace domain suffix is not a valid DNS name" } } if valid && !ready { @@ -276,8 +279,14 @@ func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, as if owner := destinations[key]; owner != "" && owner != a.UID { valid = false } else if valid { - if r.Options.AddressAllocator == nil || r.Options.AddressAllocator.ClaimDestination(ctx, r.Options.ProjectUID, contextObject.UID, spec.Region, netip.MustParseAddr(spec.DestinationAddress).String(), 53) != nil { + if r.Options.AddressAllocator == nil { + valid = false + } else if err := r.Options.AddressAllocator.ClaimDestination(ctx, r.Options.ProjectUID, contextObject.UID, spec.Region, netip.MustParseAddr(spec.DestinationAddress).String(), 53); errors.Is(err, model.ErrDestinationRefused) { valid = false + } else if err != nil { + // A transient API failure is not a refusal: retry rather + // than flip an accepted binding to AccessRejected. + return fmt.Errorf("claim service destination: %w", err) } else { destinations[key] = a.UID } diff --git a/internal/internaldns/controlplane/context_reconciler_test.go b/internal/internaldns/controlplane/context_reconciler_test.go index 1f948b7e..bec59e58 100644 --- a/internal/internaldns/controlplane/context_reconciler_test.go +++ b/internal/internaldns/controlplane/context_reconciler_test.go @@ -318,3 +318,66 @@ func TestNewContextCannotDisplaceExistingConsumerOwner(t *testing.T) { t.Fatal("duplicate consumer was not rejected") } } + +func TestInvalidManagedDomainIsReported(t *testing.T) { + ctx := context.Background() + now := time.Now().UTC() + s := runtime.NewScheme() + if err := dnsv1alpha1.AddToScheme(s); err != nil { + t.Fatal(err) + } + contextObject := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "ctx", Namespace: "project", UID: "context-uid", Generation: 1}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "consumer", ManagedNamespace: dnsv1alpha1.DNSResolverContextManagedNamespace{Enabled: true}}} + c := fake.NewClientBuilder().WithScheme(s).WithObjects(contextObject).WithStatusSubresource(&dnsv1alpha1.DNSResolverContext{}).Build() + r := &Reconciler{Client: c, Scheme: s, Options: ReconcilerOptions{PlatformNamespace: "platform", ManagedDomainSuffix: "not a domain"}} + if err := r.reconcileResolverContexts(ctx, "project", now); err != nil { + t.Fatal(err) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(contextObject), contextObject); err != nil { + t.Fatal(err) + } + if condition := apimeta.FindStatusCondition(contextObject.Status.Conditions, "Ready"); condition == nil || condition.Status != metav1.ConditionFalse || condition.Reason != "InvalidManagedDomain" { + t.Fatalf("invalid managed domain was not reported: %#v", contextObject.Status.Conditions) + } +} + +type claimAllocator struct { + testAllocator + err error +} + +func (a claimAllocator) ClaimDestination(context.Context, types.UID, types.UID, string, string, int32) error { + return a.err +} + +func TestDestinationClaimAPIErrorsAreNotRefusals(t *testing.T) { + ctx := context.Background() + now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) + s := runtime.NewScheme() + if err := dnsv1alpha1.AddToScheme(s); err != nil { + t.Fatal(err) + } + contextObject := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "ctx", Namespace: "project", UID: "context-uid", Generation: 1}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "consumer-uid"}} + setCondition(&contextObject.Status.Conditions, "Ready", metav1.ConditionTrue, "Ready", "", 1, now) + access := &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "access", Namespace: "project", UID: "access-uid", Generation: 1}, Spec: dnsv1alpha1.DNSResolverAccessBindingSpec{ContextRef: dnsv1alpha1.DNSObjectReference{Name: "ctx", UID: "context-uid"}, Region: "central", DestinationAddress: "fd70:100::10"}} + c := fake.NewClientBuilder().WithScheme(s).WithObjects(contextObject, access).WithStatusSubresource(&dnsv1alpha1.DNSResolverContext{}, &dnsv1alpha1.DNSResolverAccessBinding{}, &dnsv1alpha1.DNSResolverBinding{}).Build() + r := &Reconciler{Client: c, Scheme: s, Options: ReconcilerOptions{ProjectUID: "project-uid", PlatformNamespace: "platform", Region: "central", Shard: "shared", AddressAllocator: claimAllocator{err: fmt.Errorf("etcdserver: request timed out")}}} + if err := r.reconcileContextBindings(ctx, "project", nil, map[string]*dnsv1alpha1.DNSZone{}, now); err == nil { + t.Fatal("transient destination claim failure was swallowed") + } + if err := c.Get(ctx, client.ObjectKeyFromObject(access), access); err != nil { + t.Fatal(err) + } + if ready := apimeta.FindStatusCondition(access.Status.Conditions, "Ready"); ready != nil && ready.Reason == "AccessRejected" { + t.Fatalf("transient claim failure rejected access: %#v", access.Status.Conditions) + } + r.Options.AddressAllocator = claimAllocator{err: fmt.Errorf("%w: service destination is already reserved", model.ErrDestinationRefused)} + if err := r.reconcileContextBindings(ctx, "project", nil, map[string]*dnsv1alpha1.DNSZone{}, now); err != nil { + t.Fatal(err) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(access), access); err != nil { + t.Fatal(err) + } + if ready := apimeta.FindStatusCondition(access.Status.Conditions, "Ready"); ready == nil || ready.Reason != "AccessRejected" { + t.Fatalf("refused destination was not rejected: %#v", access.Status.Conditions) + } +} diff --git a/internal/internaldns/controlplane/naming.go b/internal/internaldns/controlplane/naming.go index 9ca840ac..07299f30 100644 --- a/internal/internaldns/controlplane/naming.go +++ b/internal/internaldns/controlplane/naming.go @@ -39,7 +39,9 @@ func ResolveNaming(ctx context.Context, c client.Client, namespace string, _ typ for i := range managed.Items { m := &managed.Items[i] accepted := apimeta.FindStatusCondition(m.Status.Conditions, "Ready") - if m.UID == contextUID && m.DeletionTimestamp.IsZero() && accepted != nil && accepted.Status == metav1.ConditionTrue { + // A Ready context without a managed namespace, or whose Ready + // condition predates its spec, has no current canonical zone. + if m.UID == contextUID && m.DeletionTimestamp.IsZero() && accepted != nil && accepted.Status == metav1.ConditionTrue && accepted.ObservedGeneration == m.Generation && m.Status.ManagedNamespace.DNSZoneRef.UID != "" { out = append(out, ResolvedName{DNSZoneRef: m.Status.ManagedNamespace.DNSZoneRef, Name: label, Canonical: true}) break } diff --git a/internal/internaldns/controlplane/naming_test.go b/internal/internaldns/controlplane/naming_test.go index c5423094..8439cf8d 100644 --- a/internal/internaldns/controlplane/naming_test.go +++ b/internal/internaldns/controlplane/naming_test.go @@ -26,3 +26,25 @@ func TestResolveNamingReturnsCanonicalThenAuthorizedAliases(t *testing.T) { t.Fatalf("unexpected names: %#v", names) } } + +func TestResolveNamingRequiresCurrentManagedNamespace(t *testing.T) { + s := runtime.NewScheme() + if err := dnsv1alpha1.AddToScheme(s); err != nil { + t.Fatal(err) + } + for name, mutate := range map[string]func(*dnsv1alpha1.DNSResolverContext){ + "managed namespace disabled": func(c *dnsv1alpha1.DNSResolverContext) { + c.Status.ManagedNamespace = dnsv1alpha1.DNSResolverContextManagedNamespaceStatus{} + }, + "ready for an older generation": func(c *dnsv1alpha1.DNSResolverContext) { + c.Status.Conditions[0].ObservedGeneration = c.Generation - 1 + }, + } { + managed := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "prod", Namespace: "project", UID: "vpc-a", Generation: 2}, Status: dnsv1alpha1.DNSResolverContextStatus{ManagedNamespace: dnsv1alpha1.DNSResolverContextManagedNamespaceStatus{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: "canonical", UID: "zone-c"}}, Conditions: []metav1.Condition{{Type: "Ready", Status: metav1.ConditionTrue, ObservedGeneration: 2}}}} + mutate(managed) + c := fake.NewClientBuilder().WithScheme(s).WithObjects(managed).Build() + if names, err := ResolveNaming(context.Background(), c, "project", "project-a", "vpc-a", nil, "web"); err == nil { + t.Fatalf("%s: resolved %#v", name, names) + } + } +} diff --git a/internal/internaldns/model/model.go b/internal/internaldns/model/model.go index cc29882b..3b84faa4 100644 --- a/internal/internaldns/model/model.go +++ b/internal/internaldns/model/model.go @@ -21,6 +21,11 @@ const APIVersion = "internal.dns.networking.miloapis.com/v1alpha6" var ErrUnsupportedEnvelope = errors.New("unsupported DNS envelope") +// ErrDestinationRefused marks a service destination claim that was refused on +// its merits (invalid, reserved or owned by another binding), as opposed to a +// transient failure to reach the API. +var ErrDestinationRefused = errors.New("service destination refused") + type Kind string const ( diff --git a/internal/internaldns/platform/allocator.go b/internal/internaldns/platform/allocator.go index 144ac3e7..7a725520 100644 --- a/internal/internaldns/platform/allocator.go +++ b/internal/internaldns/platform/allocator.go @@ -49,7 +49,7 @@ func (a *Allocator) Allocate(ctx context.Context, projectUID, contextUID types.U func (a *Allocator) ClaimDestination(ctx context.Context, projectUID, accessUID types.UID, region, address string, port int32) error { parsed, err := netip.ParseAddr(address) if err != nil || !parsed.IsGlobalUnicast() || projectUID == "" || accessUID == "" || region == "" || port != 53 { - return fmt.Errorf("destination claim identity, region, IP, and DNS port are required") + return fmt.Errorf("%w: destination claim identity, region, IP, and DNS port are required", model.ErrDestinationRefused) } parsed = parsed.Unmap() prefix, err := a.regionalPrefix() @@ -57,13 +57,13 @@ func (a *Allocator) ClaimDestination(ctx context.Context, projectUID, accessUID return err } if prefix.Contains(parsed) { - return fmt.Errorf("service destination must be outside the regional resolver address pool") + return fmt.Errorf("%w: service destination must be outside the regional resolver address pool", model.ErrDestinationRefused) } address = parsed.String() var regionalClaim corev1.ConfigMap claimKey := client.ObjectKey{Namespace: a.Namespace, Name: "dns-address-claim-" + model.OpaqueToken(address)} if err := a.Client.Get(ctx, claimKey, ®ionalClaim); err == nil { - return fmt.Errorf("service destination is reserved for regional resolution") + return fmt.Errorf("%w: service destination is reserved for regional resolution", model.ErrDestinationRefused) } else if !apierrors.IsNotFound(err) { return err } @@ -75,7 +75,7 @@ func (a *Allocator) ClaimDestination(ctx context.Context, projectUID, accessUID return err } if existing.Data["owner"] != owner || existing.Data["region"] != region || existing.Data["address"] != address || existing.Data["port"] != fmt.Sprint(port) { - return fmt.Errorf("service destination is already reserved") + return fmt.Errorf("%w: service destination is already reserved", model.ErrDestinationRefused) } return nil } else {