diff --git a/internal/internaldns/controlplane/access_recovery_envtest_test.go b/internal/internaldns/controlplane/access_recovery_envtest_test.go new file mode 100644 index 00000000..ba61169e --- /dev/null +++ b/internal/internaldns/controlplane/access_recovery_envtest_test.go @@ -0,0 +1,134 @@ +package controlplane + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" + "go.miloapis.com/dns-operator/internal/internaldns/platform" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +// TestAccessRecoveryPreservesSourceFencesWithCEL exercises the actual CRD +// transition rules together with controller withdrawal, renewal, and UID pins. +func TestAccessRecoveryPreservesSourceFencesWithCEL(t *testing.T) { + assets := os.Getenv("KUBEBUILDER_ASSETS") + if assets == "" { + t.Skip("set KUBEBUILDER_ASSETS to run access recovery against the API server") + } + environment := &envtest.Environment{BinaryAssetsDirectory: assets, CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, ErrorIfCRDPathMissing: true} + cfg, err := environment.Start() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := environment.Stop(); err != nil { + t.Error(err) + } + }) + scheme := runtime.NewScheme() + requireAccessTestSuccess(t, dnsv1alpha1.AddToScheme(scheme)) + requireAccessTestSuccess(t, corev1.AddToScheme(scheme)) + cl, err := client.New(cfg, client.Options{Scheme: scheme}) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + for _, ns := range []string{"project", "platform"} { + requireAccessTestSuccess(t, cl.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}})) + } + now := time.Now().UTC().Truncate(time.Second) + resolverContext := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "context", Namespace: "project"}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "opaque-consumer"}} + readyContext := func(c *dnsv1alpha1.DNSResolverContext) { + t.Helper() + requireAccessTestSuccess(t, cl.Create(ctx, c)) + c.Generation = 1 + apimeta.SetStatusCondition(&c.Status.Conditions, metav1.Condition{Type: "Ready", Status: metav1.ConditionTrue, ObservedGeneration: c.Generation, Reason: "Accepted", LastTransitionTime: metav1.NewTime(now)}) + requireAccessTestSuccess(t, cl.Status().Update(ctx, c)) + } + readyContext(resolverContext) + access := &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "access", Namespace: "project"}, Spec: dnsv1alpha1.DNSResolverAccessBindingSpec{ContextRef: dnsv1alpha1.DNSObjectReference{Name: resolverContext.Name, UID: resolverContext.UID}, Region: "central", DestinationAddress: "fd70:100::10"}} + requireAccessTestSuccess(t, cl.Create(ctx, access)) + r := &Reconciler{Client: cl, Scheme: scheme, Options: ReconcilerOptions{ProjectUID: "project-uid", PlatformNamespace: "platform", Region: "central", Shard: "shared", AddressAllocator: &platform.Allocator{Client: cl, Namespace: "platform"}}} + reconcile := func(at time.Time) { + t.Helper() + requireAccessTestSuccess(t, r.reconcileContextBindings(ctx, "project", nil, map[string]*dnsv1alpha1.DNSZone{}, at)) + } + reconcile(now) + var bindings dnsv1alpha1.DNSResolverBindingList + requireAccessTestSuccess(t, cl.List(ctx, &bindings, client.InNamespace("platform"))) + if len(bindings.Items) != 1 { + t.Fatalf("got %d bindings", len(bindings.Items)) + } + binding := bindings.Items[0] + key := client.ObjectKeyFromObject(&binding) + reconcile(now.Add(365 * 24 * time.Hour)) + requireAccessTestSuccess(t, cl.Get(ctx, key, &binding)) + if binding.Spec.Tombstone || binding.Spec.Version.Sequence != 1 || binding.Spec.Configuration.Revision != 1 { + t.Fatalf("elapsed time changed access: %#v", binding.Spec) + } + requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(access), access)) + // Schema validation preserves listener identity. + for _, mutate := range []func(*dnsv1alpha1.DNSResolverAccessBinding){func(a *dnsv1alpha1.DNSResolverAccessBinding) { a.Spec.DestinationAddress = "fd70:100::20" }, func(a *dnsv1alpha1.DNSResolverAccessBinding) { a.Spec.ContextRef.UID = "another-context" }} { + candidate := access.DeepCopy() + mutate(candidate) + if err := cl.Update(ctx, candidate); !apierrors.IsInvalid(err) { + t.Fatalf("identity mutation accepted: %v", err) + } + } + // An incomplete cached List cannot revoke a live access binding. + stale := &Reconciler{Client: &accessListSnapshotClient{Client: cl, accesses: nil}, Scheme: scheme, Options: r.Options} + requireAccessTestSuccess(t, stale.reconcileContextBindings(ctx, "project", nil, nil, now.Add(365*24*time.Hour))) + requireAccessTestSuccess(t, cl.Get(ctx, key, &binding)) + if binding.Spec.Tombstone { + t.Fatal("incomplete List withdrew live access") + } + requireAccessTestSuccess(t, cl.Delete(ctx, resolverContext)) + reconcile(now.Add(365 * 24 * time.Hour)) + requireAccessTestSuccess(t, cl.Get(ctx, key, &binding)) + if !binding.Spec.Tombstone || binding.Spec.Version.Sequence != 2 { + t.Fatal("explicit context deletion did not withdraw access") + } + replacement := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: resolverContext.Name, Namespace: "project"}, Spec: resolverContext.Spec} + readyContext(replacement) + reconcile(now.Add(365 * 24 * time.Hour)) + requireAccessTestSuccess(t, cl.Get(ctx, key, &binding)) + if !binding.Spec.Tombstone { + t.Fatal("replacement context inherited retired access") + } + requireAccessTestSuccess(t, cl.Delete(ctx, access)) + access = &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "replacement", Namespace: "project"}, Spec: access.Spec} + access.Spec.ContextRef.UID = replacement.UID + access.Spec.DestinationAddress = "fd70:100::20" + requireAccessTestSuccess(t, cl.Create(ctx, access)) + reconcile(now.Add(365 * 24 * time.Hour)) + requireAccessTestSuccess(t, cl.List(ctx, &bindings, client.InNamespace("platform"))) + active := 0 + for _, b := range bindings.Items { + if !b.Spec.Tombstone { + active++ + if b.Spec.Source.ResolverContextRef.UID != replacement.UID { + t.Fatal("new binding uses retired context") + } + } + } + if active != 1 { + t.Fatalf("active bindings=%d", active) + } +} + +func requireAccessTestSuccess(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } +} diff --git a/internal/internaldns/controlplane/context_reconciler.go b/internal/internaldns/controlplane/context_reconciler.go new file mode 100644 index 00000000..c09efda6 --- /dev/null +++ b/internal/internaldns/controlplane/context_reconciler.go @@ -0,0 +1,619 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controlplane + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "encoding/json" + "errors" + "net/netip" + "reflect" + "sort" + "time" + + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" + "go.miloapis.com/dns-operator/internal/internaldns/model" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +func jsonEqual(a, b any) bool { return reflect.DeepEqual(a, b) } + +func (r *Reconciler) reconcileNamingPolicies(ctx context.Context, items []dnsv1alpha1.DNSNamingPolicy, assocs []dnsv1alpha1.DNSZoneAssociation, zones map[string]*dnsv1alpha1.DNSZone, now time.Time) error { + for i := range items { + p := &items[i] + contextUID, contextName := types.UID(""), "" + valid := false + + var resolverContext dnsv1alpha1.DNSResolverContext + err := r.Get(ctx, client.ObjectKey{Namespace: p.Namespace, Name: p.Spec.ResolverContextRef.Name}, &resolverContext) + ready := apimeta.FindStatusCondition(resolverContext.Status.Conditions, "Ready") + valid = err == nil && p.Spec.ResolverContextRef.UID != "" && p.Spec.ResolverContextRef.UID == resolverContext.UID && resolverContext.DeletionTimestamp.IsZero() && ready != nil && ready.Status == metav1.ConditionTrue && ready.ObservedGeneration == resolverContext.Generation + if valid { + contextUID = resolverContext.UID + contextName = resolverContext.Name + } + + resolved := []dnsv1alpha1.DNSResolvedAdditionalNameRule{} + seen := map[string]bool{} + for _, rule := range p.Spec.AdditionalNames { + z := zones[rule.DNSZoneRef.Name] + associated := associatedWithContext(assocs, zUID(z), contextUID) + + if z == nil || z.Spec.Visibility != dnsv1alpha1.DNSZoneVisibilityPrivate || !refMatches(rule.DNSZoneRef, z.Name, z.UID, z.Generation) || !associated { + valid = false + continue + } + selector, err := metav1.LabelSelectorAsSelector(rule.Selector.AsLabelSelector()) + if err != nil { + valid = false + continue + } + key := rule.DNSZoneRef.Name + "\x00" + canonicalName(rule.NamePrefix) + "\x00" + selector.String() + if rule.NamePrefix == "" || seen[key] { + valid = false + continue + } + seen[key] = true + resolved = append(resolved, dnsv1alpha1.DNSResolvedAdditionalNameRule{Selector: rule.Selector.AsLabelSelector().DeepCopy(), DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: z.Name, UID: z.UID, Generation: z.Generation}, NamePrefix: canonicalName(rule.NamePrefix)}) + } + base := p.DeepCopy() + + p.Status.ResolvedResolverContextRef = dnsv1alpha1.DNSObjectReference{Name: contextName, UID: contextUID} + + p.Status.ResolvedAdditionalNames = resolved + status := metav1.ConditionTrue + reason, msg := dnsValueAccepted, "naming policy is valid" + if !valid { + status = metav1.ConditionFalse + reason = "InvalidPolicy" + msg = "all rules require an associated private zone and unique naming rules" + } + if setCondition(&p.Status.Conditions, dnsValueAccepted, status, reason, msg, p.Generation, now) { + if err := r.Status().Patch(ctx, p, client.MergeFrom(base)); err != nil { + return err + } + } + } + return nil +} + +func zUID(z *dnsv1alpha1.DNSZone) types.UID { + if z == nil { + return "" + } + return z.UID +} + +func associatedWithContext(assocs []dnsv1alpha1.DNSZoneAssociation, zoneUID, contextUID types.UID) bool { + for _, a := range assocs { + c := apimeta.FindStatusCondition(a.Status.Conditions, dnsValueAccepted) + if c != nil && c.Status == metav1.ConditionTrue && a.Status.ResolvedDNSZoneRef.UID == zoneUID && a.Status.ResolvedResolverContextRef.UID == contextUID { + return true + } + } + return false +} + +func (r *Reconciler) reconcileResolverContexts(ctx context.Context, ns string, now time.Time) error { + var contexts dnsv1alpha1.DNSResolverContextList + var existing dnsv1alpha1.DNSResolverBindingList + if err := r.List(ctx, &existing, client.InNamespace(r.Options.PlatformNamespace)); err != nil { + return err + } + if err := r.List(ctx, &contexts, client.InNamespace(ns)); err != nil { + return err + } + sort.Slice(contexts.Items, func(i, j int) bool { return contexts.Items[i].Name < contexts.Items[j].Name }) + consumerOwners := map[string]types.UID{} + for i := range contexts.Items { + c := &contexts.Items[i] + base := c.DeepCopy() + valid := c.Spec.ConsumerID != "" && c.DeletionTimestamp.IsZero() + duplicate := consumerOwners[c.Spec.ConsumerID] != "" && consumerOwners[c.Spec.ConsumerID] != c.UID + if valid && !duplicate { + consumerOwners[c.Spec.ConsumerID] = c.UID + } + valid = valid && !duplicate + invalidDomain := c.Spec.ManagedNamespace.Enabled && len(validation.IsDNS1123Subdomain(r.Options.ManagedDomainSuffix)) > 0 + valid = valid && !invalidDomain + c.Status.ServingTargets = nil + if valid { + for _, target := range r.selectServingTargets(c.UID, existing.Items) { + c.Status.ServingTargets = append(c.Status.ServingTargets, dnsv1alpha1.DNSResolverContextServingTarget{Region: target.Region, Shard: target.Shard}) + } + } + ready := valid && !c.Spec.ManagedNamespace.Enabled + if c.Spec.ManagedNamespace.Enabled && valid { + var err error + ready, err = r.reconcileManagedZone(ctx, c) + if err != nil { + return err + } + } + status, reason, msg := metav1.ConditionTrue, dnsValueAccepted, "resolver context is active" + if !valid { + status, reason, msg = metav1.ConditionFalse, "InvalidIdentity", "consumerID is required and context must be live" + if duplicate { + reason, msg = "DuplicateConsumerID", "another live resolver context owns this consumerID" + } + } + if valid && !ready { + status, reason, msg = metav1.ConditionFalse, "Preparing", "managed namespace is not ready" + } + setCondition(&c.Status.Conditions, "Ready", status, reason, msg, c.Generation, now) + if !jsonEqual(base.Status, c.Status) { + if err := r.Status().Patch(ctx, c, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { + return err + } + } + } + return nil +} + +func (r *Reconciler) reconcileAssociations(ctx context.Context, items []dnsv1alpha1.DNSZoneAssociation, zones map[string]*dnsv1alpha1.DNSZone, now time.Time) error { + type apexKey struct { + resolverContextUID types.UID + apex string + } + owners := map[apexKey]types.UID{} + sort.Slice(items, func(i, j int) bool { return items[i].Name < items[j].Name }) + for i := range items { + a := &items[i] + z := zones[a.Spec.DNSZoneRef.Name] + status := metav1.ConditionTrue + reason, msg := dnsValueAccepted, "association accepted" + contextUID, contextErr := types.UID(""), error(nil) + + var resolverContext dnsv1alpha1.DNSResolverContext + contextErr = r.Get(ctx, client.ObjectKey{Namespace: a.Namespace, Name: a.Spec.ResolverContextRef.Name}, &resolverContext) + if contextErr == nil && (a.Spec.ResolverContextRef.UID == "" || a.Spec.ResolverContextRef.UID != resolverContext.UID || !resolverContext.DeletionTimestamp.IsZero()) { + contextErr = errors.New("resolver context lifetime is not current") + } + if contextErr == nil { + contextUID = resolverContext.UID + } + + if !a.DeletionTimestamp.IsZero() || z == nil || z.Spec.Visibility != dnsv1alpha1.DNSZoneVisibilityPrivate || contextErr != nil || !refMatches(a.Spec.DNSZoneRef, z.Name, z.UID, z.Generation) { + status = metav1.ConditionFalse + reason = "InvalidReference" + msg = "private zone and pinned context/zone UIDs are required" + } else if old := owners[apexKey{contextUID, canonicalName(z.Spec.DomainName)}]; old != "" && old != z.UID { + status = metav1.ConditionFalse + reason = "ApexConflict" + msg = "another zone owns this apex in the resolver context" + } else { + owners[apexKey{contextUID, canonicalName(z.Spec.DomainName)}] = z.UID + } + base := a.DeepCopy() + + a.Status.ResolvedResolverContextRef = dnsv1alpha1.DNSObjectReference{Name: a.Spec.ResolverContextRef.Name, UID: contextUID} + + if z != nil { + a.Status.ResolvedDNSZoneRef = dnsv1alpha1.DNSObjectReference{Name: z.Name, UID: z.UID, Generation: z.Generation} + } + if setCondition(&a.Status.Conditions, dnsValueAccepted, status, reason, msg, a.Generation, now) { + if err := r.Status().Patch(ctx, a, client.MergeFrom(base)); err != nil { + return err + } + } + } + return nil +} + +func (r *Reconciler) reconcileBindings(ctx context.Context, ns string, assocs []dnsv1alpha1.DNSZoneAssociation, zones map[string]*dnsv1alpha1.DNSZone, now time.Time) error { + return r.reconcileContextBindings(ctx, ns, assocs, zones, now) +} + +func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, assocs []dnsv1alpha1.DNSZoneAssociation, zones map[string]*dnsv1alpha1.DNSZone, now time.Time) error { + var accesses dnsv1alpha1.DNSResolverAccessBindingList + var existing dnsv1alpha1.DNSResolverBindingList + if err := r.List(ctx, &accesses, client.InNamespace(ns)); err != nil { + return err + } + if err := r.List(ctx, &existing, client.InNamespace(r.Options.PlatformNamespace)); err != nil { + return err + } + zonesByContext := map[types.UID][]types.UID{} + for _, a := range assocs { + z := zones[a.Spec.DNSZoneRef.Name] + accepted := apimeta.FindStatusCondition(a.Status.Conditions, dnsValueAccepted) + if z != nil && accepted != nil && accepted.Status == metav1.ConditionTrue { + zonesByContext[a.Status.ResolvedResolverContextRef.UID] = append(zonesByContext[a.Status.ResolvedResolverContextRef.UID], z.UID) + } + } + sort.Slice(accesses.Items, func(i, j int) bool { return accesses.Items[i].Name < accesses.Items[j].Name }) + destinations := map[string]types.UID{} + desired := map[string]bool{} + for i := range accesses.Items { + a := &accesses.Items[i] + base := a.DeepCopy() + var current dnsv1alpha1.DNSResolverAccessBinding + if err := r.sourceReader().Get(ctx, client.ObjectKeyFromObject(a), ¤t); err != nil { + if apierrors.IsNotFound(err) { + continue + } + return err + } + if current.UID != a.UID || current.ResourceVersion != a.ResourceVersion { + continue + } + spec := a.Spec + var contextObject *dnsv1alpha1.DNSResolverContext + var currentContext dnsv1alpha1.DNSResolverContext + if err := r.sourceReader().Get(ctx, client.ObjectKey{Namespace: ns, Name: spec.ContextRef.Name}, ¤tContext); err != nil { + if !apierrors.IsNotFound(err) { + return err + } + contextObject = nil + } else if currentContext.UID == spec.ContextRef.UID { + contextObject = ¤tContext + } else { + contextObject = nil + } + contextReady := resolverContextReady(contextObject) + valid := a.DeletionTimestamp.IsZero() && contextObject != nil && contextObject.DeletionTimestamp.IsZero() && contextObject.Name == spec.ContextRef.Name && contextReady + if _, err := netip.ParseAddr(spec.DestinationAddress); err != nil { + valid = false + } + if _, ok := r.targetForRegion(spec.Region); !ok { + valid = false + } + key := spec.Region + "\x00" + spec.DestinationAddress + if owner := destinations[key]; owner != "" && owner != a.UID { + valid = false + } else if valid { + if r.Options.AddressAllocator == nil || r.Options.AddressAllocator.ClaimDestination(ctx, r.Options.ProjectUID, contextObject.UID, spec.Region, netip.MustParseAddr(spec.DestinationAddress).String(), 53) != nil { + valid = false + } else { + destinations[key] = a.UID + } + } + status, reason, msg := metav1.ConditionTrue, dnsValueAccepted, "access binding is current" + if !valid { + status, reason, msg = metav1.ConditionFalse, "InvalidOrStaleAccess", "context lifetime and destination must be current" + } + setCondition(&a.Status.Conditions, dnsValueAccepted, status, reason, msg, a.Generation, now) + if !valid { + a.Status.BindingRef = nil + setCondition(&a.Status.Conditions, "Ready", metav1.ConditionFalse, "AccessRejected", "access binding is not accepted", a.Generation, now) + if !jsonEqual(base.Status, a.Status) { + if err := r.Status().Patch(ctx, a, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { + return err + } + } + continue + } + zids := append([]types.UID(nil), zonesByContext[contextObject.UID]...) + sort.Slice(zids, func(i, j int) bool { return zids[i] < zids[j] }) + name := "binding-" + model.OpaqueToken(string(r.Options.ProjectUID) + "/" + string(a.UID))[:20] + desired[name] = true + var old dnsv1alpha1.DNSResolverBinding + err := r.Get(ctx, client.ObjectKey{Namespace: r.Options.PlatformNamespace, Name: name}, &old) + exists := err == nil + if err != nil && !apierrors.IsNotFound(err) { + return err + } + cluster := "" + generation, revision := int64(1), int64(1) + version := dnsv1alpha1.DNSResolverAccessVersion{Epoch: 1, Sequence: 1} + if exists { + version = old.Spec.Version + if old.Spec.Tombstone { + version.Sequence++ + } + cluster = old.Spec.Configuration.Query.RegionalAddress + generation = old.Spec.Configuration.Generation + revision = old.Spec.Configuration.Revision + if !zoneUIDsEqual(old.Spec.Configuration.ZoneRefs, zids) { + generation++ + revision++ + } else if old.Spec.Tombstone { + revision++ + } + } else { + if r.Options.AddressAllocator == nil { + return errors.New("address allocator is required for context binding") + } + cluster, err = r.Options.AddressAllocator.Allocate(ctx, r.Options.ProjectUID, contextObject.UID) + if err != nil { + return err + } + } + binding := &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: r.Options.PlatformNamespace, Labels: map[string]string{"internal-dns.miloapis.com/context-uid": string(contextObject.UID), "internal-dns.miloapis.com/access-uid": string(a.UID), "internal-dns.miloapis.com/region": model.SafeToken(spec.Region), "internal-dns.miloapis.com/shard": model.SafeToken(r.shardForContextRegion(contextObject.UID, spec.Region, existing.Items))}}, Spec: dnsv1alpha1.DNSResolverBindingSpec{Source: dnsv1alpha1.DNSResolverBindingSource{ProjectUID: r.Options.ProjectUID, ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: contextObject.Name, UID: contextObject.UID}, AccessBindingRef: dnsv1alpha1.DNSObjectReference{Name: a.Name, UID: a.UID, Generation: a.Generation}}, Placement: dnsv1alpha1.DNSResolverBindingPlacement{Region: spec.Region, Shard: r.shardForContextRegion(contextObject.UID, spec.Region, existing.Items)}, Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Generation: generation, Revision: revision, Query: dnsv1alpha1.DNSResolverBindingQuery{NodeAddress: spec.DestinationAddress, RegionalAddress: cluster}, ZoneRefs: zoneReferences(zids, zones), ZoneApices: zoneApices(zids, zones)}, Version: version}} + if !exists { + if err := r.Create(ctx, binding); err != nil { + return err + } + if err := r.Get(ctx, client.ObjectKeyFromObject(binding), binding); err != nil { + return err + } + } else if !bindingEqual(old.Spec, binding.Spec) { + baseOld := old.DeepCopy() + old.Spec = binding.Spec + old.Labels = binding.Labels + if err := r.Patch(ctx, &old, client.MergeFromWithOptions(baseOld, client.MergeFromWithOptimisticLock{})); err != nil { + return err + } + binding = &old + } else { + binding = &old + } + a.Status.ObservedGeneration = a.Generation + a.Status.BindingRef = &dnsv1alpha1.DNSObjectReference{Name: binding.Name, UID: binding.UID} + setCondition(&a.Status.Conditions, "Ready", metav1.ConditionFalse, "PendingReplicaVerification", "serving fleet acknowledgement is pending", a.Generation, now) + if c := apimeta.FindStatusCondition(binding.Status.Conditions, "Serving"); c != nil && c.Status == metav1.ConditionTrue && binding.Status.ObservedConfigurationRevision == binding.Spec.Configuration.Revision && bindingAckCurrent(binding, now) { + setCondition(&a.Status.Conditions, "Ready", metav1.ConditionTrue, "Ready", "serving capacity applied this access generation", a.Generation, now) + } else if c != nil && c.Status == metav1.ConditionUnknown { + setCondition(&a.Status.Conditions, "Ready", metav1.ConditionUnknown, "ObservationUnavailable", "current serving health is unavailable", a.Generation, now) + } + if !jsonEqual(base.Status, a.Status) { + if err := r.Status().Patch(ctx, a, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { + return err + } + } + } + for i := range existing.Items { + b := &existing.Items[i] + if b.Spec.Source.ProjectUID != r.Options.ProjectUID || desired[b.Name] || b.Spec.Tombstone { + continue + } + if b.Labels["internal-dns.miloapis.com/context-uid"] == "" { + continue + } + withdraw, err := r.confirmAccessWithdrawal(ctx, ns, b) + if err != nil { + return err + } + if !withdraw { + continue + } + base := b.DeepCopy() + b.Spec.Tombstone = true + b.Spec.Configuration.Revision++ + b.Spec.Version.Sequence++ + if err := r.Patch(ctx, b, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { + return err + } + } + return nil +} + +// confirmAccessWithdrawal prevents a stale or incomplete source List from +// withdrawing a binding renewed by another controller. Read uncertainty leaves +// the existing version intact; serving agents retain the applied configuration. +func (r *Reconciler) confirmAccessWithdrawal(ctx context.Context, namespace string, binding *dnsv1alpha1.DNSResolverBinding) (bool, error) { + var access dnsv1alpha1.DNSResolverAccessBinding + if err := r.sourceReader().Get(ctx, client.ObjectKey{Namespace: namespace, Name: binding.Spec.Source.AccessBindingRef.Name}, &access); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, err + } + if access.UID != binding.Spec.Source.AccessBindingRef.UID || !access.DeletionTimestamp.IsZero() { + return true, nil + } + if access.Spec.ContextRef.Name != binding.Spec.Source.ResolverContextRef.Name || access.Spec.ContextRef.UID != binding.Spec.Source.ResolverContextRef.UID { + return true, nil + } + var resolverContext dnsv1alpha1.DNSResolverContext + if err := r.sourceReader().Get(ctx, client.ObjectKey{Namespace: namespace, Name: binding.Spec.Source.ResolverContextRef.Name}, &resolverContext); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, err + } + if resolverContext.UID != binding.Spec.Source.ResolverContextRef.UID || !resolverContext.DeletionTimestamp.IsZero() { + return true, nil + } + ready := apimeta.FindStatusCondition(resolverContext.Status.Conditions, "Ready") + if ready == nil || ready.ObservedGeneration != resolverContext.Generation { + return false, nil + } + if ready.Status != metav1.ConditionTrue { + return ready.Reason == "InvalidIdentity" || ready.Reason == "DuplicateConsumerID" || ready.Reason == "InvalidManagedDomain", nil + } + if access.Generation < binding.Spec.Source.AccessBindingRef.Generation { + return false, nil + } + valid := access.Spec.Region == binding.Spec.Placement.Region && access.Spec.DestinationAddress == binding.Spec.Configuration.Query.NodeAddress + return !valid, nil +} + +func (r *Reconciler) shardForContextRegion(contextUID types.UID, region string, existing []dnsv1alpha1.DNSResolverBinding) string { + for _, target := range r.selectServingTargets(contextUID, existing) { + if target.Region == region { + return target.Shard + } + } + return "" +} + +func (r *Reconciler) targetForRegion(region string) (ServingRegion, bool) { + for _, s := range r.Options.servingRegions() { + if s.Region == region { + return s, true + } + } + return ServingRegion{}, false +} + +func bindingAckCurrent(binding *dnsv1alpha1.DNSResolverBinding, now time.Time) bool { + for _, ack := range binding.Status.MemberAcknowledgements { + // Serving=True is the aggregate full-member gate computed by AckSink. + // This row check additionally prevents a stale condition from surviving + // past the exact configuration/version generation or its local ACK lease. + if ack.Phase == string(model.AckVerified) && ack.Revision == binding.Spec.Configuration.Revision && ack.AccessEpoch == binding.Spec.Version.Epoch && ack.AccessSequence == binding.Spec.Version.Sequence && ack.ValidUntil != nil && ack.ValidUntil.After(now) { + return true + } + } + return false +} + +// selectServingTargets assigns one shared shard in each region. An existing +// live assignment wins while its shard remains configured, so adding capacity +// does not silently move established resolver listeners. Explicit migration is a +// separate operation; only a missing/removed assignment invokes rendezvous. +func (r *Reconciler) selectServingTargets(resolverContextUID types.UID, existing []dnsv1alpha1.DNSResolverBinding) []ServingRegion { + byRegion := map[string][]ServingRegion{} + for _, target := range r.Options.servingRegions() { + byRegion[target.Region] = append(byRegion[target.Region], target) + } + regions := make([]string, 0, len(byRegion)) + for region := range byRegion { + regions = append(regions, region) + } + sort.Strings(regions) + var selected []ServingRegion + for _, region := range regions { + candidates := byRegion[region] + sort.Slice(candidates, func(i, j int) bool { return candidates[i].Shard < candidates[j].Shard }) + var preserved *ServingRegion + for _, binding := range existing { + if binding.Spec.Source.ProjectUID != r.Options.ProjectUID || binding.Spec.Source.ResolverContextRef.UID != resolverContextUID || binding.Spec.Placement.Region != region || binding.Spec.Tombstone || !binding.DeletionTimestamp.IsZero() { + continue + } + for _, candidate := range candidates { + if binding.Spec.Placement.Shard == candidate.Shard { + c := candidate + if preserved == nil || c.Shard < preserved.Shard { + preserved = &c + } + } + } + } + if preserved != nil { + selected = append(selected, *preserved) + continue + } + best := candidates[0] + bestScore := "" + for _, candidate := range candidates { + sum := sha256.Sum256([]byte(string(r.Options.ProjectUID) + "/" + string(resolverContextUID) + "/" + region + "/" + candidate.Shard)) + score := string(sum[:]) + if bestScore == "" || score > bestScore { + best, bestScore = candidate, score + } + } + selected = append(selected, best) + } + return selected +} + +func (r *Reconciler) createIfAbsent(ctx context.Context, o client.Object) error { + err := r.Create(ctx, o) + if apierrors.IsAlreadyExists(err) { + return nil + } + return err +} + +func setCondition(cs *[]metav1.Condition, t string, status metav1.ConditionStatus, reason, msg string, generation int64, now time.Time) bool { + return apimeta.SetStatusCondition(cs, metav1.Condition{Type: t, Status: status, Reason: reason, Message: msg, ObservedGeneration: generation, LastTransitionTime: metav1.NewTime(now)}) +} + +// BindingAddressCandidates supplies stable IPv6 candidates to a durable global +// allocator. Callers must collision-check, persist, and quarantine leases. +func BindingAddressCandidates(uid types.UID) (string, string) { + sum := sha256.Sum256([]byte(uid)) + a := binary.BigEndian.Uint64(sum[:8]) + b := binary.BigEndian.Uint64(sum[8:16]) + return netip.AddrFrom16([16]byte{0xfd, 0x53, byte(a >> 56), byte(a >> 48), byte(a >> 40), byte(a >> 32), byte(a >> 24), byte(a >> 16), byte(a >> 8), byte(a), byte(b >> 40), byte(b >> 32), byte(b >> 24), byte(b >> 16), byte(b >> 8), byte(b)}).String(), netip.AddrFrom16([16]byte{0xfd, 0x54, byte(a >> 56), byte(a >> 48), byte(a >> 40), byte(a >> 32), byte(a >> 24), byte(a >> 16), byte(a >> 8), byte(a), byte(b >> 40), byte(b >> 32), byte(b >> 24), byte(b >> 16), byte(b >> 8), byte(b)}).String() +} + +func bindingEqual(a, b dnsv1alpha1.DNSResolverBindingSpec) bool { + x := a + y := b + xb, _ := json.Marshal(x) + yb, _ := json.Marshal(y) + return string(xb) == string(yb) +} + +func zoneApices(uids []types.UID, zones map[string]*dnsv1alpha1.DNSZone) map[string]string { + wanted := map[types.UID]bool{} + for _, uid := range uids { + wanted[uid] = true + } + apices := map[string]string{} + for _, zone := range zones { + if wanted[zone.UID] { + apices[string(zone.UID)] = zone.Spec.DomainName + } + } + return apices +} + +func zoneReferences(uids []types.UID, zones map[string]*dnsv1alpha1.DNSZone) []dnsv1alpha1.DNSObjectReference { + refs := make([]dnsv1alpha1.DNSObjectReference, 0, len(uids)) + for _, uid := range uids { + for _, zone := range zones { + if zone.UID == uid { + refs = append(refs, dnsv1alpha1.DNSObjectReference{Name: zone.Name, UID: uid}) + break + } + } + } + return refs +} + +func zoneUIDsEqual(refs []dnsv1alpha1.DNSObjectReference, uids []types.UID) bool { + if len(refs) != len(uids) { + return false + } + for i := range refs { + if refs[i].UID != uids[i] { + return false + } + } + return true +} + +func resolverContextReady(contextObject *dnsv1alpha1.DNSResolverContext) bool { + if contextObject == nil { + return false + } + ready := apimeta.FindStatusCondition(contextObject.Status.Conditions, "Ready") + return ready != nil && ready.Status == metav1.ConditionTrue && ready.ObservedGeneration == contextObject.Generation +} + +func (r *Reconciler) reconcileManagedZone(ctx context.Context, c *dnsv1alpha1.DNSResolverContext) (bool, error) { + suffix := canonicalName(model.OpaqueToken(string(c.UID)) + "." + r.Options.ManagedDomainSuffix) + zoneName := "managed-" + model.OpaqueToken(string(c.UID))[:12] + zone := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: zoneName, Namespace: c.Namespace, Labels: map[string]string{"dns.networking.miloapis.com/managed": "true"}}, Spec: dnsv1alpha1.DNSZoneSpec{DomainName: suffix, DNSZoneClassName: r.Options.PrivateZoneClassName, Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}} + if r.Scheme != nil { + if err := controllerutil.SetControllerReference(c, zone, r.Scheme, controllerutil.WithBlockOwnerDeletion(false)); err != nil { + return false, err + } + } + if err := r.createIfAbsent(ctx, zone); err != nil { + return false, err + } + if err := r.Get(ctx, client.ObjectKeyFromObject(zone), zone); err != nil { + return false, err + } + association := &dnsv1alpha1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: zoneName + "-context", Namespace: c.Namespace, Labels: map[string]string{"dns.networking.miloapis.com/managed": "true"}}, Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name, UID: zone.UID}, ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: c.Name, UID: c.UID}, Managed: true}} + if r.Scheme != nil { + if err := controllerutil.SetControllerReference(c, association, r.Scheme, controllerutil.WithBlockOwnerDeletion(false)); err != nil { + return false, err + } + } + if err := r.createIfAbsent(ctx, association); err != nil { + return false, err + } + if err := r.Get(ctx, client.ObjectKeyFromObject(association), association); err != nil { + return false, err + } + accepted := apimeta.FindStatusCondition(association.Status.Conditions, dnsValueAccepted) + ready := metav1.IsControlledBy(zone, c) && metav1.IsControlledBy(association, c) && zone.Spec.Visibility == dnsv1alpha1.DNSZoneVisibilityPrivate && zone.Spec.DomainName == suffix && zone.DeletionTimestamp.IsZero() && accepted != nil && accepted.Status == metav1.ConditionTrue && accepted.ObservedGeneration == association.Generation && association.Status.ResolvedDNSZoneRef.UID == zone.UID && association.Status.ResolvedResolverContextRef.UID == c.UID + if ready { + c.Status.ManagedNamespace = dnsv1alpha1.DNSResolverContextManagedNamespaceStatus{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name, UID: zone.UID}, Suffix: suffix} + } + return ready, nil +} diff --git a/internal/internaldns/controlplane/context_reconciler_test.go b/internal/internaldns/controlplane/context_reconciler_test.go new file mode 100644 index 00000000..ceced401 --- /dev/null +++ b/internal/internaldns/controlplane/context_reconciler_test.go @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controlplane + +import ( + "context" + "fmt" + "testing" + "time" + + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" + "go.miloapis.com/dns-operator/internal/internaldns/model" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +type testAllocator struct{} + +// accessListSnapshotClient models a source List taken before another +// controller commits a renewed platform binding. Gets remain authoritative. +type accessListSnapshotClient struct { + client.Client + accesses []dnsv1alpha1.DNSResolverAccessBinding + getErrorFor string +} + +func (c *accessListSnapshotClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + _, access := obj.(*dnsv1alpha1.DNSResolverAccessBinding) + _, resolverContext := obj.(*dnsv1alpha1.DNSResolverContext) + if (access && c.getErrorFor == "access") || (resolverContext && c.getErrorFor == "context") { + return fmt.Errorf("injected %s read uncertainty", c.getErrorFor) + } + return c.Client.Get(ctx, key, obj, opts...) +} + +func (c *accessListSnapshotClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + if accesses, ok := list.(*dnsv1alpha1.DNSResolverAccessBindingList); ok { + accesses.Items = append([]dnsv1alpha1.DNSResolverAccessBinding(nil), c.accesses...) + return nil + } + return c.Client.List(ctx, list, opts...) +} + +func (testAllocator) Allocate(context.Context, types.UID, types.UID) (string, error) { + return "fd54::10", nil +} + +func (testAllocator) ClaimDestination(context.Context, types.UID, types.UID, string, string, int32) error { + return nil +} + +func TestContextAccessPersistsUntilExplicitWithdrawal(t *testing.T) { + ctx := context.Background() + now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) + s := runtime.NewScheme() + if err := dnsv1alpha1.AddToScheme(s); err != nil { + t.Fatal(err) + } + contextObject := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "ctx", Namespace: "project", UID: "context-uid", Generation: 1}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "consumer-uid"}, Status: dnsv1alpha1.DNSResolverContextStatus{}} + setCondition(&contextObject.Status.Conditions, "Ready", metav1.ConditionTrue, "Ready", "", 1, now) + zone := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "zone", Namespace: "project", UID: "zone-uid", Generation: 1}, Spec: dnsv1alpha1.DNSZoneSpec{Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}} + association := &dnsv1alpha1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: "assoc", Namespace: "project", UID: "assoc-uid", Generation: 1}, Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: "zone", UID: "zone-uid"}, ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: "ctx", UID: "context-uid"}}, Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: "zone", UID: "zone-uid", Generation: 1}, ResolvedResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: "ctx", UID: "context-uid"}}} + setCondition(&association.Status.Conditions, "Accepted", metav1.ConditionTrue, "Accepted", "", 1, now) + access := &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "access", Namespace: "project", UID: "access-uid", Generation: 1}, Spec: dnsv1alpha1.DNSResolverAccessBindingSpec{ContextRef: dnsv1alpha1.DNSObjectReference{Name: "ctx", UID: "context-uid"}, Region: "central", DestinationAddress: "fd70:100::10"}} + c := fake.NewClientBuilder().WithScheme(s).WithObjects(contextObject, zone, association, access).WithStatusSubresource(&dnsv1alpha1.DNSResolverContext{}, &dnsv1alpha1.DNSResolverAccessBinding{}, &dnsv1alpha1.DNSResolverAccessBinding{}, &dnsv1alpha1.DNSResolverBinding{}).Build() + r := &Reconciler{Client: c, Scheme: s, Options: ReconcilerOptions{ProjectUID: "project-uid", PlatformNamespace: "platform", Region: "central", Shard: "shared", AddressAllocator: testAllocator{}}} + if err := r.reconcileContextBindings(ctx, "project", []dnsv1alpha1.DNSZoneAssociation{*association}, map[string]*dnsv1alpha1.DNSZone{"zone": zone}, now); err != nil { + t.Fatal(err) + } + var bindings dnsv1alpha1.DNSResolverBindingList + if err := c.List(ctx, &bindings, client.InNamespace("platform")); err != nil { + t.Fatal(err) + } + if len(bindings.Items) != 1 { + t.Fatalf("got %d bindings", len(bindings.Items)) + } + b := bindings.Items[0] + if b.Spec.Configuration.Query.NodeAddress != "fd70:100::10" || b.Spec.Source.ResolverContextRef.UID != "context-uid" || b.Spec.Version.Epoch != 1 || b.Spec.Version.Sequence != 1 { + t.Fatalf("access projection drifted: %#v", b.Spec) + } + if b.Spec.Source.ProjectUID != "project-uid" || b.Spec.Source.ResolverContextRef.Name != "ctx" || b.Spec.Source.AccessBindingRef.Name != "access" || b.Spec.Source.AccessBindingRef.UID != "access-uid" || b.Spec.Configuration.ZoneRefs[0].Name != "zone" || b.Spec.Configuration.ZoneRefs[0].UID != "zone-uid" || b.Spec.Placement.Region != "central" { + t.Fatalf("source identity or zone references drifted: %#v", b.Spec) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(access), access); err != nil { + t.Fatal(err) + } + if access.Status.ObservedGeneration != 1 { + t.Fatalf("source generation not observed: %#v", access.Status) + } + extraZone := zone.DeepCopy() + extraZone.Name = "extra" + extraZone.UID = "extra-uid" + extraAssociation := association.DeepCopy() + extraAssociation.Name = "extra-assoc" + extraAssociation.Spec.DNSZoneRef = dnsv1alpha1.DNSObjectReference{Name: extraZone.Name, UID: extraZone.UID} + extraAssociation.Status.ResolvedDNSZoneRef = extraAssociation.Spec.DNSZoneRef + if err := r.reconcileContextBindings(ctx, "project", []dnsv1alpha1.DNSZoneAssociation{*association, *extraAssociation}, map[string]*dnsv1alpha1.DNSZone{"zone": zone, "extra": extraZone}, now); err != nil { + t.Fatal(err) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(&b), &b); err != nil { + t.Fatal(err) + } + if b.Spec.Configuration.Generation != 2 || b.Spec.Configuration.Revision != 2 { + t.Fatalf("zone membership change reused configuration fences: %#v", b.Spec.Configuration) + } + + setCondition(&contextObject.Status.Conditions, "Ready", metav1.ConditionFalse, "InvalidIdentity", "", contextObject.Generation, now) + if err := c.Status().Update(ctx, contextObject); err != nil { + t.Fatal(err) + } + if err := r.reconcileContextBindings(ctx, "project", []dnsv1alpha1.DNSZoneAssociation{*association}, map[string]*dnsv1alpha1.DNSZone{"zone": zone}, now.Add(3*time.Minute)); err != nil { + t.Fatal(err) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(&b), &b); err != nil { + t.Fatal(err) + } + if !b.Spec.Tombstone || b.Spec.Version.Sequence != 2 || b.Spec.Configuration.Revision != 3 { + t.Fatalf("withdrawal failed to advance private fencing: %#v", b.Spec) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(access), access); err != nil { + t.Fatal(err) + } + ready := apimeta.FindStatusCondition(access.Status.Conditions, "Ready") + if ready == nil || ready.Status != metav1.ConditionFalse { + t.Fatalf("invalid access retained Ready: %#v", access.Status.Conditions) + } + staleAccess := *access.DeepCopy() + if err := c.Update(ctx, access); err != nil { + t.Fatal(err) + } + if err := r.reconcileContextBindings(ctx, "project", []dnsv1alpha1.DNSZoneAssociation{*association}, map[string]*dnsv1alpha1.DNSZone{"zone": zone}, now.Add(3*time.Minute)); err != nil { + t.Fatal(err) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(&b), &b); err != nil { + t.Fatal(err) + } + if !b.Spec.Tombstone { + t.Fatal("same-fence replay reopened access") + } + if err := c.Get(ctx, client.ObjectKeyFromObject(access), access); err != nil { + t.Fatal(err) + } + setCondition(&contextObject.Status.Conditions, "Ready", metav1.ConditionTrue, "Ready", "", contextObject.Generation, now) + if err := c.Status().Update(ctx, contextObject); err != nil { + t.Fatal(err) + } + if err := c.Update(ctx, access); err != nil { + t.Fatal(err) + } + if err := r.reconcileContextBindings(ctx, "project", []dnsv1alpha1.DNSZoneAssociation{*association}, map[string]*dnsv1alpha1.DNSZone{"zone": zone}, now.Add(3*time.Minute)); err != nil { + t.Fatal(err) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(&b), &b); err != nil { + t.Fatal(err) + } + if b.Spec.Tombstone || b.Spec.Version.Sequence != 3 || b.Spec.Configuration.Revision != 4 { + t.Fatalf("current context did not restore access: %#v", b.Spec) + } + for _, snapshot := range [][]dnsv1alpha1.DNSResolverAccessBinding{{staleAccess}, nil} { + staleController := &Reconciler{Client: &accessListSnapshotClient{Client: c, accesses: snapshot}, Scheme: s, Options: r.Options} + if err := staleController.reconcileContextBindings(ctx, "project", nil, nil, now.Add(3*time.Minute)); err != nil { + t.Fatal(err) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(&b), &b); err != nil { + t.Fatal(err) + } + if b.Spec.Tombstone || b.Spec.Configuration.Revision != 4 || b.Spec.Version.Sequence != 3 { + t.Fatalf("stale source snapshot withdrew renewed binding: %#v", b.Spec) + } + } + for _, kind := range []string{"access", "context"} { + uncertainController := &Reconciler{Client: &accessListSnapshotClient{Client: c, getErrorFor: kind}, Scheme: s, Options: r.Options} + if err := uncertainController.reconcileContextBindings(ctx, "project", nil, nil, now.Add(3*time.Minute)); err == nil { + t.Fatalf("%s read uncertainty was not surfaced", kind) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(&b), &b); err != nil { + t.Fatal(err) + } + if b.Spec.Tombstone || b.Spec.Configuration.Revision != 4 { + t.Fatalf("%s read uncertainty withdrew access", kind) + } + } + if err := c.Get(ctx, client.ObjectKeyFromObject(contextObject), contextObject); err != nil { + t.Fatal(err) + } + setCondition(&contextObject.Status.Conditions, "Ready", metav1.ConditionFalse, "Reconciling", "", contextObject.Generation, now.Add(3*time.Minute)) + if err := c.Status().Update(ctx, contextObject); err != nil { + t.Fatal(err) + } + uncertainController := &Reconciler{Client: &accessListSnapshotClient{Client: c}, Scheme: s, Options: r.Options} + if err := uncertainController.reconcileContextBindings(ctx, "project", nil, nil, now.Add(3*time.Minute)); err != nil { + t.Fatal(err) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(&b), &b); err != nil { + t.Fatal(err) + } + if b.Spec.Tombstone || b.Spec.Configuration.Revision != 4 { + t.Fatal("transient context readiness revoked a live authorization") + } +} + +func TestBindingAckCurrentRequiresVerifiedExactAuthorization(t *testing.T) { + now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) + ackDeadline := metav1.NewTime(now.Add(30 * time.Second)) + b := &dnsv1alpha1.DNSResolverBinding{Spec: dnsv1alpha1.DNSResolverBindingSpec{Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Revision: 7}, Version: dnsv1alpha1.DNSResolverAccessVersion{Epoch: 3, Sequence: 11}}, Status: dnsv1alpha1.DNSResolverBindingStatus{MemberAcknowledgements: []dnsv1alpha1.DNSApplyAcknowledgement{{Phase: string(model.AckVerified), Revision: 7, AccessEpoch: 3, AccessSequence: 11, ValidUntil: &ackDeadline}}}} + if !bindingAckCurrent(b, now) { + t.Fatal("exact verified ACK was rejected") + } + b.Status.MemberAcknowledgements[0].Phase = string(model.AckRejected) + if bindingAckCurrent(b, now) { + t.Fatal("rejected ACK was accepted") + } + b.Status.MemberAcknowledgements[0].Phase = string(model.AckVerified) + b.Status.MemberAcknowledgements[0].AccessSequence = 10 + if bindingAckCurrent(b, now) { + t.Fatal("stale authorization ACK was accepted") + } + b.Status.MemberAcknowledgements[0].AccessSequence = 11 +} + +func TestContextNamingPolicyResolvesMultipleZonesWithoutNetworking(t *testing.T) { + ctx := context.Background() + now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) + s := runtime.NewScheme() + if err := dnsv1alpha1.AddToScheme(s); err != nil { + t.Fatal(err) + } + resolverContext := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "ctx", Namespace: "project", UID: "ctx-uid", Generation: 2}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "consumer-uid"}} + setCondition(&resolverContext.Status.Conditions, "Ready", metav1.ConditionTrue, "Ready", "", 2, now) + z1 := dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "managed", Namespace: "project", UID: "zone-1", Generation: 1}, Spec: dnsv1alpha1.DNSZoneSpec{Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}} + z2 := dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "services", Namespace: "project", UID: "zone-2", Generation: 3}, Spec: dnsv1alpha1.DNSZoneSpec{Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}} + associations := []dnsv1alpha1.DNSZoneAssociation{{Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSObjectReference{UID: z1.UID}, ResolvedResolverContextRef: dnsv1alpha1.DNSObjectReference{UID: resolverContext.UID}}}, {Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSObjectReference{UID: z2.UID}, ResolvedResolverContextRef: dnsv1alpha1.DNSObjectReference{UID: resolverContext.UID}}}} + for i := range associations { + setCondition(&associations[i].Status.Conditions, "Accepted", metav1.ConditionTrue, "Accepted", "", 1, now) + } + policy := &dnsv1alpha1.DNSNamingPolicy{ObjectMeta: metav1.ObjectMeta{Name: "names", Namespace: "project", UID: "policy-uid", Generation: 1}, Spec: dnsv1alpha1.DNSNamingPolicySpec{ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: resolverContext.Name, UID: resolverContext.UID}, AdditionalNames: []dnsv1alpha1.DNSAdditionalNameRule{{Selector: nil, DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: z1.Name, UID: z1.UID, Generation: z1.Generation}, NamePrefix: "instances"}, {Selector: nil, DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: z2.Name, UID: z2.UID, Generation: z2.Generation}, NamePrefix: "services"}}}} + c := fake.NewClientBuilder().WithScheme(s).WithObjects(resolverContext, policy).WithStatusSubresource(&dnsv1alpha1.DNSResolverContext{}, &dnsv1alpha1.DNSResolverAccessBinding{}, &dnsv1alpha1.DNSNamingPolicy{}).Build() + r := &Reconciler{Client: c, Options: ReconcilerOptions{}} + if err := r.reconcileNamingPolicies(ctx, []dnsv1alpha1.DNSNamingPolicy{*policy}, associations, map[string]*dnsv1alpha1.DNSZone{z1.Name: &z1, z2.Name: &z2}, now); err != nil { + t.Fatal(err) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(policy), policy); err != nil { + t.Fatal(err) + } + accepted := apimeta.FindStatusCondition(policy.Status.Conditions, "Accepted") + if accepted == nil || accepted.Status != metav1.ConditionTrue || len(policy.Status.ResolvedAdditionalNames) != 2 { + t.Fatalf("context naming policy not resolved: %#v", policy.Status) + } + if policy.Status.ResolvedResolverContextRef.UID != resolverContext.UID { + t.Fatalf("identity projection incorrect: %#v", policy.Status) + } +} + +func TestStableOneShardPerRegionAssignment(t *testing.T) { + r := &Reconciler{Options: ReconcilerOptions{ProjectUID: "project", ServingRegions: []ServingRegion{{Region: "central", Shard: "a"}, {Region: "central", Shard: "b"}, {Region: "west", Shard: "w"}}}} + first := r.selectServingTargets("vpc-1", nil) + if len(first) != 2 { + t.Fatalf("got %d targets, want one per region", len(first)) + } + central := first[0] + if central.Region != "central" { + t.Fatalf("targets not sorted: %#v", first) + } + // Find another immutable VPC identity that rendezvous assigns to the other + // shard, proving the pool is shared rather than duplicated per VPC. + different := false + for i := 2; i < 200; i++ { + candidate := r.selectServingTargets(types.UID(fmt.Sprintf("vpc-%d", i)), nil) + if candidate[0].Shard != central.Shard { + different = true + break + } + } + if !different { + t.Fatal("rendezvous assignment never used the other shard") + } + existing := []dnsv1alpha1.DNSResolverBinding{{Spec: dnsv1alpha1.DNSResolverBindingSpec{Source: dnsv1alpha1.DNSResolverBindingSource{ProjectUID: "project", ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: string("vpc-1"), UID: "vpc-1"}}, Placement: dnsv1alpha1.DNSResolverBindingPlacement{Region: "central", Shard: central.Shard}}}} + r.Options.ServingRegions = append(r.Options.ServingRegions, ServingRegion{Region: "central", Shard: "c"}) + after := r.selectServingTargets("vpc-1", existing) + if after[0].Shard != central.Shard { + t.Fatalf("adding a shard moved live VPC: before=%s after=%s", central.Shard, after[0].Shard) + } +} diff --git a/internal/internaldns/controlplane/naming.go b/internal/internaldns/controlplane/naming.go new file mode 100644 index 00000000..9ca840ac --- /dev/null +++ b/internal/internaldns/controlplane/naming.go @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controlplane + +import ( + "context" + "fmt" + "sort" + "strings" + + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type ResolvedName struct { + DNSZoneRef dnsv1alpha1.DNSObjectReference + Name string + Canonical bool +} + +// ResolveNaming returns the stable platform namespace first and configured +// additive policy names after it. It never selects an arbitrary associated +// customer zone. +// Selectors match DNSRegistration metadata labels supplied by the publisher. +func ResolveNaming(ctx context.Context, c client.Client, namespace string, _ types.UID, contextUID types.UID, registrationLabels map[string]string, allocatedName string) ([]ResolvedName, error) { + label := dnsLabel(allocatedName) + if label == "" { + return nil, fmt.Errorf("allocated name %q has no DNS-safe label", allocatedName) + } + var managed dnsv1alpha1.DNSResolverContextList + if err := c.List(ctx, &managed, client.InNamespace(namespace)); err != nil { + return nil, err + } + var out []ResolvedName + for i := range managed.Items { + m := &managed.Items[i] + accepted := apimeta.FindStatusCondition(m.Status.Conditions, "Ready") + if m.UID == contextUID && m.DeletionTimestamp.IsZero() && accepted != nil && accepted.Status == metav1.ConditionTrue { + out = append(out, ResolvedName{DNSZoneRef: m.Status.ManagedNamespace.DNSZoneRef, Name: label, Canonical: true}) + break + } + } + if len(out) == 0 { + return nil, fmt.Errorf("managed DNS namespace for resolver context %s is not ready", contextUID) + } + seen := map[string]bool{string(out[0].DNSZoneRef.UID) + "\x00" + out[0].Name: true} + var policies dnsv1alpha1.DNSNamingPolicyList + if err := c.List(ctx, &policies, client.InNamespace(namespace)); err != nil { + return nil, err + } + sort.Slice(policies.Items, func(i, j int) bool { + if policies.Items[i].Spec.Priority == policies.Items[j].Spec.Priority { + return policies.Items[i].Name < policies.Items[j].Name + } + return policies.Items[i].Spec.Priority > policies.Items[j].Spec.Priority + }) + for i := range policies.Items { + p := &policies.Items[i] + accepted := apimeta.FindStatusCondition(p.Status.Conditions, dnsValueAccepted) + if p.Status.ResolvedResolverContextRef.UID != contextUID || !p.DeletionTimestamp.IsZero() || accepted == nil || accepted.Status != metav1.ConditionTrue || accepted.ObservedGeneration != p.Generation { + continue + } + for _, rule := range p.Status.ResolvedAdditionalNames { + selector, err := namingSelector(rule.Selector) + if err != nil { + return nil, fmt.Errorf("naming policy %s has an invalid selector: %w", p.Name, err) + } + name := label + "." + canonicalName(rule.NamePrefix) + key := string(rule.DNSZoneRef.UID) + "\x00" + name + if selector.Matches(labels.Set(registrationLabels)) && !seen[key] { + out = append(out, ResolvedName{DNSZoneRef: rule.DNSZoneRef, Name: name}) + seen[key] = true + } + } + } + return out, nil +} + +func namingSelector(selector *metav1.LabelSelector) (labels.Selector, error) { + if selector == nil { + return labels.Everything(), nil + } + return metav1.LabelSelectorAsSelector(selector) +} + +func dnsLabel(s string) string { + s = strings.ToLower(strings.TrimSpace(s)) + var b strings.Builder + dash := false + for _, r := range s { + if r >= 'a' && r <= 'z' || r >= '0' && r <= '9' { + b.WriteRune(r) + dash = false + } else if !dash && b.Len() > 0 { + b.WriteByte('-') + dash = true + } + } + return strings.Trim(b.String(), "-") +} diff --git a/internal/internaldns/controlplane/naming_test.go b/internal/internaldns/controlplane/naming_test.go new file mode 100644 index 00000000..c5423094 --- /dev/null +++ b/internal/internaldns/controlplane/naming_test.go @@ -0,0 +1,28 @@ +package controlplane + +import ( + "context" + "testing" + + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestResolveNamingReturnsCanonicalThenAuthorizedAliases(t *testing.T) { + s := runtime.NewScheme() + if err := dnsv1alpha1.AddToScheme(s); err != nil { + t.Fatal(err) + } + managed := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "prod", Namespace: "project", UID: "vpc-a"}, Status: dnsv1alpha1.DNSResolverContextStatus{ManagedNamespace: dnsv1alpha1.DNSResolverContextManagedNamespaceStatus{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: "canonical", UID: "zone-c"}}, Conditions: []metav1.Condition{{Type: "Ready", Status: metav1.ConditionTrue}}}} + policy := &dnsv1alpha1.DNSNamingPolicy{ObjectMeta: metav1.ObjectMeta{Name: "custom", Namespace: "project"}, Spec: dnsv1alpha1.DNSNamingPolicySpec{Priority: 10}, Status: dnsv1alpha1.DNSNamingPolicyStatus{ResolvedResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: "prod", UID: "vpc-a"}, ResolvedAdditionalNames: []dnsv1alpha1.DNSResolvedAdditionalNameRule{{Selector: nil, DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: "custom", UID: "zone-x"}, NamePrefix: "instances"}}, Conditions: []metav1.Condition{{Type: "Accepted", Status: metav1.ConditionTrue}}}} + c := fake.NewClientBuilder().WithScheme(s).WithObjects(managed, policy).Build() + names, err := ResolveNaming(context.Background(), c, "project", "project-a", "vpc-a", nil, "Web 01") + if err != nil { + t.Fatal(err) + } + if len(names) != 2 || names[0].Name != "web-01" || !names[0].Canonical || names[1].Name != "web-01.instances" || names[1].Canonical { + t.Fatalf("unexpected names: %#v", names) + } +} diff --git a/internal/internaldns/controlplane/reconciler.go b/internal/internaldns/controlplane/reconciler.go new file mode 100644 index 00000000..98b947eb --- /dev/null +++ b/internal/internaldns/controlplane/reconciler.go @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controlplane + +import ( + "context" + "errors" + "time" + + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" + "go.miloapis.com/dns-operator/internal/internaldns/model" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +type ReconcilerOptions struct { + ProjectUID types.UID + ProjectNamespace string + PlatformNamespace string + Region string + Shard string + Identity string + ChunkSize int + LeaseDuration time.Duration + Now func() time.Time + AddressAllocator AddressAllocator + ManagedDomainSuffix string + PrivateZoneClassName string + ServingRegions []ServingRegion +} + +type ServingRegion struct { + Region string + Shard string +} + +// AddressAllocator durably leases globally unique listener destinations. Its +// implementation must retain quarantine state after release; a pure hash is +// only a candidate and is never sufficient proof of uniqueness. +type AddressAllocator interface { + Allocate(context.Context, types.UID, types.UID) (cluster string, err error) + ClaimDestination(context.Context, types.UID, types.UID, string, string, int32) error +} + +func (o *ReconcilerOptions) defaults() { + if o.PlatformNamespace == "" { + o.PlatformNamespace = "internal-dns-system" + } + if o.Region == "" { + o.Region = "default" + } + if o.Shard == "" { + o.Shard = "private-dns-01" + } + if o.Identity == "" { + o.Identity = "internal-dns-controller" + } + if o.ChunkSize <= 0 { + o.ChunkSize = 256 * 1024 + } + if o.LeaseDuration <= 0 { + o.LeaseDuration = 30 * time.Second + } + + if o.Now == nil { + o.Now = func() time.Time { return time.Now().UTC() } + } +} + +func (o ReconcilerOptions) servingRegions() []ServingRegion { + if len(o.ServingRegions) > 0 { + return o.ServingRegions + } + return []ServingRegion{{Region: o.Region, Shard: o.Shard}} +} + +type Options = ReconcilerOptions + +// Reconciler compiles records within a configured project scope. +type Reconciler struct { + client.Client + APIReader client.Reader + Scheme *runtime.Scheme + Options ReconcilerOptions +} + +func (r *Reconciler) sourceReader() client.Reader { + if r.APIReader != nil { + return r.APIReader + } + return r.Client +} + +func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + r.Options.defaults() + if r.Options.ProjectUID == "" { + return ctrl.Result{}, errors.New("projectUID is required") + } + ns := r.Options.ProjectNamespace + if ns == "" { + ns = req.Namespace + } + if ns == "" || ns == r.Options.PlatformNamespace { + return ctrl.Result{}, nil + } + return r.reconcileProject(ctx, ns) +} + +func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { + r.Options.defaults() + if r.APIReader == nil { + r.APIReader = mgr.GetAPIReader() + } + enqueue := handler.EnqueueRequestsFromMapFunc(func(_ context.Context, o client.Object) []reconcile.Request { + return []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: o.GetNamespace(), Name: "project"}}} + }) + controllerName := "internal-dns-control-plane-" + model.OpaqueToken(string(r.Options.ProjectUID))[:12] + b := ctrl.NewControllerManagedBy(mgr).Named(controllerName).For(&dnsv1alpha1.DNSResolverContext{}, builder.WithPredicates()). + Watches(&dnsv1alpha1.DNSZone{}, enqueue).Watches(&dnsv1alpha1.DNSZoneAssociation{}, enqueue). + Watches(&dnsv1alpha1.DNSNamingPolicy{}, enqueue) + + return b.Watches(&dnsv1alpha1.DNSResolverAccessBinding{}, enqueue).Complete(r) + +} + +func (r *Reconciler) reconcileProject(ctx context.Context, ns string) (ctrl.Result, error) { + now := r.Options.Now().UTC() + + if err := r.reconcileResolverContexts(ctx, ns, now); err != nil { + return ctrl.Result{}, err + } + + var zones dnsv1alpha1.DNSZoneList + var assocs dnsv1alpha1.DNSZoneAssociationList + var policies dnsv1alpha1.DNSNamingPolicyList + for _, item := range []client.ObjectList{&zones, &assocs, &policies} { + if err := r.List(ctx, item, client.InNamespace(ns)); err != nil { + return ctrl.Result{}, err + } + } + zoneByName := map[string]*dnsv1alpha1.DNSZone{} + for i := range zones.Items { + z := &zones.Items[i] + zoneByName[z.Name] = z + } + if err := r.reconcileAssociations(ctx, assocs.Items, zoneByName, now); err != nil { + return ctrl.Result{}, err + } + if err := r.reconcileNamingPolicies(ctx, policies.Items, assocs.Items, zoneByName, now); err != nil { + return ctrl.Result{}, err + } + if err := r.reconcileBindings(ctx, ns, assocs.Items, zoneByName, now); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: r.Options.LeaseDuration / 2}, nil +} diff --git a/internal/internaldns/controlplane/references.go b/internal/internaldns/controlplane/references.go new file mode 100644 index 00000000..bb0410d7 --- /dev/null +++ b/internal/internaldns/controlplane/references.go @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controlplane + +import ( + "strings" + + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" + "k8s.io/apimachinery/pkg/types" +) + +func refMatches(ref dnsv1alpha1.DNSObjectReference, name string, uid types.UID, generation int64) bool { + return ref.Name == name && (ref.UID != "" && ref.UID == uid) && (ref.Generation == 0 || ref.Generation == generation) +} + +func canonicalName(s string) string { + return strings.ToLower(strings.TrimSuffix(strings.TrimSpace(s), ".")) +} diff --git a/internal/internaldns/controlplane/references_test.go b/internal/internaldns/controlplane/references_test.go new file mode 100644 index 00000000..32c10734 --- /dev/null +++ b/internal/internaldns/controlplane/references_test.go @@ -0,0 +1,24 @@ +package controlplane + +import ( + "testing" + + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" +) + +func TestReferenceRequiresExactLifetime(t *testing.T) { + for _, tt := range []struct { + ref dnsv1alpha1.DNSObjectReference + matches bool + }{ + {dnsv1alpha1.DNSObjectReference{Name: "zone"}, false}, + {dnsv1alpha1.DNSObjectReference{Name: "zone", UID: "old"}, false}, + {dnsv1alpha1.DNSObjectReference{Name: "zone", UID: "current"}, true}, + {dnsv1alpha1.DNSObjectReference{Name: "zone", UID: "current", Generation: 1}, false}, + {dnsv1alpha1.DNSObjectReference{Name: "zone", UID: "current", Generation: 2}, true}, + } { + if got := refMatches(tt.ref, "zone", "current", 2); got != tt.matches { + t.Errorf("%#v matches=%v", tt.ref, got) + } + } +} diff --git a/internal/internaldns/controlplane/values.go b/internal/internaldns/controlplane/values.go new file mode 100644 index 00000000..011b19dc --- /dev/null +++ b/internal/internaldns/controlplane/values.go @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controlplane + +const dnsValueAccepted = "Accepted" diff --git a/internal/internaldns/platform/allocator.go b/internal/internaldns/platform/allocator.go new file mode 100644 index 00000000..b9ebbaf4 --- /dev/null +++ b/internal/internaldns/platform/allocator.go @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package platform + +import ( + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "net/netip" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const leaseLabel = "internal-dns.miloapis.com/address-lease" + +type AddressLease struct { + Owner string `json:"owner"` + ProjectUID string `json:"projectUID"` + ContextUID string `json:"contextUID"` + ClusterAddress string `json:"clusterAddress"` + CreatedAt time.Time `json:"createdAt"` +} + +// Allocator reserves all generated addresses through atomic, globally named Kubernetes address claims. +// Each owner has its own lease, avoiding a fleet-wide size limit or write lock. Leases are retained after revocation: addresses are never reused by +// another resolver context lifetime. Explicit garbage collection needs a quarantine policy. +type Allocator struct { + Client client.Client + Namespace string + ClusterPrefix string +} + +func (a *Allocator) Allocate(ctx context.Context, projectUID, contextUID types.UID) (string, error) { + lease, err := a.reserve(ctx, string(projectUID), string(contextUID)) + return lease.ClusterAddress, err +} + +// ClaimDestination atomically reserves an integration-selected service +// destination across every project sharing this platform control plane. Claims +// are retained after revocation so a stale route cannot become valid for a new +// context lifetime. +func (a *Allocator) ClaimDestination(ctx context.Context, projectUID, accessUID types.UID, region, address string, port int32) error { + parsed, err := netip.ParseAddr(address) + if err != nil || parsed.IsUnspecified() || projectUID == "" || accessUID == "" || region == "" || port != 53 { + return fmt.Errorf("destination claim identity, region, IP, and DNS port are required") + } + address = parsed.String() + owner := string(projectUID) + "/" + string(accessUID) + claim := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "dns-destination-claim-" + model.OpaqueToken(region+"/"+address+"/"+fmt.Sprint(port)), Namespace: a.Namespace, Labels: map[string]string{leaseLabel: "true"}}, Data: map[string]string{"owner": owner, "region": region, "address": address, "port": fmt.Sprint(port)}} + if err := a.Client.Create(ctx, claim); apierrors.IsAlreadyExists(err) { + var existing corev1.ConfigMap + if err := a.Client.Get(ctx, client.ObjectKeyFromObject(claim), &existing); err != nil { + return err + } + if existing.Data["owner"] != owner || existing.Data["region"] != region || existing.Data["address"] != address || existing.Data["port"] != fmt.Sprint(port) { + return fmt.Errorf("service destination is already reserved") + } + return nil + } else { + return err + } +} + +func (a *Allocator) reserve(ctx context.Context, projectUID, contextUID string) (AddressLease, error) { + if a.Client == nil || a.Namespace == "" || projectUID == "" || contextUID == "" { + return AddressLease{}, fmt.Errorf("allocator requires a client, namespace, and immutable project/context identities") + } + owner := projectUID + "/" + contextUID + + prefix := a.ClusterPrefix + if prefix == "" { + prefix = "fd54::/64" + } + parsed, err := netip.ParsePrefix(prefix) + if err != nil || !parsed.Addr().Is6() || parsed.Bits() > 112 { + return AddressLease{}, fmt.Errorf("allocator prefix must be IPv6 with at least 16 host bits: %q", prefix) + } + parsed = parsed.Masked() + key := client.ObjectKey{Namespace: a.Namespace, Name: "dns-address-owner-" + model.OpaqueToken(owner)} + var stored corev1.ConfigMap + if err := a.Client.Get(ctx, key, &stored); err == nil { + return decodeLease(stored, owner) + } else if !apierrors.IsNotFound(err) { + return AddressLease{}, err + } + for salt := 0; salt < 1024; salt++ { + candidate := AddressLease{Owner: owner, ProjectUID: projectUID, ContextUID: contextUID, CreatedAt: time.Now().UTC()} + + candidate.ClusterAddress = addressInPrefix(parsed, owner+"/cluster", salt) + claim := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "dns-address-claim-" + model.OpaqueToken(candidate.ClusterAddress), Namespace: a.Namespace, Labels: map[string]string{leaseLabel: "true"}}, Data: map[string]string{"owner": owner, "address": candidate.ClusterAddress}} + if err := a.Client.Create(ctx, claim); apierrors.IsAlreadyExists(err) { + var existing corev1.ConfigMap + if err := a.Client.Get(ctx, client.ObjectKeyFromObject(claim), &existing); err != nil { + return AddressLease{}, err + } + if existing.Data["owner"] != owner || existing.Data["address"] != candidate.ClusterAddress { + continue + } + } else if err != nil { + return AddressLease{}, err + } + b, err := json.Marshal(candidate) + if err != nil { + return AddressLease{}, err + } + lease := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: key.Name, Namespace: key.Namespace, Labels: map[string]string{leaseLabel: "true"}}, Data: map[string]string{"lease": string(b)}} + if err := a.Client.Create(ctx, lease); apierrors.IsAlreadyExists(err) { + if err := a.Client.Get(ctx, key, &stored); err != nil { + return AddressLease{}, err + } + return decodeLease(stored, owner) + } else if err != nil { + return AddressLease{}, err + } + return candidate, nil + } + return AddressLease{}, fmt.Errorf("address collision budget exhausted") +} + +func decodeLease(stored corev1.ConfigMap, owner string) (AddressLease, error) { + var lease AddressLease + if err := json.Unmarshal([]byte(stored.Data["lease"]), &lease); err != nil { + return lease, fmt.Errorf("corrupt address lease: %w", err) + } + if lease.Owner != owner || lease.ProjectUID == "" || lease.ContextUID == "" { + return lease, fmt.Errorf("address lease identity mismatch") + } + return lease, nil +} + +func addressInPrefix(prefix netip.Prefix, owner string, salt int) string { + base := prefix.Addr().As16() + hash := sha256.Sum256([]byte(fmt.Sprintf("%s/%d", owner, salt))) + for bit := prefix.Bits(); bit < 128; bit++ { + i, mask := bit/8, byte(1< 0 { + t.Fatal(failures) + } + if len(addresses) != 40 { + t.Fatalf("got %d address claims", len(addresses)) + } + a, err := allocator.Allocate(ctx, "project", "vpc-1") + if err != nil { + t.Fatal(err) + } + a2, err := (&Allocator{Client: cl, Namespace: "platform"}).Allocate(ctx, "project", "vpc-1") + if err != nil || a != a2 { + t.Fatal("allocator restart changed an existing lease") + } + replacement, err := allocator.Allocate(ctx, "project", "new-vpc-lifetime") + if err != nil || replacement == a { + t.Fatal("new resource lifetime reused the old address") + } +} diff --git a/internal/internaldns/projectstatus/status.go b/internal/internaldns/projectstatus/status.go new file mode 100644 index 00000000..a6da7001 --- /dev/null +++ b/internal/internaldns/projectstatus/status.go @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +// Package projectstatus defines the single project-facing DNS publication +// projection used by both the source reconciler and the acknowledgement sink. +package projectstatus + +import ( + "time" + + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func Published(conditions *[]metav1.Condition, complete bool, generation int64, now time.Time) bool { + reason, message := "PendingReplicaVerification", "publication awaits required replica verification" + if complete { + reason, message = "Published", "required serving replicas verified the publication" + } + return set(conditions, "Published", complete, reason, message, generation, now) +} + +func Programmed(conditions *[]metav1.Condition, applied bool, generation int64, now time.Time) bool { + reason, message := "PendingReplicaApplication", "publication awaits serving replica application" + if applied { + reason, message = "Programmed", "at least one serving replica applied the publication" + } + return set(conditions, "Programmed", applied, reason, message, generation, now) +} + +func Available(conditions *[]metav1.Condition, available bool, generation int64, now time.Time) bool { + reason, message := "NoEligibleEndpoints", "publication contains no eligible endpoint" + if available { + reason, message = "Available", "publication contains an eligible endpoint" + } + return set(conditions, "Available", available, reason, message, generation, now) +} + +func set(conditions *[]metav1.Condition, conditionType string, value bool, reason, message string, generation int64, now time.Time) bool { + status := metav1.ConditionFalse + if value { + status = metav1.ConditionTrue + } + return apimeta.SetStatusCondition(conditions, metav1.Condition{Type: conditionType, Status: status, Reason: reason, Message: message, ObservedGeneration: generation, LastTransitionTime: metav1.NewTime(now)}) +} diff --git a/internal/internaldns/projectstatus/status_test.go b/internal/internaldns/projectstatus/status_test.go new file mode 100644 index 00000000..2033ad8b --- /dev/null +++ b/internal/internaldns/projectstatus/status_test.go @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package projectstatus + +import ( + "reflect" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestProjectionIsQuiescentAfterConvergence(t *testing.T) { + now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) + var conditions []metav1.Condition + if !Published(&conditions, true, 7, now) || !Programmed(&conditions, true, 7, now) || !Available(&conditions, false, 7, now) { + t.Fatal("initial projection did not report changes") + } + converged := append([]metav1.Condition(nil), conditions...) + if Published(&conditions, true, 7, now.Add(time.Minute)) || Programmed(&conditions, true, 7, now.Add(time.Minute)) || Available(&conditions, false, 7, now.Add(time.Minute)) { + t.Fatal("unchanged projection reported a status mutation") + } + if !reflect.DeepEqual(conditions, converged) { + t.Fatalf("unchanged projection churned conditions:\n%#v\n%#v", converged, conditions) + } +}