From 97301057565ed34c1f40f5c550fc93242975b55f Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 7 Oct 2026 15:16:21 -0500 Subject: [PATCH 01/33] feat: Publish private DNS names for compute instances --- cmd/main.go | 54 +- cmd/main_test.go | 22 + .../internal-dns-rbac/kustomization.yaml | 5 + config/components/internal-dns-rbac/role.yaml | 14 + .../internal-dns-rbac/role_binding.yaml | 12 + .../internal-dns-instance-publishing.md | 65 ++ internal/config/config.go | 28 + internal/config/config_test.go | 25 + internal/config/zz_generated.deepcopy.go | 36 ++ internal/controller/internaldns_publisher.go | 604 ++++++++++++++++++ .../controller/internaldns_publisher_test.go | 213 ++++++ internal/features/features.go | 10 + internal/features/features_test.go | 17 + test/internaldns/README.md | 38 ++ test/internaldns/fleet_smoke.py | 168 +++++ test/internaldns/publisher/main.go | 94 +++ test/internaldns/results/fleet-smoke.json | 235 +++++++ 17 files changed, 1639 insertions(+), 1 deletion(-) create mode 100644 config/components/internal-dns-rbac/kustomization.yaml create mode 100644 config/components/internal-dns-rbac/role.yaml create mode 100644 config/components/internal-dns-rbac/role_binding.yaml create mode 100644 docs/enhancements/internal-dns-instance-publishing.md create mode 100644 internal/controller/internaldns_publisher.go create mode 100644 internal/controller/internaldns_publisher_test.go create mode 100644 test/internaldns/README.md create mode 100644 test/internaldns/fleet_smoke.py create mode 100644 test/internaldns/publisher/main.go create mode 100644 test/internaldns/results/fleet-smoke.json diff --git a/cmd/main.go b/cmd/main.go index d015f74b..cc12fce2 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -159,7 +159,8 @@ func main() { flag.StringVar(&featureGatesFlag, "feature-gates", "", "A set of key=value pairs that describe feature gates for the compute operator. "+ "Example: --feature-gates=NetworkingIntegration=true. "+ - "Available features: NetworkingIntegration (default=false).") + "Available features: NetworkingIntegration, RuntimeClasses, InstanceTypes, InstanceConsoleSessions, "+ + "InternalDNSPublishing (all default=false).") opts := zap.Options{ Development: true, @@ -181,6 +182,7 @@ func main() { "RuntimeClasses", features.FeatureGate.Enabled(features.RuntimeClasses), "InstanceTypes", features.FeatureGate.Enabled(features.InstanceTypes), "InstanceConsoleSessions", features.FeatureGate.Enabled(features.InstanceConsoleSessions), + "InternalDNSPublishing", features.FeatureGate.Enabled(features.InternalDNSPublishing), ) ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) @@ -444,6 +446,29 @@ func main() { } } + // DNS publication is its own asynchronous management-plane controller. It is + // not registered at all while its default-off feature gate is disabled. + if features.FeatureGate.Enabled(features.InternalDNSPublishing) { + if serverConfig.Discovery.Mode != multiclusterproviders.ProviderMilo || !enableManagementControllers { + setupLog.Error(nil, "InternalDNSPublishing requires Milo discovery mode and management controllers so leases are renewed only from the authoritative project Instance") + os.Exit(1) + } + identities, identityErr := internalDNSProjectIdentities(serverConfig.InternalDNS) + if identityErr != nil { + setupLog.Error(identityErr, "invalid internal DNS publisher configuration") + os.Exit(1) + } + dnsPublisher := &controller.InternalDNSPublisherReconciler{ + ProjectIdentities: identities, + PrincipalSubject: serverConfig.InternalDNS.PrincipalSubject, + LeaseDuration: serverConfig.InternalDNS.LeaseDuration.Duration, + } + if err = dnsPublisher.SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "InternalDNSPublisher") + os.Exit(1) + } + } + // Instance types live in project control planes, which only the management // manager engages; cells read instance types but don't maintain them. if enableManagementControllers && features.FeatureGate.Enabled(features.InstanceTypes) { @@ -941,3 +966,30 @@ func setupManagementControllers(mgr mcmanager.Manager, federationClient client.C func computeWatchProviderClaims(mode multiclusterproviders.Provider) bool { return mode == multiclusterproviders.ProviderMilo } + +func internalDNSProjectIdentities(cfg config.InternalDNSConfig) (map[string]controller.InternalDNSProjectIdentity, error) { + if cfg.PrincipalSubject == "" { + return nil, errors.New("internalDNS.principalSubject is required") + } + identities := make(map[string]controller.InternalDNSProjectIdentity, len(cfg.Projects)) + for i, project := range cfg.Projects { + if project.Name == "" || project.ProjectUID == "" || project.SourceClusterUID == "" { + return nil, fmt.Errorf("internalDNS.projects[%d] requires name, projectUID, and sourceClusterUID", i) + } + if _, duplicate := identities[project.Name]; duplicate { + return nil, fmt.Errorf("internalDNS.projects contains duplicate project %q", project.Name) + } + identities[project.Name] = controller.InternalDNSProjectIdentity{ + ProjectName: project.Name, + ProjectUID: project.ProjectUID, + SourceClusterUID: project.SourceClusterUID, + } + } + if len(identities) == 0 { + return nil, errors.New("internalDNS.projects must contain at least one project") + } + if cfg.LeaseDuration.Duration > 90*time.Second { + return nil, fmt.Errorf("internalDNS.leaseDuration %s exceeds the DNS platform maximum 90s", cfg.LeaseDuration.Duration) + } + return identities, nil +} diff --git a/cmd/main_test.go b/cmd/main_test.go index df27d639..52d3eb45 100644 --- a/cmd/main_test.go +++ b/cmd/main_test.go @@ -7,10 +7,12 @@ import ( "path/filepath" "slices" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/yaml" @@ -19,9 +21,29 @@ import ( locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" multiclusterproviders "go.miloapis.com/milo/pkg/multicluster-runtime" + "go.datum.net/compute/internal/config" "go.datum.net/compute/internal/locations" ) +func TestInternalDNSProjectIdentities(t *testing.T) { + identities, err := internalDNSProjectIdentities(config.InternalDNSConfig{ + PrincipalSubject: "system:serviceaccount:compute-system:compute-manager", + LeaseDuration: metav1.Duration{Duration: 60 * time.Second}, + Projects: []config.InternalDNSProjectIdentity{{Name: "project-a", ProjectUID: "project-uid", SourceClusterUID: "source-uid"}}, + }) + require.NoError(t, err) + assert.Equal(t, "source-uid", identities["project-a"].SourceClusterUID) + + _, err = internalDNSProjectIdentities(config.InternalDNSConfig{PrincipalSubject: "subject"}) + assert.Error(t, err, "enabled publishing requires at least one pinned project identity") + _, err = internalDNSProjectIdentities(config.InternalDNSConfig{ + PrincipalSubject: "subject", + LeaseDuration: metav1.Duration{Duration: 91 * time.Second}, + Projects: []config.InternalDNSProjectIdentity{{Name: "project-a", ProjectUID: "project-uid", SourceClusterUID: "source-uid"}}, + }) + assert.Error(t, err, "a producer must not configure a lease DNS admission rejects") +} + // TestComputeWatchProviderClaims is the #171 guard: quota enforcement (and thus // the ResourceClaim watch) is wired only in Milo mode. Single/cluster mode must // stay false, so the manager never engages a ResourceClaim watch against a cell diff --git a/config/components/internal-dns-rbac/kustomization.yaml b/config/components/internal-dns-rbac/kustomization.yaml new file mode 100644 index 00000000..c451c7b9 --- /dev/null +++ b/config/components/internal-dns-rbac/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +resources: + - role.yaml + - role_binding.yaml diff --git a/config/components/internal-dns-rbac/role.yaml b/config/components/internal-dns-rbac/role.yaml new file mode 100644 index 00000000..d4b0057b --- /dev/null +++ b/config/components/internal-dns-rbac/role.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: compute-internal-dns-publisher +rules: + - apiGroups: ["dns.networking.miloapis.com"] + resources: ["dnsmanagednamespaces"] + verbs: ["get", "list"] + - apiGroups: ["dns.networking.miloapis.com"] + resources: ["dnsregistrations", "dnscontributiongrants", "dnsrecordcontributions"] + verbs: ["get", "list", "create", "update", "patch", "delete"] + - apiGroups: ["dns.networking.miloapis.com"] + resources: ["dnsrecordcontributions/status"] + verbs: ["get", "update", "patch"] diff --git a/config/components/internal-dns-rbac/role_binding.yaml b/config/components/internal-dns-rbac/role_binding.yaml new file mode 100644 index 00000000..dc6b0a43 --- /dev/null +++ b/config/components/internal-dns-rbac/role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: compute-internal-dns-publisher +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: compute-internal-dns-publisher +subjects: + - kind: ServiceAccount + name: compute-manager + namespace: compute-system diff --git a/docs/enhancements/internal-dns-instance-publishing.md b/docs/enhancements/internal-dns-instance-publishing.md new file mode 100644 index 00000000..7fbd2608 --- /dev/null +++ b/docs/enhancements/internal-dns-instance-publishing.md @@ -0,0 +1,65 @@ +# Internal DNS instance publishing + +Compute can publish stable private instance identity names through the internal +DNS API. The integration is disabled by default and supports `Instance` +resources only. It does not infer service discovery intent or publish public +addresses. + +Enable it with `--feature-gates=InternalDNSPublishing=true` and configure the +identity that the DNS control plane trusts for each Milo project: + +```yaml +apiVersion: apiserver.config.datumapis.com/v1alpha1 +kind: WorkloadOperator +internalDNS: + principalSubject: system:serviceaccount:compute-system:compute-manager + leaseDuration: 60s + projects: + - name: example-project + projectUID: 00000000-0000-0000-0000-000000000001 + sourceClusterUID: 00000000-0000-0000-0000-000000000010 +``` + +`projectUID` and `sourceClusterUID` must match the trusted project entry in the +DNS control-plane configuration. The principal must be the authenticated +Kubernetes username used for project API writes. These values select and +authorize an existing DNS-owned namespace; they do not authenticate the +publisher by themselves. + +The project API must separately authorize the authenticated principal to read +managed namespaces and to manage registrations, grants, contributions, and +producer-owned contribution status. In Milo this requires the corresponding DNS +`ProtectedResource`/IAM policy (or equivalent project authorization) for the +client certificate or token subject. The +`config/components/internal-dns-rbac` component supplies direct Kubernetes RBAC +for a same-API development environment; applying it only to Compute's deployment +cluster does not grant access through a Milo project control-plane endpoint. +Production rollout therefore requires matching infra/IAM provisioning before +the feature gate is enabled. + +The first integration requires `discovery.mode: milo` and management +controllers. It uses the engaged project client's authenticated connection and +renews leases only from an uncached read of the authoritative project Instance. +An edge or federation copy with a different UID is rejected, preventing an old +copy from publishing into a replacement Instance lifetime. + +For each VPC attachment, the publisher reads the project `Network` UID and +selects an accepted `DNSManagedNamespace` whose project and VPC UIDs both +match. It creates a generation-pinned `DNSRegistration`, +`DNSContributionGrant`, and `DNSRecordContribution`. The name combines the +immutable Instance UID with its allocated name, so display-name reuse cannot +adopt an earlier identity. A separate registration is used for every VPC. + +Only allocated and programmed private host addresses are published. IPv4 and +IPv6 are supported; a delegated prefix is not treated as one instance address. +Application readiness does not affect instance identity. Address release or +network deprogramming writes an ineligible observation, and deletion is +garbage-collected through the project-side Instance owner reference. The +publisher refreshes a bounded lease from current Instance/interface state, so a +disconnected publisher cannot retain an address indefinitely. + +DNS reconciliation errors requeue this controller and do not modify Compute +readiness. Publication state remains available on the DNS registration and +contribution resources. With the feature gate disabled, Compute does not +register the publisher, discover DNS resources, make DNS API calls, or require +the optional DNS RBAC component. diff --git a/internal/config/config.go b/internal/config/config.go index 2a7540d0..17a8b022 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -44,6 +44,11 @@ type WorkloadOperator struct { // ReferencedData configures the ReferencedDataController. ReferencedData ReferencedDataConfig `json:"referencedData,omitempty"` + // InternalDNS configures producer identity for the optional instance DNS + // publisher. It is ignored unless the InternalDNSPublishing feature gate is + // enabled. + InternalDNS InternalDNSConfig `json:"internalDNS,omitempty"` + // LocationSource names the API group locations are read from. Use // "NetworkServices" for networking.datumapis.com LocationBindings and // ServingLocations, or "Locations" for the dedicated locations.miloapis.com @@ -52,6 +57,29 @@ type WorkloadOperator struct { LocationSource locations.Source `json:"locationSource,omitempty"` } +// +k8s:deepcopy-gen=true +type InternalDNSConfig struct { + // PrincipalSubject is the authenticated Kubernetes username used by this + // process when writing project DNS resources. DNS admission verifies it; it + // is not an authentication substitute. + PrincipalSubject string `json:"principalSubject,omitempty"` + + // LeaseDuration bounds how long an instance address can remain published + // without a fresh observation. DNS admission currently permits at most 90s. + LeaseDuration metav1.Duration `json:"leaseDuration,omitempty"` + + // Projects pins each Milo project name to the trusted project and source API + // UIDs configured by the DNS control plane. + Projects []InternalDNSProjectIdentity `json:"projects,omitempty"` +} + +// +k8s:deepcopy-gen=true +type InternalDNSProjectIdentity struct { + Name string `json:"name"` + ProjectUID types.UID `json:"projectUID"` + SourceClusterUID string `json:"sourceClusterUID"` +} + func SetDefaults_WorkloadOperator(obj *WorkloadOperator) { if obj.LocationSource == "" { obj.LocationSource = locations.SourceNetworkServices diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 61c4ca33..eab9af75 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -4,6 +4,7 @@ import ( "os" "path/filepath" "testing" + "time" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -28,6 +29,30 @@ func decode(t *testing.T, data string) *WorkloadOperator { return &cfg } +func TestInternalDNSConfigDecodeAndDeepCopy(t *testing.T) { + cfg := decode(t, ` +apiVersion: apiserver.config.datumapis.com/v1alpha1 +kind: WorkloadOperator +metricsServer: + bindAddress: "0" +internalDNS: + principalSubject: system:serviceaccount:compute-system:compute-manager + leaseDuration: 60s + projects: + - name: project-a + projectUID: project-uid + sourceClusterUID: source-uid +`) + if cfg.InternalDNS.LeaseDuration.Duration != 60*time.Second { + t.Fatalf("leaseDuration = %s, want 60s", cfg.InternalDNS.LeaseDuration.Duration) + } + copy := cfg.DeepCopy() + copy.InternalDNS.Projects[0].ProjectUID = "changed" + if cfg.InternalDNS.Projects[0].ProjectUID != "project-uid" { + t.Fatal("WorkloadOperator DeepCopy aliases internal DNS project identities") + } +} + func TestWebhookServer_Absent(t *testing.T) { cfg := decode(t, ` apiVersion: apiserver.config.datumapis.com/v1alpha1 diff --git a/internal/config/zz_generated.deepcopy.go b/internal/config/zz_generated.deepcopy.go index 191a9cbc..4cac042d 100644 --- a/internal/config/zz_generated.deepcopy.go +++ b/internal/config/zz_generated.deepcopy.go @@ -66,6 +66,41 @@ func (in *FeatureFlagsConfig) DeepCopy() *FeatureFlagsConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InternalDNSConfig) DeepCopyInto(out *InternalDNSConfig) { + *out = *in + if in.Projects != nil { + in, out := &in.Projects, &out.Projects + *out = make([]InternalDNSProjectIdentity, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InternalDNSConfig. +func (in *InternalDNSConfig) DeepCopy() *InternalDNSConfig { + if in == nil { + return nil + } + out := new(InternalDNSConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InternalDNSProjectIdentity) DeepCopyInto(out *InternalDNSProjectIdentity) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InternalDNSProjectIdentity. +func (in *InternalDNSProjectIdentity) DeepCopy() *InternalDNSProjectIdentity { + if in == nil { + return nil + } + out := new(InternalDNSProjectIdentity) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MetricsServerConfig) DeepCopyInto(out *MetricsServerConfig) { *out = *in @@ -156,6 +191,7 @@ func (in *WorkloadOperator) DeepCopyInto(out *WorkloadOperator) { in.Discovery.DeepCopyInto(&out.Discovery) out.FeatureFlags = in.FeatureFlags in.ReferencedData.DeepCopyInto(&out.ReferencedData) + in.InternalDNS.DeepCopyInto(&out.InternalDNS) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadOperator. diff --git a/internal/controller/internaldns_publisher.go b/internal/controller/internaldns_publisher.go new file mode 100644 index 00000000..98726a82 --- /dev/null +++ b/internal/controller/internaldns_publisher.go @@ -0,0 +1,604 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/netip" + "reflect" + "sort" + "strings" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + ctrl "sigs.k8s.io/controller-runtime/pkg/reconcile" + mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder" + mccontroller "sigs.k8s.io/multicluster-runtime/pkg/controller" + mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" + "sigs.k8s.io/multicluster-runtime/pkg/multicluster" + mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" + + computev1alpha "go.datum.net/compute/api/v1alpha" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +const ( + internalDNSGroup = "dns.networking.miloapis.com" + internalDNSVersion = "v1alpha1" + internalDNSInstanceUID = "internal-dns.compute.datumapis.com/instance-uid" + internalDNSVPCUID = "internal-dns.compute.datumapis.com/vpc-uid" + internalDNSManagedBy = "internal-dns.compute.datumapis.com/managed-by" + internalDNSManager = "compute-instance-publisher" + defaultDNSLease = 60 * time.Second + maxDNSLease = 90 * time.Second + internalDNSPendingWait = 10 * time.Second + internalDNSAPITimeout = 10 * time.Second + defaultDNSConcurrency = 4 +) + +var ( + dnsManagedNamespaceGVK = schema.GroupVersionKind{Group: internalDNSGroup, Version: internalDNSVersion, Kind: "DNSManagedNamespace"} + dnsRegistrationGVK = schema.GroupVersionKind{Group: internalDNSGroup, Version: internalDNSVersion, Kind: "DNSRegistration"} + dnsGrantGVK = schema.GroupVersionKind{Group: internalDNSGroup, Version: internalDNSVersion, Kind: "DNSContributionGrant"} + dnsContributionGVK = schema.GroupVersionKind{Group: internalDNSGroup, Version: internalDNSVersion, Kind: "DNSRecordContribution"} +) + +// InternalDNSProjectIdentity is the trusted identity shared with the DNS +// control plane for one Milo project. ProjectName is the discovery key, while +// the two UIDs are immutable authorization identities. +type InternalDNSProjectIdentity struct { + ProjectName string + ProjectUID types.UID + SourceClusterUID string +} + +// InternalDNSProjectAccess supplies an uncached reader and authenticated writer +// for the owning project API. The writer's Kubernetes subject must match the +// configured principal subject; DNS admission verifies both that subject and +// SourceClusterUID. +type InternalDNSProjectAccess struct { + Reader client.Reader + Writer client.Client +} + +type InternalDNSProjectAccessFunc func( + ctx context.Context, + projectID string, + providerCluster cluster.Cluster, +) (InternalDNSProjectAccess, error) + +type internalDNSClusterGetter interface { + GetCluster(ctx context.Context, clusterName multicluster.ClusterName) (cluster.Cluster, error) +} + +// InternalDNSPublisherReconciler publishes the private addresses of an Instance +// under one stable identity name per attached VPC. DNS failures only requeue +// this controller; they never alter Compute readiness. +type InternalDNSPublisherReconciler struct { + mgr internalDNSClusterGetter + ProjectIDForInstance func(context.Context, multicluster.ClusterName, *computev1alpha.Instance) (string, error) + ProjectNamespaceForInstance func(context.Context, multicluster.ClusterName, *computev1alpha.Instance) (string, error) + ProjectAccess InternalDNSProjectAccessFunc + ProjectIdentities map[string]InternalDNSProjectIdentity + PrincipalSubject string + LeaseDuration time.Duration + MaxConcurrentReconciles int + Now func() time.Time +} + +func (r *InternalDNSPublisherReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) { + ctx, cancel := context.WithTimeout(ctx, internalDNSAPITimeout) + defer cancel() + + providerCluster, err := r.mgr.GetCluster(ctx, req.ClusterName) + if err != nil { + return ctrl.Result{}, err + } + + var instance computev1alpha.Instance + // Lease renewal must come from a live API read. A cached projection can + // survive a disconnected source and would incorrectly keep old addresses + // eligible forever. + if err := providerCluster.GetAPIReader().Get(ctx, req.NamespacedName, &instance); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + projectID, err := r.projectID(ctx, req.ClusterName, &instance) + if err != nil { + return ctrl.Result{}, fmt.Errorf("resolve DNS project identity: %w", err) + } + projectNamespace, err := r.projectNamespace(ctx, req.ClusterName, &instance) + if err != nil { + return ctrl.Result{}, fmt.Errorf("resolve DNS project namespace: %w", err) + } + identity, ok := r.ProjectIdentities[projectID] + if !ok || identity.ProjectUID == "" || identity.SourceClusterUID == "" { + return ctrl.Result{}, fmt.Errorf("internal DNS identity is not configured for project %q", projectID) + } + access, err := r.projectAccess(ctx, projectID, providerCluster) + if err != nil { + return ctrl.Result{}, err + } + // DNS resources live in the project API and are owned by the project-side + // Instance lifetime, even when this observation came from an edge copy with + // a different Kubernetes UID. This makes normal deletion and replacement + // cleanup independent of receiving a final reconcile event. + var projectInstance computev1alpha.Instance + if err := access.Reader.Get(ctx, client.ObjectKey{Namespace: projectNamespace, Name: instance.Name}, &projectInstance); err != nil { + return ctrl.Result{}, fmt.Errorf("resolve project Instance owner for DNS publication: %w", err) + } + if projectInstance.UID == "" { + return ctrl.Result{}, fmt.Errorf("project Instance %s/%s has no API-assigned UID", projectNamespace, instance.Name) + } + if projectInstance.UID != instance.UID { + return ctrl.Result{}, fmt.Errorf("refusing DNS publication from Instance copy UID %q for current project Instance UID %q", instance.UID, projectInstance.UID) + } + owner := metav1.OwnerReference{ + APIVersion: computev1alpha.GroupVersion.String(), + Kind: "Instance", + Name: projectInstance.Name, + UID: projectInstance.UID, + Controller: ptrBool(true), + } + + if !instance.DeletionTimestamp.IsZero() { + return ctrl.Result{}, r.deletePublishedResources(ctx, access, projectNamespace, projectInstance.UID, nil) + } + + attachments, err := instanceDNSAttachments(ctx, access.Reader, projectNamespace, &projectInstance) + if err != nil { + return ctrl.Result{}, err + } + desired := make(map[string]struct{}, len(attachments)) + pending := false + for _, attachment := range attachments { + resourceName := internalDNSResourceName(projectInstance.UID, attachment.VPCUID) + desired[resourceName] = struct{}{} + ready, err := r.reconcileAttachment(ctx, access, identity, projectNamespace, &projectInstance, owner, attachment, resourceName) + if err != nil { + return ctrl.Result{}, err + } + pending = pending || !ready + } + if err := r.deletePublishedResources(ctx, access, projectNamespace, projectInstance.UID, desired); err != nil { + return ctrl.Result{}, err + } + if pending { + return ctrl.Result{RequeueAfter: internalDNSPendingWait}, nil + } + return ctrl.Result{RequeueAfter: r.leaseDuration() / 2}, nil +} + +type instanceDNSAttachment struct { + VPCUID types.UID + RecordSets []any + Eligible bool +} + +func instanceDNSAttachments(ctx context.Context, reader client.Reader, projectNamespace string, instance *computev1alpha.Instance) ([]instanceDNSAttachment, error) { + statusByName := make(map[string]computev1alpha.InstanceNetworkInterfaceStatus, len(instance.Status.NetworkInterfaces)) + for _, status := range instance.Status.NetworkInterfaces { + statusByName[status.Name] = status + } + byVPC := map[types.UID]*instanceDNSAttachment{} + for _, nic := range instance.Spec.NetworkInterfaces { + namespace := nic.Network.Namespace + if namespace == "" { + namespace = projectNamespace + } + var network networkingv1alpha.Network + if err := reader.Get(ctx, client.ObjectKey{Namespace: namespace, Name: nic.Network.Name}, &network); err != nil { + return nil, fmt.Errorf("resolve VPC %s/%s for instance DNS: %w", namespace, nic.Network.Name, err) + } + if network.UID == "" { + return nil, fmt.Errorf("network %s/%s has no API-assigned UID", namespace, nic.Network.Name) + } + attachment := byVPC[network.UID] + if attachment == nil { + attachment = &instanceDNSAttachment{VPCUID: network.UID, RecordSets: []any{}} + byVPC[network.UID] = attachment + } + + status, found := statusByName[internalDNSInterfaceName(nic)] + if !found || !interfaceDNSPublishable(status) { + continue + } + attachment.Eligible = true + for _, address := range status.Addresses { + addr, ok := hostAddress(address.Address) + if !ok { + continue + } + rrtype, field := "A", "a" + if addr.Is6() { + rrtype, field = "AAAA", "aaaa" + } + attachment.RecordSets = append(attachment.RecordSets, map[string]any{ + "recordType": rrtype, + "records": []any{map[string]any{"name": "", field: map[string]any{"content": addr.String()}}}, + }) + } + } + + attachments := make([]instanceDNSAttachment, 0, len(byVPC)) + for _, attachment := range byVPC { + attachment.Eligible = attachment.Eligible && len(attachment.RecordSets) > 0 + sort.Slice(attachment.RecordSets, func(i, j int) bool { + return fmt.Sprint(attachment.RecordSets[i]) < fmt.Sprint(attachment.RecordSets[j]) + }) + attachments = append(attachments, *attachment) + } + sort.Slice(attachments, func(i, j int) bool { return attachments[i].VPCUID < attachments[j].VPCUID }) + return attachments, nil +} + +func internalDNSInterfaceName(nic computev1alpha.InstanceNetworkInterface) string { + if nic.Name != "" { + return nic.Name + } + return "eth0" +} + +func interfaceDNSPublishable(status computev1alpha.InstanceNetworkInterfaceStatus) bool { + allocated := apimeta.FindStatusCondition(status.Conditions, computev1alpha.InstanceNetworkInterfaceAllocated) + programmed := apimeta.FindStatusCondition(status.Conditions, computev1alpha.InstanceNetworkInterfaceProgrammed) + return allocated != nil && allocated.Status == metav1.ConditionTrue && programmed != nil && programmed.Status == metav1.ConditionTrue +} + +func hostAddress(raw string) (netip.Addr, bool) { + if addr, err := netip.ParseAddr(raw); err == nil { + return addr.Unmap(), true + } + prefix, err := netip.ParsePrefix(raw) + if err != nil || prefix.Bits() != prefix.Addr().BitLen() { + return netip.Addr{}, false + } + return prefix.Addr().Unmap(), true +} + +func (r *InternalDNSPublisherReconciler) reconcileAttachment(ctx context.Context, access InternalDNSProjectAccess, identity InternalDNSProjectIdentity, namespace string, instance *computev1alpha.Instance, owner metav1.OwnerReference, attachment instanceDNSAttachment, resourceName string) (bool, error) { + managed, err := findManagedNamespace(ctx, access.Reader, namespace, identity.ProjectUID, attachment.VPCUID) + if err != nil { + return false, err + } + if managed == nil { + return false, nil + } + zoneRef, _, _ := unstructured.NestedMap(managed.Object, "status", "dnsZoneRef") + if stringValue(zoneRef, "name") == "" || stringValue(zoneRef, "uid") == "" { + return false, nil + } + + labels := map[string]string{ + internalDNSInstanceUID: string(instance.UID), + internalDNSVPCUID: string(attachment.VPCUID), + internalDNSManagedBy: internalDNSManager, + } + ownerName := allocatedInstanceDNSName(instance) + for i := range attachment.RecordSets { + rrset := attachment.RecordSets[i].(map[string]any) + records := rrset["records"].([]any) + for _, record := range records { + record.(map[string]any)["name"] = ownerName + } + } + + registrationSpec := map[string]any{ + "dnsZoneRef": zoneRef, + "name": ownerName, + "recordTypes": []any{"A", "AAAA"}, + "publicationPolicy": "EligibleContributions", + "ttlSeconds": int64(30), + } + registration, changed, err := ensureDNSObject(ctx, access, dnsRegistrationGVK, namespace, resourceName, labels, owner, registrationSpec) + if err != nil || changed { + return false, err + } + registrationRef := objectReference(registration, true) + + grantSpec := map[string]any{ + "registrationRef": registrationRef, + "producerID": internalDNSManager, + "principal": map[string]any{ + "clusterUID": identity.SourceClusterUID, + "subject": r.PrincipalSubject, + }, + "recordTypes": []any{"A", "AAAA"}, + } + grant, changed, err := ensureDNSObject(ctx, access, dnsGrantGVK, namespace, resourceName, labels, owner, grantSpec) + if err != nil || changed { + return false, err + } + writerEpoch, _, _ := unstructured.NestedInt64(grant.Object, "status", "activeWriterEpoch") + observedGrant, _, _ := unstructured.NestedInt64(grant.Object, "status", "observedGrantGeneration") + observedRegistration, _, _ := unstructured.NestedInt64(grant.Object, "status", "observedRegistrationGeneration") + if writerEpoch < 1 || observedGrant != grant.GetGeneration() || observedRegistration != registration.GetGeneration() { + return false, nil + } + + contributionSpec := map[string]any{ + "registrationRef": registrationRef, + "grantRef": objectReference(grant, false), + "recordSets": attachment.RecordSets, + } + contribution, changed, err := ensureDNSObject(ctx, access, dnsContributionGVK, namespace, resourceName, labels, owner, contributionSpec) + if err != nil || changed { + return false, err + } + boundEpoch, _, _ := unstructured.NestedInt64(contribution.Object, "status", "writerEpoch") + if boundEpoch != writerEpoch { + return false, nil + } + + return r.refreshObservation(ctx, access.Writer, contribution, writerEpoch, attachment.Eligible) +} + +func (r *InternalDNSPublisherReconciler) refreshObservation(ctx context.Context, writer client.Client, contribution *unstructured.Unstructured, writerEpoch int64, eligible bool) (bool, error) { + now := time.Now() + if r.Now != nil { + now = r.Now() + } + oldEligible, _, _ := unstructured.NestedBool(contribution.Object, "status", "eligible") + observedGeneration, _, _ := unstructured.NestedInt64(contribution.Object, "status", "observedGeneration") + validUntilRaw, _, _ := unstructured.NestedString(contribution.Object, "status", "validUntil") + validUntil, _ := time.Parse(time.RFC3339Nano, validUntilRaw) + if oldEligible == eligible && observedGeneration == contribution.GetGeneration() && validUntil.After(now.Add(r.leaseDuration()/2)) { + return true, nil + } + + base := contribution.DeepCopy() + sequence, _, _ := unstructured.NestedInt64(contribution.Object, "status", "sequence") + owned := []struct { + path []string + value any + }{ + {[]string{"status", "observedGeneration"}, contribution.GetGeneration()}, + {[]string{"status", "writerEpoch"}, writerEpoch}, + {[]string{"status", "sequence"}, sequence + 1}, + {[]string{"status", "eligible"}, eligible}, + {[]string{"status", "reason"}, map[bool]string{true: "AddressAllocatedAndProgrammed", false: "AddressUnavailable"}[eligible]}, + {[]string{"status", "validUntil"}, now.Add(r.leaseDuration()).UTC().Format(time.RFC3339Nano)}, + } + for _, field := range owned { + if err := unstructured.SetNestedField(contribution.Object, field.value, field.path...); err != nil { + return false, err + } + } + // Optimistic locking turns a concurrent DNS status write into a conflict and + // retry. The retry reads the live sequence and platform-owned status fields; + // it cannot overwrite a publication condition from a stale snapshot. + if err := writer.Status().Patch(ctx, contribution, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { + return false, fmt.Errorf("refresh DNS contribution observation: %w", err) + } + return true, nil +} + +func findManagedNamespace(ctx context.Context, reader client.Reader, namespace string, projectUID, vpcUID types.UID) (*unstructured.Unstructured, error) { + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(dnsManagedNamespaceGVK.GroupVersion().WithKind(dnsManagedNamespaceGVK.Kind + "List")) + if err := reader.List(ctx, list, client.InNamespace(namespace)); err != nil { + return nil, fmt.Errorf("list managed DNS namespaces: %w", err) + } + var match *unstructured.Unstructured + for i := range list.Items { + item := &list.Items[i] + if !item.GetDeletionTimestamp().IsZero() { + continue + } + puid, _, _ := unstructured.NestedString(item.Object, "spec", "projectUID") + vid, _, _ := unstructured.NestedString(item.Object, "spec", "vpcRef", "uid") + if puid != string(projectUID) || vid != string(vpcUID) || !unstructuredConditionTrue(item, "Accepted") { + continue + } + if match != nil { + return nil, fmt.Errorf("multiple accepted managed DNS namespaces match project UID %q and VPC UID %q", projectUID, vpcUID) + } + match = item.DeepCopy() + } + return match, nil +} + +func unstructuredConditionTrue(obj *unstructured.Unstructured, conditionType string) bool { + conditions, _, _ := unstructured.NestedSlice(obj.Object, "status", "conditions") + for _, raw := range conditions { + condition, ok := raw.(map[string]any) + if ok && condition["type"] == conditionType && condition["status"] == "True" { + return true + } + } + return false +} + +func ensureDNSObject(ctx context.Context, access InternalDNSProjectAccess, gvk schema.GroupVersionKind, namespace, name string, labels map[string]string, owner metav1.OwnerReference, spec map[string]any) (*unstructured.Unstructured, bool, error) { + desired := newDNSObject(gvk, namespace, name) + desired.SetLabels(labels) + desired.SetOwnerReferences([]metav1.OwnerReference{owner}) + desired.Object["spec"] = spec + current := newDNSObject(gvk, namespace, name) + err := access.Reader.Get(ctx, client.ObjectKeyFromObject(desired), current) + if apierrors.IsNotFound(err) { + if err := access.Writer.Create(ctx, desired); err != nil { + return nil, false, fmt.Errorf("create %s %s/%s: %w", gvk.Kind, namespace, name, err) + } + return desired, true, nil + } + if err != nil { + return nil, false, err + } + if !current.GetDeletionTimestamp().IsZero() { + return nil, false, fmt.Errorf("%s %s/%s is terminating", gvk.Kind, namespace, name) + } + if current.GetLabels()[internalDNSManagedBy] != internalDNSManager || current.GetLabels()[internalDNSInstanceUID] != labels[internalDNSInstanceUID] || !ownedBy(current, owner) { + return nil, false, fmt.Errorf("refusing to adopt foreign %s %s/%s", gvk.Kind, namespace, name) + } + currentSpec, _, _ := unstructured.NestedMap(current.Object, "spec") + if authorityReferencesChanged(gvk, currentSpec, spec) { + if err := access.Writer.Delete(ctx, current); client.IgnoreNotFound(err) != nil { + return nil, false, err + } + return current, true, nil + } + if !reflect.DeepEqual(currentSpec, spec) || !reflect.DeepEqual(current.GetLabels(), labels) || !reflect.DeepEqual(current.GetOwnerReferences(), []metav1.OwnerReference{owner}) { + current.Object["spec"] = spec + current.SetLabels(labels) + current.SetOwnerReferences([]metav1.OwnerReference{owner}) + if err := access.Writer.Update(ctx, current); err != nil { + return nil, false, fmt.Errorf("update %s %s/%s: %w", gvk.Kind, namespace, name, err) + } + return current, true, nil + } + return current, false, nil +} + +func ownedBy(obj metav1.Object, owner metav1.OwnerReference) bool { + for _, ref := range obj.GetOwnerReferences() { + if ref.UID == owner.UID && ref.APIVersion == owner.APIVersion && ref.Kind == owner.Kind && ref.Name == owner.Name { + return true + } + } + return false +} + +func ptrBool(value bool) *bool { return &value } + +func authorityReferencesChanged(gvk schema.GroupVersionKind, current, desired map[string]any) bool { + if gvk != dnsContributionGVK { + return false + } + return !reflect.DeepEqual(current["registrationRef"], desired["registrationRef"]) || !reflect.DeepEqual(current["grantRef"], desired["grantRef"]) +} + +func (r *InternalDNSPublisherReconciler) deletePublishedResources(ctx context.Context, access InternalDNSProjectAccess, namespace string, instanceUID types.UID, keep map[string]struct{}) error { + selector := client.MatchingLabels{internalDNSInstanceUID: string(instanceUID), internalDNSManagedBy: internalDNSManager} + for _, gvk := range []schema.GroupVersionKind{dnsContributionGVK, dnsGrantGVK, dnsRegistrationGVK} { + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(gvk.GroupVersion().WithKind(gvk.Kind + "List")) + if err := access.Reader.List(ctx, list, client.InNamespace(namespace), selector); err != nil { + return fmt.Errorf("list stale %s resources: %w", gvk.Kind, err) + } + for i := range list.Items { + item := &list.Items[i] + if _, retained := keep[item.GetName()]; retained { + continue + } + if err := access.Writer.Delete(ctx, item); client.IgnoreNotFound(err) != nil { + return fmt.Errorf("delete stale %s %s/%s: %w", gvk.Kind, namespace, item.GetName(), err) + } + } + } + return nil +} + +func newDNSObject(gvk schema.GroupVersionKind, namespace, name string) *unstructured.Unstructured { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(gvk) + obj.SetNamespace(namespace) + obj.SetName(name) + return obj +} + +func objectReference(obj *unstructured.Unstructured, includeGeneration bool) map[string]any { + ref := map[string]any{"name": obj.GetName(), "uid": string(obj.GetUID())} + if includeGeneration { + ref["generation"] = obj.GetGeneration() + } + return ref +} + +func stringValue(values map[string]any, key string) string { + value, _ := values[key].(string) + return value +} + +func allocatedInstanceDNSName(instance *computev1alpha.Instance) string { + base := strings.ToLower(instance.Name) + var b strings.Builder + for _, r := range base { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '-' { + b.WriteRune(r) + } else { + b.WriteByte('-') + } + } + base = strings.Trim(b.String(), "-") + if base == "" { + base = "instance" + } + if len(base) > 40 { + base = strings.TrimRight(base[:40], "-") + } + return base + "-" + shortHash(string(instance.UID)) + ".instances" +} + +func internalDNSResourceName(instanceUID, vpcUID types.UID) string { + return "instance-" + shortHash(string(instanceUID)+"/"+string(vpcUID)) +} + +func shortHash(value string) string { + sum := sha256.Sum256([]byte(value)) + return hex.EncodeToString(sum[:])[:16] +} + +func (r *InternalDNSPublisherReconciler) projectID(ctx context.Context, clusterName multicluster.ClusterName, instance *computev1alpha.Instance) (string, error) { + if r.ProjectIDForInstance != nil { + return r.ProjectIDForInstance(ctx, clusterName, instance) + } + return string(clusterName), nil +} + +func (r *InternalDNSPublisherReconciler) projectNamespace(ctx context.Context, clusterName multicluster.ClusterName, instance *computev1alpha.Instance) (string, error) { + if r.ProjectNamespaceForInstance != nil { + return r.ProjectNamespaceForInstance(ctx, clusterName, instance) + } + return instance.Namespace, nil +} + +func (r *InternalDNSPublisherReconciler) projectAccess(ctx context.Context, projectID string, providerCluster cluster.Cluster) (InternalDNSProjectAccess, error) { + if r.ProjectAccess != nil { + return r.ProjectAccess(ctx, projectID, providerCluster) + } + return InternalDNSProjectAccess{Reader: providerCluster.GetAPIReader(), Writer: providerCluster.GetClient()}, nil +} + +func (r *InternalDNSPublisherReconciler) leaseDuration() time.Duration { + if r.LeaseDuration <= 0 { + return defaultDNSLease + } + if r.LeaseDuration > maxDNSLease { + return maxDNSLease + } + return r.LeaseDuration +} + +func (r *InternalDNSPublisherReconciler) SetupWithManager(mgr mcmanager.Manager) error { + if strings.TrimSpace(r.PrincipalSubject) == "" { + return errors.New("internal DNS principal subject is required") + } + if len(r.ProjectIdentities) == 0 { + return errors.New("at least one internal DNS project identity is required") + } + if r.LeaseDuration > maxDNSLease { + return fmt.Errorf("internal DNS lease duration %s exceeds the platform maximum %s", r.LeaseDuration, maxDNSLease) + } + r.mgr = mgr + concurrency := r.MaxConcurrentReconciles + if concurrency <= 0 { + concurrency = defaultDNSConcurrency + } + return mcbuilder.ControllerManagedBy(mgr). + Named("internal-dns-instance-publisher"). + For(&computev1alpha.Instance{}, mcbuilder.WithEngageWithLocalCluster(false)). + WithOptions(mccontroller.Options{MaxConcurrentReconciles: concurrency}). + Complete(r) +} diff --git a/internal/controller/internaldns_publisher_test.go b/internal/controller/internaldns_publisher_test.go new file mode 100644 index 00000000..a80efe77 --- /dev/null +++ b/internal/controller/internaldns_publisher_test.go @@ -0,0 +1,213 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + computev1alpha "go.datum.net/compute/api/v1alpha" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +func internalDNSTestScheme(t *testing.T) *runtime.Scheme { + t.Helper() + s := runtime.NewScheme() + require.NoError(t, computev1alpha.AddToScheme(s)) + require.NoError(t, networkingv1alpha.AddToScheme(s)) + for _, gvk := range []struct { + kind schemaKind + }{ + {schemaKind{dnsManagedNamespaceGVK}}, + {schemaKind{dnsRegistrationGVK}}, + {schemaKind{dnsGrantGVK}}, + {schemaKind{dnsContributionGVK}}, + } { + s.AddKnownTypeWithName(gvk.kind.gvk, &unstructured.Unstructured{}) + s.AddKnownTypeWithName(gvk.kind.gvk.GroupVersion().WithKind(gvk.kind.gvk.Kind+"List"), &unstructured.UnstructuredList{}) + } + return s +} + +type schemaKind struct{ gvk schema.GroupVersionKind } + +func TestInstanceDNSAttachmentsMapsTenancyAndAddressFamilies(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, networkingv1alpha.AddToScheme(scheme)) + network := &networkingv1alpha.Network{ObjectMeta: metav1.ObjectMeta{Name: "prod", Namespace: "default", UID: "vpc-uid"}} + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(network).Build() + instance := &computev1alpha.Instance{ + ObjectMeta: metav1.ObjectMeta{Name: "api", Namespace: "default", UID: "instance-uid"}, + Spec: computev1alpha.InstanceSpec{NetworkInterfaces: []computev1alpha.InstanceNetworkInterface{{Name: "eth0", Network: networkingv1alpha.NetworkRef{Name: "prod"}}}}, + Status: computev1alpha.InstanceStatus{ + // Application readiness is deliberately false. Instance identity is + // governed by the interface allocation/programming lifecycle. + Conditions: []metav1.Condition{{Type: computev1alpha.InstanceReady, Status: metav1.ConditionFalse}}, + NetworkInterfaces: []computev1alpha.InstanceNetworkInterfaceStatus{{ + Name: "eth0", + Addresses: []computev1alpha.InstanceNetworkInterfaceAddress{ + {Address: "10.0.0.8/32"}, + {Address: "2001:db8::8/128"}, + {Address: "2001:db8:1::/96"}, // delegated block is not one host identity + }, + Conditions: []metav1.Condition{ + {Type: computev1alpha.InstanceNetworkInterfaceAllocated, Status: metav1.ConditionTrue}, + {Type: computev1alpha.InstanceNetworkInterfaceProgrammed, Status: metav1.ConditionTrue}, + }, + }}, + }, + } + + attachments, err := instanceDNSAttachments(context.Background(), cl, "default", instance) + require.NoError(t, err) + require.Len(t, attachments, 1) + assert.Equal(t, types.UID("vpc-uid"), attachments[0].VPCUID) + assert.True(t, attachments[0].Eligible) + assert.Len(t, attachments[0].RecordSets, 2, "A and AAAA host addresses should be retained") + + instance.Status.NetworkInterfaces[0].Conditions[1].Status = metav1.ConditionFalse + attachments, err = instanceDNSAttachments(context.Background(), cl, "default", instance) + require.NoError(t, err) + require.Len(t, attachments, 1) + assert.False(t, attachments[0].Eligible) + assert.NotNil(t, attachments[0].RecordSets) + assert.Empty(t, attachments[0].RecordSets) +} + +func TestFindManagedNamespaceRequiresProjectAndVPCUID(t *testing.T) { + s := runtime.NewScheme() + s.AddKnownTypeWithName(dnsManagedNamespaceGVK, &unstructured.Unstructured{}) + s.AddKnownTypeWithName(dnsManagedNamespaceGVK.GroupVersion().WithKind("DNSManagedNamespaceList"), &unstructured.UnstructuredList{}) + wrong := managedNamespaceObject("wrong", "other-project", "vpc-a", true) + right := managedNamespaceObject("right", "project-a", "vpc-a", true) + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(wrong, right).Build() + + got, err := findManagedNamespace(context.Background(), cl, "default", "project-a", "vpc-a") + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, "right", got.GetName()) + + got, err = findManagedNamespace(context.Background(), cl, "default", "project-b", "vpc-a") + require.NoError(t, err) + assert.Nil(t, got) +} + +func TestFindManagedNamespaceRejectsAmbiguityAndIgnoresTerminatingBinding(t *testing.T) { + s := runtime.NewScheme() + s.AddKnownTypeWithName(dnsManagedNamespaceGVK, &unstructured.Unstructured{}) + s.AddKnownTypeWithName(dnsManagedNamespaceGVK.GroupVersion().WithKind("DNSManagedNamespaceList"), &unstructured.UnstructuredList{}) + first := managedNamespaceObject("first", "project-a", "vpc-a", true) + second := managedNamespaceObject("second", "project-a", "vpc-a", true) + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(first, second).Build() + + _, err := findManagedNamespace(context.Background(), cl, "default", "project-a", "vpc-a") + require.Error(t, err) + assert.Contains(t, err.Error(), "multiple accepted") + + now := metav1.Now() + second.SetDeletionTimestamp(&now) + second.SetFinalizers([]string{"test.example/finalizer"}) + cl = fake.NewClientBuilder().WithScheme(s).WithObjects(first, second).Build() + got, err := findManagedNamespace(context.Background(), cl, "default", "project-a", "vpc-a") + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, "first", got.GetName()) +} + +func managedNamespaceObject(name, projectUID, vpcUID string, accepted bool) *unstructured.Unstructured { + status := "False" + if accepted { + status = "True" + } + obj := newDNSObject(dnsManagedNamespaceGVK, "default", name) + obj.Object["spec"] = map[string]any{"projectUID": projectUID, "vpcRef": map[string]any{"uid": vpcUID}} + obj.Object["status"] = map[string]any{ + "conditions": []any{map[string]any{"type": "Accepted", "status": status}}, + "dnsZoneRef": map[string]any{"name": "zone", "uid": "zone-uid", "generation": int64(1)}, + } + return obj +} + +func TestRefreshObservationPreservesPlatformStatusAndIncrementsFence(t *testing.T) { + s := runtime.NewScheme() + s.AddKnownTypeWithName(dnsContributionGVK, &unstructured.Unstructured{}) + s.AddKnownTypeWithName(dnsContributionGVK.GroupVersion().WithKind("DNSRecordContributionList"), &unstructured.UnstructuredList{}) + contribution := newDNSObject(dnsContributionGVK, "default", "endpoint") + contribution.SetUID("contribution-uid") + contribution.SetGeneration(3) + contribution.SetResourceVersion("1") + contribution.Object["spec"] = map[string]any{"recordSets": []any{}} + contribution.Object["status"] = map[string]any{ + "writerEpoch": int64(7), + "sequence": int64(11), + "eligible": false, + "publishedRevision": int64(44), + "conditions": []any{map[string]any{"type": "Published", "status": "True"}}, + } + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(contribution).WithStatusSubresource(contribution).Build() + now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) + r := &InternalDNSPublisherReconciler{LeaseDuration: 60 * time.Second, Now: func() time.Time { return now }} + + var live unstructured.Unstructured + live.SetGroupVersionKind(dnsContributionGVK) + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(contribution), &live)) + ready, err := r.refreshObservation(context.Background(), cl, &live, 7, true) + require.NoError(t, err) + assert.True(t, ready) + + var got unstructured.Unstructured + got.SetGroupVersionKind(dnsContributionGVK) + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(contribution), &got)) + sequence, _, _ := unstructured.NestedInt64(got.Object, "status", "sequence") + published, _, _ := unstructured.NestedInt64(got.Object, "status", "publishedRevision") + conditions, _, _ := unstructured.NestedSlice(got.Object, "status", "conditions") + assert.Equal(t, int64(12), sequence) + assert.Equal(t, int64(44), published) + assert.Len(t, conditions, 1) +} + +func TestEnsureDNSObjectRefusesForeignCollision(t *testing.T) { + s := runtime.NewScheme() + s.AddKnownTypeWithName(dnsRegistrationGVK, &unstructured.Unstructured{}) + s.AddKnownTypeWithName(dnsRegistrationGVK.GroupVersion().WithKind("DNSRegistrationList"), &unstructured.UnstructuredList{}) + foreign := newDNSObject(dnsRegistrationGVK, "default", "instance-collision") + foreign.SetUID("foreign") + foreign.Object["spec"] = map[string]any{"name": "someone-else"} + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(foreign).Build() + owner := metav1.OwnerReference{APIVersion: computev1alpha.GroupVersion.String(), Kind: "Instance", Name: "api", UID: "instance-uid"} + + _, _, err := ensureDNSObject(context.Background(), InternalDNSProjectAccess{Reader: cl, Writer: cl}, dnsRegistrationGVK, "default", foreign.GetName(), map[string]string{ + internalDNSManagedBy: internalDNSManager, internalDNSInstanceUID: "instance-uid", + }, owner, map[string]any{"name": "api"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "refusing to adopt foreign") +} + +func TestEnsureDNSObjectCreatesProjectInstanceOwnershipWithoutDeleteBlock(t *testing.T) { + s := runtime.NewScheme() + s.AddKnownTypeWithName(dnsRegistrationGVK, &unstructured.Unstructured{}) + s.AddKnownTypeWithName(dnsRegistrationGVK.GroupVersion().WithKind("DNSRegistrationList"), &unstructured.UnstructuredList{}) + cl := fake.NewClientBuilder().WithScheme(s).Build() + owner := metav1.OwnerReference{APIVersion: computev1alpha.GroupVersion.String(), Kind: "Instance", Name: "api", UID: "project-instance-uid", Controller: ptrBool(true)} + + created, changed, err := ensureDNSObject(context.Background(), InternalDNSProjectAccess{Reader: cl, Writer: cl}, dnsRegistrationGVK, "default", "instance-owned", map[string]string{ + internalDNSManagedBy: internalDNSManager, internalDNSInstanceUID: "project-instance-uid", + }, owner, map[string]any{"name": "api"}) + require.NoError(t, err) + assert.True(t, changed) + require.Len(t, created.GetOwnerReferences(), 1) + assert.Equal(t, types.UID("project-instance-uid"), created.GetOwnerReferences()[0].UID) + assert.Nil(t, created.GetOwnerReferences()[0].BlockOwnerDeletion, + "publisher must not require delete permission on the owning Instance") +} diff --git a/internal/features/features.go b/internal/features/features.go index a93573ec..fe058975 100644 --- a/internal/features/features.go +++ b/internal/features/features.go @@ -108,6 +108,15 @@ const ( // // alpha: v0.1 InstanceConsoleSessions featuregate.Feature = "InstanceConsoleSessions" + + // InternalDNSPublishing enables the instance identity publisher. The + // publisher reads DNS-owned managed namespace bindings and writes + // registrations, grants, and leased contributions to project control planes. + // It is deliberately off by default: an installation with the gate disabled + // neither discovers DNS API types nor needs DNS RBAC. + // + // alpha: v0.1 + InternalDNSPublishing featuregate.Feature = "InternalDNSPublishing" ) // MutableFeatureGate is the mutable feature gate for the compute operator. @@ -127,6 +136,7 @@ func init() { RuntimeClasses: {Default: false, PreRelease: featuregate.Alpha}, InstanceTypes: {Default: false, PreRelease: featuregate.Alpha}, InstanceConsoleSessions: {Default: false, PreRelease: featuregate.Alpha}, + InternalDNSPublishing: {Default: false, PreRelease: featuregate.Alpha}, }); err != nil { panic(err) } diff --git a/internal/features/features_test.go b/internal/features/features_test.go index c1aa5076..b8caa131 100644 --- a/internal/features/features_test.go +++ b/internal/features/features_test.go @@ -135,3 +135,20 @@ func TestInstanceConsoleSessions_ExplicitlyEnabled(t *testing.T) { t.Error("InstanceConsoleSessions = false after Set=true, want true") } } + +func TestInternalDNSPublishing_DefaultDisabled(t *testing.T) { + gate := MutableFeatureGate.DeepCopy() + if gate.Enabled(InternalDNSPublishing) { + t.Error("InternalDNSPublishing default = true, want false") + } +} + +func TestInternalDNSPublishing_ExplicitlyEnabled(t *testing.T) { + gate := MutableFeatureGate.DeepCopy() + if err := gate.Set("InternalDNSPublishing=true"); err != nil { + t.Fatalf("Set(InternalDNSPublishing=true): %v", err) + } + if !gate.Enabled(InternalDNSPublishing) { + t.Error("InternalDNSPublishing = false after Set=true, want true") + } +} diff --git a/test/internaldns/README.md b/test/internaldns/README.md new file mode 100644 index 00000000..2a1877cd --- /dev/null +++ b/test/internaldns/README.md @@ -0,0 +1,38 @@ +# Internal DNS publisher smoke test + +This test runs the production Compute publisher against a disposable Kubernetes +API and the shared DNS serving fleet in the sibling `dns-operator` repository. +The DNS checkout must include the internal DNS implementation and its local test +harness. The test requires Go, Docker Compose, Kind, kubectl, OpenSSL, and an +ARM64 Docker runtime. + +```sh +go build -o /tmp/compute-dns-publisher ./test/internaldns/publisher +python3 test/internaldns/fleet_smoke.py \ + --dns-repo ../dns-operator \ + --publisher-binary /tmp/compute-dns-publisher +``` + +The runner starts only the production publisher controller. Its default feature +gate is off, and the test first checks that it exits without constructing a +Kubernetes client. The enabled phase uses a scoped Compute service account and +the real DNS admission, publication, transport, and serving components. + +The checks cover IPv4 and IPv6 over UDP and TCP, VPC isolation, application +readiness independence, address changes, allocation withdrawal and recovery, +publisher lease expiry and restart, and instance deletion with DNS resource +cleanup. Serving process identities must remain constant. + +The recorded run passed all 23 checks. Both VPCs served their own Compute +records. A query for the other VPC's managed suffix returned `SERVFAIL` with no +private answer because the fixture's public fallback is unavailable. The test +records the response code and checks isolation; it does not qualify public +recursion or require a specific negative response code. + +Instance objects and allocated-address observations are fixtures. This test does +not boot a guest, allocate addresses through NSO, validate Milo IAM deployment, +or test regional replication. The full Compute binary separately requires Milo +discovery and management controllers when the feature gate is enabled. + +The harness removes its own Kind cluster, containers, volumes, and temporary +files. Results and redacted evidence are saved under `test/internaldns/results`. diff --git a/test/internaldns/fleet_smoke.py b/test/internaldns/fleet_smoke.py new file mode 100644 index 00000000..65140d2f --- /dev/null +++ b/test/internaldns/fleet_smoke.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Run the production Compute publisher through the disposable DNS serving fleet.""" +import argparse +import hashlib +import importlib.util +import json +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[2] + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--dns-repo", type=Path, default=ROOT.parent / "dns-operator") + parser.add_argument("--publisher-binary", type=Path, help="Use a prebuilt test runner") + parser.add_argument("--results", type=Path, default=ROOT / "test/internaldns/results/fleet-smoke.json") + args = parser.parse_args() + dns = args.dns_repo.resolve() + spec = importlib.util.spec_from_file_location("dns_fleet", dns / "test/internaldns/run.py") + fleet = importlib.util.module_from_spec(spec) + spec.loader.exec_module(fleet) + h = fleet.Harness(False, args.results) + error = None + + def obj(kind, name): + return json.loads(h.kubectl("-n", fleet.PROJECT_NS, "get", kind, name, "-o", "json").stdout) + + def status(name, allocated=True, address="10.0.1.30", app_ready=False): + condition = lambda typ, value: {"type": typ, "status": str(value), "reason": "TestObservation", "message": "Smoke fixture", "lastTransitionTime": "2026-10-07T00:00:00Z"} + value = {"status": { + "conditions": [condition("Ready", "True" if app_ready else "False")], + "networkInterfaces": [{"name": "eth0", "addresses": [ + {"family": "IPv4", "address": address + "/32"}, + {"family": "IPv6", "address": "2001:db8::30/128"}, + ], "conditions": [condition("Allocated", "True" if allocated else "False"), condition("Programmed", "True")] }], + }} + h.kubectl("-n", fleet.PROJECT_NS, "patch", "instance", name, "--subresource=status", "--type=merge", "-p", json.dumps(value)) + + def query(server, name, rrtype="A", tcp=False): + output = h.compose("exec", "-T", "probe", "dig", "@" + server, name, rrtype, "+noall", "+comments", "+answer", *(["+tcp"] if tcp else [])).stdout + code = next((line.split("status:", 1)[1].split(",", 1)[0].strip() for line in output.splitlines() if "status:" in line), "UNKNOWN") + answers = [line.split()[-1] for line in output.splitlines() if not line.startswith(";") and len(line.split()) >= 5 and line.split()[-2] == rrtype] + return code, answers + + def expect(label, server, name, answers, rrtype="A", tcp=False, absent=False): + observed = {} + def matches(): + code, found = query(server, name, rrtype, tcp) + observed["status"] = code + return (code in ("NOERROR", "NXDOMAIN", "REFUSED", "SERVFAIL") if absent else code == "NOERROR") and found == answers + h.wait(label, matches, timeout=75, interval=0.5) + h.pass_check(label, name=name, recordType=rrtype, answers=answers, tcp=tcp, **observed) + + try: + binary = args.publisher_binary.resolve() if args.publisher_binary else h.work / "compute-dns-publisher" + if not args.publisher_binary: + h.run("go", "build", "-o", str(binary), "./test/internaldns/publisher", cwd=ROOT) + disabled = h.run(str(binary), "--kubeconfig", "/does/not/exist") + if "disabled" not in disabled.stdout: + raise AssertionError("default-off runner tried to start") + h.pass_check("default-off publisher starts without a Kubernetes/DNS configuration") + h.setup() + h.kubectl("apply", "-f", str(ROOT / "config/base/crd/bases/compute.datumapis.com_instances.yaml")) + role = obj("role", "compute-dns-publisher") + role["rules"].extend([ + {"apiGroups": ["compute.datumapis.com"], "resources": ["instances"], "verbs": ["get", "list", "watch"]}, + {"apiGroups": ["networking.datumapis.com"], "resources": ["networks"], "verbs": ["get"]}, + ]) + h.kubectl("replace", "-f", "-", stdin=json.dumps(role)) + configs = h.write_configs() + h.start("control", *h.command("control-plane", configs["control"])) + h.install_webhook() + h.kubectl("apply", "-f", str(dns / "test/internaldns/fixtures/vpc-a.yaml")) + h.create_association("prod-a-vpc-a", "prod-a", "vpc-a") + h.publish("prod-a", "bootstrap", "bootstrap", "10.0.1.1") + bindings = h.sync_addresses(1) + a_uid = obj("network", "vpc-a")["metadata"]["uid"] + vip_a = bindings[a_uid]["spec"]["consumerAddress"] + h.wait_for_bootstrap() + for member in ("node-0", "cluster-0"): + h.start_container(member, configs[member], network="datum-internal-dns_dns") + for member in ("pdns-0", "pdns-1"): + h.start_container(member, configs[member], network="container:datum-internal-dns-cluster-bind-1") + h.wait_for_agent_leases(("node-0", "cluster-0", "pdns-0", "pdns-1")) + for member in ("node-0", "cluster-0", "pdns-0", "pdns-1"): + h.start_container(member + "-watchdog", configs[member + "-watchdog"], network="none", watchdog=True) + h.kubectl("apply", "-f", str(dns / "test/internaldns/fixtures/vpc-b.yaml")) + h.create_association("prod-b-vpc-b", "prod-b", "vpc-b") + bindings = h.sync_addresses(2) + b_uid = obj("network", "vpc-b")["metadata"]["uid"] + vip_b = bindings[b_uid]["spec"]["consumerAddress"] + process_ids = h.container_ids() + h.start("compute", str(binary), "--feature-gates=InternalDNSPublishing=true", "--kubeconfig", str(h.compute_kubeconfig), "--lease=60s") + instance = {"apiVersion": "compute.datumapis.com/v1alpha", "kind": "Instance", "metadata": {"name": "web-01", "namespace": fleet.PROJECT_NS}, "spec": { + "runtime": {"resources": {"instanceType": "datumcloud/d1-standard-2"}, "sandbox": {"containers": [{"name": "web", "image": "docker.io/library/nginx:stable"}]}}, + "networkInterfaces": [{"name": "eth0", "network": {"name": "vpc-a"}}], + }} + h.kubectl("apply", "-f", "-", stdin=json.dumps(instance)) + status("web-01") + + def publication(instance_name): + uid = obj("instance", instance_name)["metadata"]["uid"] + registrations = json.loads(h.kubectl("-n", fleet.PROJECT_NS, "get", "dnsregistrations", "-l", "internal-dns.compute.datumapis.com/instance-uid=" + uid, "-o", "json").stdout)["items"] + if not registrations: + return None + reg = registrations[0] + zone = obj("dnszone", reg["spec"]["dnsZoneRef"]["name"]) + return reg, reg["spec"]["name"] + "." + zone["spec"]["domainName"].rstrip(".") + reg, fqdn = h.wait("production Compute registration", lambda: publication("web-01")) + instance["metadata"]["name"] = "web-02" + instance["spec"]["networkInterfaces"][0]["network"]["name"] = "vpc-b" + h.kubectl("apply", "-f", "-", stdin=json.dumps(instance)) + status("web-02", address="10.0.1.20") + _, fqdn_b = h.wait("second VPC Compute registration", lambda: publication("web-02")) + for tcp in (False, True): + expect("second VPC serves its own Compute IPv4" + (" TCP" if tcp else " UDP"), vip_b, fqdn_b, ["10.0.1.20"], tcp=tcp) + expect("second VPC serves its own Compute IPv6" + (" TCP" if tcp else " UDP"), vip_b, fqdn_b, ["2001:db8::30"], rrtype="AAAA", tcp=tcp) + for tcp in (False, True): + expect("production Compute IPv4" + (" TCP" if tcp else " UDP"), vip_a, fqdn, ["10.0.1.30"], tcp=tcp) + expect("production Compute IPv6" + (" TCP" if tcp else " UDP"), vip_a, fqdn, ["2001:db8::30"], rrtype="AAAA", tcp=tcp) + expect("other VPC cannot resolve instance" + (" TCP" if tcp else " UDP"), vip_b, fqdn, [], tcp=tcp, absent=True) + h.pass_check("application Ready=False leaves instance identity published") + status("web-01", address="10.0.1.31", app_ready=True) + expect("address update reaches DNS", vip_a, fqdn, ["10.0.1.31"]) + status("web-01", allocated=False) + expect("interface deallocation withdraws IPv4", vip_a, fqdn, [], absent=True) + expect("interface deallocation withdraws IPv6", vip_a, fqdn, [], rrtype="AAAA", absent=True) + expect("other VPC retains its Compute record during withdrawal", vip_b, fqdn_b, ["10.0.1.20"]) + status("web-01") + expect("interface recovery republishes", vip_a, fqdn, ["10.0.1.30"]) + h.stop("compute") + expect("stopped publisher lease expires", vip_a, fqdn, [], absent=True) + h.start("compute", str(binary), "--feature-gates=InternalDNSPublishing=true", "--kubeconfig", str(h.compute_kubeconfig), "--lease=60s") + expect("restarted publisher renews observation", vip_a, fqdn, ["10.0.1.30"]) + h.kubectl("-n", fleet.PROJECT_NS, "delete", "instance", "web-01", "--wait=true") + expect("instance deletion withdraws answer", vip_a, fqdn, [], absent=True) + expect("other VPC retains its Compute record during deletion", vip_b, fqdn_b, ["10.0.1.20"]) + h.kubectl("-n", fleet.PROJECT_NS, "delete", "instance", "web-02", "--wait=true") + h.wait("instance DNS objects garbage collected", lambda: all(not json.loads(h.kubectl("-n", fleet.PROJECT_NS, "get", kind, "-l", "internal-dns.compute.datumapis.com/managed-by=compute-instance-publisher", "-o", "json").stdout)["items"] for kind in ("dnsregistrations", "dnscontributiongrants", "dnsrecordcontributions"))) + h.pass_check("instance deletion cleans registration, grant and contribution") + if process_ids != h.container_ids(): + raise AssertionError("serving process count/identity changed") + h.pass_check("shared serving process identities remain constant") + except BaseException as exc: + error = exc + print("FAIL", exc, file=sys.stderr) + for name in ("compute", "control"): + log = h.logs / (name + ".log") + if log.exists(): + print(name + ": " + log.read_text()[-6000:], file=sys.stderr) + finally: + h.write_results(error) + data = json.loads(args.results.read_text()) + data["suite"] = "compute-internal-dns-fleet-smoke" + data["computePublisherSourceSHA256"] = hashlib.sha256((ROOT / "internal/controller/internaldns_publisher.go").read_bytes()).hexdigest() + if binary.is_file(): + data["computePublisherBinarySHA256"] = hashlib.sha256(binary.read_bytes()).hexdigest() + data["limitations"] = ["Instance addressing/status are fixtures; no guest boots or real NSO allocation.", "One local source API and one local broker; no regional HA or Milo IAM deployment validation.", "Uses the production Compute publisher controller, started by a test runner; default-off production wiring is covered separately."] + args.results.write_text(json.dumps(data, indent=2) + "\n") + h.cleanup() + if error: + raise error + print(f"PASS {len(fleet.RESULTS)} full-path checks") + + +if __name__ == "__main__": + main() diff --git a/test/internaldns/publisher/main.go b/test/internaldns/publisher/main.go new file mode 100644 index 00000000..ab5074e3 --- /dev/null +++ b/test/internaldns/publisher/main.go @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +// This test runner starts the production publisher against a disposable project +// API without starting unrelated Compute controllers. It is not a product entrypoint. +package main + +import ( + "context" + "flag" + "fmt" + "os" + "time" + + computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/controller" + "go.datum.net/compute/internal/features" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/clientcmd" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" + "sigs.k8s.io/multicluster-runtime/pkg/multicluster" + mcsingle "sigs.k8s.io/multicluster-runtime/providers/single" +) + +func main() { + var kubeconfig, namespace, projectUID, sourceUID, subject, gates string + var lease time.Duration + flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError) + flags.StringVar(&kubeconfig, "kubeconfig", "", "Disposable project kubeconfig") + flags.StringVar(&namespace, "namespace", "project-e2e", "Project namespace") + flags.StringVar(&projectUID, "project-uid", "project-e2e-uid", "DNS project UID") + flags.StringVar(&sourceUID, "source-uid", "internal-dns-e2e-cluster", "DNS trusted source UID") + flags.StringVar(&subject, "subject", "system:serviceaccount:compute-system:dns-publisher", "Authenticated writer") + flags.StringVar(&gates, "feature-gates", "", "Compute feature gates") + flags.DurationVar(&lease, "lease", 60*time.Second, "Contribution lease") + flags.Parse(os.Args[1:]) + ctrl.SetLogger(zap.New(zap.UseDevMode(true))) + if err := features.MutableFeatureGate.Set(gates); err != nil { + panic(err) + } + if !features.FeatureGate.Enabled(features.InternalDNSPublishing) { + fmt.Println("InternalDNSPublishing is disabled; no DNS client or controller started") + return + } + cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + panic(err) + } + cfg.Timeout = 10 * time.Second + scheme := runtime.NewScheme() + for _, add := range []func(*runtime.Scheme) error{corev1.AddToScheme, computev1alpha.AddToScheme, networkingv1alpha.AddToScheme} { + if err := add(scheme); err != nil { + panic(err) + } + } + cl, err := cluster.New(cfg, func(o *cluster.Options) { + o.Scheme = scheme + o.Cache.DefaultNamespaces = map[string]cache.Config{namespace: {}} + }) + if err != nil { + panic(err) + } + mgr, err := mcmanager.New(cfg, mcsingle.New(multicluster.ClusterName("single"), cl), ctrl.Options{ + Scheme: scheme, Metrics: metricsserver.Options{BindAddress: "0"}, HealthProbeBindAddress: "0", + Cache: cache.Options{DefaultNamespaces: map[string]cache.Config{namespace: {}}}, + }) + if err != nil { + panic(err) + } + publisher := &controller.InternalDNSPublisherReconciler{ + ProjectIdentities: map[string]controller.InternalDNSProjectIdentity{"single": { + ProjectName: "single", ProjectUID: types.UID(projectUID), SourceClusterUID: sourceUID, + }}, PrincipalSubject: subject, LeaseDuration: lease, + } + if err := publisher.SetupWithManager(mgr); err != nil { + panic(err) + } + ctx, cancel := context.WithCancel(ctrl.SetupSignalHandler()) + defer cancel() + errs := make(chan error, 2) + go func() { errs <- cl.Start(ctx) }() + go func() { errs <- mgr.Start(ctx) }() + if err := <-errs; err != nil && ctx.Err() == nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/test/internaldns/results/fleet-smoke.json b/test/internaldns/results/fleet-smoke.json new file mode 100644 index 00000000..888c69a5 --- /dev/null +++ b/test/internaldns/results/fleet-smoke.json @@ -0,0 +1,235 @@ +{ + "passed": true, + "error": null, + "checks": [ + { + "label": "default-off publisher starts without a Kubernetes/DNS configuration", + "passed": true + }, + { + "label": "second VPC serves its own Compute IPv4 UDP", + "passed": true, + "name": "web-02-1333f06e881549a7.instances.53992e2b22a3a5f1bb360315.managed.internal", + "recordType": "A", + "answers": [ + "10.0.1.20" + ], + "tcp": false, + "status": "NOERROR" + }, + { + "label": "second VPC serves its own Compute IPv6 UDP", + "passed": true, + "name": "web-02-1333f06e881549a7.instances.53992e2b22a3a5f1bb360315.managed.internal", + "recordType": "AAAA", + "answers": [ + "2001:db8::30" + ], + "tcp": false, + "status": "NOERROR" + }, + { + "label": "second VPC serves its own Compute IPv4 TCP", + "passed": true, + "name": "web-02-1333f06e881549a7.instances.53992e2b22a3a5f1bb360315.managed.internal", + "recordType": "A", + "answers": [ + "10.0.1.20" + ], + "tcp": true, + "status": "NOERROR" + }, + { + "label": "second VPC serves its own Compute IPv6 TCP", + "passed": true, + "name": "web-02-1333f06e881549a7.instances.53992e2b22a3a5f1bb360315.managed.internal", + "recordType": "AAAA", + "answers": [ + "2001:db8::30" + ], + "tcp": true, + "status": "NOERROR" + }, + { + "label": "production Compute IPv4 UDP", + "passed": true, + "name": "web-01-7411484c38c1f2a7.instances.56a4888365a3749b85e6f47c.managed.internal", + "recordType": "A", + "answers": [ + "10.0.1.30" + ], + "tcp": false, + "status": "NOERROR" + }, + { + "label": "production Compute IPv6 UDP", + "passed": true, + "name": "web-01-7411484c38c1f2a7.instances.56a4888365a3749b85e6f47c.managed.internal", + "recordType": "AAAA", + "answers": [ + "2001:db8::30" + ], + "tcp": false, + "status": "NOERROR" + }, + { + "label": "other VPC cannot resolve instance UDP", + "passed": true, + "name": "web-01-7411484c38c1f2a7.instances.56a4888365a3749b85e6f47c.managed.internal", + "recordType": "A", + "answers": [], + "tcp": false, + "status": "SERVFAIL" + }, + { + "label": "production Compute IPv4 TCP", + "passed": true, + "name": "web-01-7411484c38c1f2a7.instances.56a4888365a3749b85e6f47c.managed.internal", + "recordType": "A", + "answers": [ + "10.0.1.30" + ], + "tcp": true, + "status": "NOERROR" + }, + { + "label": "production Compute IPv6 TCP", + "passed": true, + "name": "web-01-7411484c38c1f2a7.instances.56a4888365a3749b85e6f47c.managed.internal", + "recordType": "AAAA", + "answers": [ + "2001:db8::30" + ], + "tcp": true, + "status": "NOERROR" + }, + { + "label": "other VPC cannot resolve instance TCP", + "passed": true, + "name": "web-01-7411484c38c1f2a7.instances.56a4888365a3749b85e6f47c.managed.internal", + "recordType": "A", + "answers": [], + "tcp": true, + "status": "SERVFAIL" + }, + { + "label": "application Ready=False leaves instance identity published", + "passed": true + }, + { + "label": "address update reaches DNS", + "passed": true, + "name": "web-01-7411484c38c1f2a7.instances.56a4888365a3749b85e6f47c.managed.internal", + "recordType": "A", + "answers": [ + "10.0.1.31" + ], + "tcp": false, + "status": "NOERROR" + }, + { + "label": "interface deallocation withdraws IPv4", + "passed": true, + "name": "web-01-7411484c38c1f2a7.instances.56a4888365a3749b85e6f47c.managed.internal", + "recordType": "A", + "answers": [], + "tcp": false, + "status": "NOERROR" + }, + { + "label": "interface deallocation withdraws IPv6", + "passed": true, + "name": "web-01-7411484c38c1f2a7.instances.56a4888365a3749b85e6f47c.managed.internal", + "recordType": "AAAA", + "answers": [], + "tcp": false, + "status": "NOERROR" + }, + { + "label": "other VPC retains its Compute record during withdrawal", + "passed": true, + "name": "web-02-1333f06e881549a7.instances.53992e2b22a3a5f1bb360315.managed.internal", + "recordType": "A", + "answers": [ + "10.0.1.20" + ], + "tcp": false, + "status": "NOERROR" + }, + { + "label": "interface recovery republishes", + "passed": true, + "name": "web-01-7411484c38c1f2a7.instances.56a4888365a3749b85e6f47c.managed.internal", + "recordType": "A", + "answers": [ + "10.0.1.30" + ], + "tcp": false, + "status": "NOERROR" + }, + { + "label": "stopped publisher lease expires", + "passed": true, + "name": "web-01-7411484c38c1f2a7.instances.56a4888365a3749b85e6f47c.managed.internal", + "recordType": "A", + "answers": [], + "tcp": false, + "status": "NOERROR" + }, + { + "label": "restarted publisher renews observation", + "passed": true, + "name": "web-01-7411484c38c1f2a7.instances.56a4888365a3749b85e6f47c.managed.internal", + "recordType": "A", + "answers": [ + "10.0.1.30" + ], + "tcp": false, + "status": "NOERROR" + }, + { + "label": "instance deletion withdraws answer", + "passed": true, + "name": "web-01-7411484c38c1f2a7.instances.56a4888365a3749b85e6f47c.managed.internal", + "recordType": "A", + "answers": [], + "tcp": false, + "status": "NXDOMAIN" + }, + { + "label": "other VPC retains its Compute record during deletion", + "passed": true, + "name": "web-02-1333f06e881549a7.instances.53992e2b22a3a5f1bb360315.managed.internal", + "recordType": "A", + "answers": [ + "10.0.1.20" + ], + "tcp": false, + "status": "NOERROR" + }, + { + "label": "instance deletion cleans registration, grant and contribution", + "passed": true + }, + { + "label": "shared serving process identities remain constant", + "passed": true + } + ], + "sourceSHA256": "46a42735831e53140feaba8b891aee6702329fe0ecf920872e1bdad694d5d6ab", + "images": { + "dnsdist": "powerdns/dnsdist-20@sha256:4ed9af56729c7021795dc478421237ab9cf6c7ceec8734677405e7617f06d52f", + "bind": "internetsystemsconsortium/bind9@sha256:071465f88068854d0ceadd9b985fb1acae4071e80754e545cd811fde57541ad0", + "powerdns": "powerdns/pdns-auth-51@sha256:bb5b1c133bcca1dd455075321de7d55db4945a8d7f2ba23339e3c7bbe416b205" + }, + "suite": "compute-internal-dns-fleet-smoke", + "computePublisherSourceSHA256": "47e2cff09e92c46e53a9664e3f4b226002a87961c0c9ce24f09b47474f508bbf", + "computePublisherBinarySHA256": "073c047232db7f5f022e569c14f73c7bee0b90b677f178948855e8a58831e455", + "limitations": [ + "Instance addressing/status are fixtures; no guest boots or real NSO allocation.", + "One local source API and one local broker; no regional HA or Milo IAM deployment validation.", + "Uses the production Compute publisher controller, started by a test runner; default-off production wiring is covered separately.", + "Unassociated managed suffixes return SERVFAIL with no private answers in this fixture; public recursion is not qualified." + ], + "buildBaseCommit": "cd312926e3370288bac62ba642cceb9fc15cfb40" +} From e957824b57c6ba1ba3e06b9ee800ceaf8d2a0f0e Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 7 Oct 2026 15:27:39 -0500 Subject: [PATCH 02/33] fix: Resolve internal DNS publisher lint findings --- cmd/main.go | 10 +- cmd/main_test.go | 8 +- internal/controller/internaldns_publisher.go | 90 +++++++++--------- .../controller/internaldns_publisher_test.go | 94 ++++++++----------- test/internaldns/publisher/main.go | 4 +- 5 files changed, 104 insertions(+), 102 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index cc12fce2..0eeaa829 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -450,7 +450,8 @@ func main() { // not registered at all while its default-off feature gate is disabled. if features.FeatureGate.Enabled(features.InternalDNSPublishing) { if serverConfig.Discovery.Mode != multiclusterproviders.ProviderMilo || !enableManagementControllers { - setupLog.Error(nil, "InternalDNSPublishing requires Milo discovery mode and management controllers so leases are renewed only from the authoritative project Instance") + setupLog.Error(nil, + "InternalDNSPublishing requires Milo discovery mode and management controllers") os.Exit(1) } identities, identityErr := internalDNSProjectIdentities(serverConfig.InternalDNS) @@ -967,7 +968,9 @@ func computeWatchProviderClaims(mode multiclusterproviders.Provider) bool { return mode == multiclusterproviders.ProviderMilo } -func internalDNSProjectIdentities(cfg config.InternalDNSConfig) (map[string]controller.InternalDNSProjectIdentity, error) { +func internalDNSProjectIdentities( + cfg config.InternalDNSConfig, +) (map[string]controller.InternalDNSProjectIdentity, error) { if cfg.PrincipalSubject == "" { return nil, errors.New("internalDNS.principalSubject is required") } @@ -989,7 +992,8 @@ func internalDNSProjectIdentities(cfg config.InternalDNSConfig) (map[string]cont return nil, errors.New("internalDNS.projects must contain at least one project") } if cfg.LeaseDuration.Duration > 90*time.Second { - return nil, fmt.Errorf("internalDNS.leaseDuration %s exceeds the DNS platform maximum 90s", cfg.LeaseDuration.Duration) + return nil, fmt.Errorf( + "internalDNS.leaseDuration %s exceeds the DNS platform maximum 90s", cfg.LeaseDuration.Duration) } return identities, nil } diff --git a/cmd/main_test.go b/cmd/main_test.go index 52d3eb45..c42cff40 100644 --- a/cmd/main_test.go +++ b/cmd/main_test.go @@ -29,7 +29,9 @@ func TestInternalDNSProjectIdentities(t *testing.T) { identities, err := internalDNSProjectIdentities(config.InternalDNSConfig{ PrincipalSubject: "system:serviceaccount:compute-system:compute-manager", LeaseDuration: metav1.Duration{Duration: 60 * time.Second}, - Projects: []config.InternalDNSProjectIdentity{{Name: "project-a", ProjectUID: "project-uid", SourceClusterUID: "source-uid"}}, + Projects: []config.InternalDNSProjectIdentity{{ + Name: "project-a", ProjectUID: "project-uid", SourceClusterUID: "source-uid", + }}, }) require.NoError(t, err) assert.Equal(t, "source-uid", identities["project-a"].SourceClusterUID) @@ -39,7 +41,9 @@ func TestInternalDNSProjectIdentities(t *testing.T) { _, err = internalDNSProjectIdentities(config.InternalDNSConfig{ PrincipalSubject: "subject", LeaseDuration: metav1.Duration{Duration: 91 * time.Second}, - Projects: []config.InternalDNSProjectIdentity{{Name: "project-a", ProjectUID: "project-uid", SourceClusterUID: "source-uid"}}, + Projects: []config.InternalDNSProjectIdentity{{ + Name: "project-a", ProjectUID: "project-uid", SourceClusterUID: "source-uid", + }}, }) assert.Error(t, err, "a producer must not configure a lease DNS admission rejects") } diff --git a/internal/controller/internaldns_publisher.go b/internal/controller/internaldns_publisher.go index 98726a82..39259ef0 100644 --- a/internal/controller/internaldns_publisher.go +++ b/internal/controller/internaldns_publisher.go @@ -34,17 +34,23 @@ import ( ) const ( - internalDNSGroup = "dns.networking.miloapis.com" - internalDNSVersion = "v1alpha1" - internalDNSInstanceUID = "internal-dns.compute.datumapis.com/instance-uid" - internalDNSVPCUID = "internal-dns.compute.datumapis.com/vpc-uid" - internalDNSManagedBy = "internal-dns.compute.datumapis.com/managed-by" - internalDNSManager = "compute-instance-publisher" - defaultDNSLease = 60 * time.Second - maxDNSLease = 90 * time.Second - internalDNSPendingWait = 10 * time.Second - internalDNSAPITimeout = 10 * time.Second - defaultDNSConcurrency = 4 + internalDNSInstanceKind = "Instance" + internalDNSRecordTypeAAAA = "AAAA" + internalDNSNameField = "name" + internalDNSStatusField = "status" + internalDNSReasonField = "reason" + internalDNSUIDField = "uid" + internalDNSGroup = "dns.networking.miloapis.com" + internalDNSVersion = "v1alpha1" + internalDNSInstanceUID = "internal-dns.compute.datumapis.com/instance-uid" + internalDNSVPCUID = "internal-dns.compute.datumapis.com/vpc-uid" + internalDNSManagedBy = "internal-dns.compute.datumapis.com/managed-by" + internalDNSManager = "compute-instance-publisher" + defaultDNSLease = 60 * time.Second + maxDNSLease = 90 * time.Second + internalDNSPendingWait = 10 * time.Second + internalDNSAPITimeout = 10 * time.Second + defaultDNSConcurrency = 4 ) var ( @@ -146,7 +152,7 @@ func (r *InternalDNSPublisherReconciler) Reconcile(ctx context.Context, req mcre } owner := metav1.OwnerReference{ APIVersion: computev1alpha.GroupVersion.String(), - Kind: "Instance", + Kind: internalDNSInstanceKind, Name: projectInstance.Name, UID: projectInstance.UID, Controller: ptrBool(true), @@ -222,11 +228,11 @@ func instanceDNSAttachments(ctx context.Context, reader client.Reader, projectNa } rrtype, field := "A", "a" if addr.Is6() { - rrtype, field = "AAAA", "aaaa" + rrtype, field = internalDNSRecordTypeAAAA, "aaaa" } attachment.RecordSets = append(attachment.RecordSets, map[string]any{ "recordType": rrtype, - "records": []any{map[string]any{"name": "", field: map[string]any{"content": addr.String()}}}, + "records": []any{map[string]any{internalDNSNameField: "", field: map[string]any{"content": addr.String()}}}, }) } } @@ -247,7 +253,7 @@ func internalDNSInterfaceName(nic computev1alpha.InstanceNetworkInterface) strin if nic.Name != "" { return nic.Name } - return "eth0" + return defaultInterfaceName } func interfaceDNSPublishable(status computev1alpha.InstanceNetworkInterfaceStatus) bool { @@ -275,8 +281,8 @@ func (r *InternalDNSPublisherReconciler) reconcileAttachment(ctx context.Context if managed == nil { return false, nil } - zoneRef, _, _ := unstructured.NestedMap(managed.Object, "status", "dnsZoneRef") - if stringValue(zoneRef, "name") == "" || stringValue(zoneRef, "uid") == "" { + zoneRef, _, _ := unstructured.NestedMap(managed.Object, internalDNSStatusField, "dnsZoneRef") + if stringValue(zoneRef, internalDNSNameField) == "" || stringValue(zoneRef, internalDNSUIDField) == "" { return false, nil } @@ -290,16 +296,16 @@ func (r *InternalDNSPublisherReconciler) reconcileAttachment(ctx context.Context rrset := attachment.RecordSets[i].(map[string]any) records := rrset["records"].([]any) for _, record := range records { - record.(map[string]any)["name"] = ownerName + record.(map[string]any)[internalDNSNameField] = ownerName } } registrationSpec := map[string]any{ - "dnsZoneRef": zoneRef, - "name": ownerName, - "recordTypes": []any{"A", "AAAA"}, - "publicationPolicy": "EligibleContributions", - "ttlSeconds": int64(30), + "dnsZoneRef": zoneRef, + internalDNSNameField: ownerName, + "recordTypes": []any{"A", internalDNSRecordTypeAAAA}, + "publicationPolicy": "EligibleContributions", + "ttlSeconds": int64(30), } registration, changed, err := ensureDNSObject(ctx, access, dnsRegistrationGVK, namespace, resourceName, labels, owner, registrationSpec) if err != nil || changed { @@ -314,15 +320,15 @@ func (r *InternalDNSPublisherReconciler) reconcileAttachment(ctx context.Context "clusterUID": identity.SourceClusterUID, "subject": r.PrincipalSubject, }, - "recordTypes": []any{"A", "AAAA"}, + "recordTypes": []any{"A", internalDNSRecordTypeAAAA}, } grant, changed, err := ensureDNSObject(ctx, access, dnsGrantGVK, namespace, resourceName, labels, owner, grantSpec) if err != nil || changed { return false, err } - writerEpoch, _, _ := unstructured.NestedInt64(grant.Object, "status", "activeWriterEpoch") - observedGrant, _, _ := unstructured.NestedInt64(grant.Object, "status", "observedGrantGeneration") - observedRegistration, _, _ := unstructured.NestedInt64(grant.Object, "status", "observedRegistrationGeneration") + writerEpoch, _, _ := unstructured.NestedInt64(grant.Object, internalDNSStatusField, "activeWriterEpoch") + observedGrant, _, _ := unstructured.NestedInt64(grant.Object, internalDNSStatusField, "observedGrantGeneration") + observedRegistration, _, _ := unstructured.NestedInt64(grant.Object, internalDNSStatusField, "observedRegistrationGeneration") if writerEpoch < 1 || observedGrant != grant.GetGeneration() || observedRegistration != registration.GetGeneration() { return false, nil } @@ -336,7 +342,7 @@ func (r *InternalDNSPublisherReconciler) reconcileAttachment(ctx context.Context if err != nil || changed { return false, err } - boundEpoch, _, _ := unstructured.NestedInt64(contribution.Object, "status", "writerEpoch") + boundEpoch, _, _ := unstructured.NestedInt64(contribution.Object, internalDNSStatusField, "writerEpoch") if boundEpoch != writerEpoch { return false, nil } @@ -349,26 +355,26 @@ func (r *InternalDNSPublisherReconciler) refreshObservation(ctx context.Context, if r.Now != nil { now = r.Now() } - oldEligible, _, _ := unstructured.NestedBool(contribution.Object, "status", "eligible") - observedGeneration, _, _ := unstructured.NestedInt64(contribution.Object, "status", "observedGeneration") - validUntilRaw, _, _ := unstructured.NestedString(contribution.Object, "status", "validUntil") + oldEligible, _, _ := unstructured.NestedBool(contribution.Object, internalDNSStatusField, "eligible") + observedGeneration, _, _ := unstructured.NestedInt64(contribution.Object, internalDNSStatusField, "observedGeneration") + validUntilRaw, _, _ := unstructured.NestedString(contribution.Object, internalDNSStatusField, "validUntil") validUntil, _ := time.Parse(time.RFC3339Nano, validUntilRaw) if oldEligible == eligible && observedGeneration == contribution.GetGeneration() && validUntil.After(now.Add(r.leaseDuration()/2)) { return true, nil } base := contribution.DeepCopy() - sequence, _, _ := unstructured.NestedInt64(contribution.Object, "status", "sequence") + sequence, _, _ := unstructured.NestedInt64(contribution.Object, internalDNSStatusField, "sequence") owned := []struct { path []string value any }{ - {[]string{"status", "observedGeneration"}, contribution.GetGeneration()}, - {[]string{"status", "writerEpoch"}, writerEpoch}, - {[]string{"status", "sequence"}, sequence + 1}, - {[]string{"status", "eligible"}, eligible}, - {[]string{"status", "reason"}, map[bool]string{true: "AddressAllocatedAndProgrammed", false: "AddressUnavailable"}[eligible]}, - {[]string{"status", "validUntil"}, now.Add(r.leaseDuration()).UTC().Format(time.RFC3339Nano)}, + {[]string{internalDNSStatusField, "observedGeneration"}, contribution.GetGeneration()}, + {[]string{internalDNSStatusField, "writerEpoch"}, writerEpoch}, + {[]string{internalDNSStatusField, "sequence"}, sequence + 1}, + {[]string{internalDNSStatusField, "eligible"}, eligible}, + {[]string{internalDNSStatusField, internalDNSReasonField}, map[bool]string{true: "AddressAllocatedAndProgrammed", false: "AddressUnavailable"}[eligible]}, + {[]string{internalDNSStatusField, "validUntil"}, now.Add(r.leaseDuration()).UTC().Format(time.RFC3339Nano)}, } for _, field := range owned { if err := unstructured.SetNestedField(contribution.Object, field.value, field.path...); err != nil { @@ -397,7 +403,7 @@ func findManagedNamespace(ctx context.Context, reader client.Reader, namespace s continue } puid, _, _ := unstructured.NestedString(item.Object, "spec", "projectUID") - vid, _, _ := unstructured.NestedString(item.Object, "spec", "vpcRef", "uid") + vid, _, _ := unstructured.NestedString(item.Object, "spec", "vpcRef", internalDNSUIDField) if puid != string(projectUID) || vid != string(vpcUID) || !unstructuredConditionTrue(item, "Accepted") { continue } @@ -410,10 +416,10 @@ func findManagedNamespace(ctx context.Context, reader client.Reader, namespace s } func unstructuredConditionTrue(obj *unstructured.Unstructured, conditionType string) bool { - conditions, _, _ := unstructured.NestedSlice(obj.Object, "status", "conditions") + conditions, _, _ := unstructured.NestedSlice(obj.Object, internalDNSStatusField, "conditions") for _, raw := range conditions { condition, ok := raw.(map[string]any) - if ok && condition["type"] == conditionType && condition["status"] == "True" { + if ok && condition["type"] == conditionType && condition[internalDNSStatusField] == string(metav1.ConditionTrue) { return true } } @@ -509,7 +515,7 @@ func newDNSObject(gvk schema.GroupVersionKind, namespace, name string) *unstruct } func objectReference(obj *unstructured.Unstructured, includeGeneration bool) map[string]any { - ref := map[string]any{"name": obj.GetName(), "uid": string(obj.GetUID())} + ref := map[string]any{internalDNSNameField: obj.GetName(), internalDNSUIDField: string(obj.GetUID())} if includeGeneration { ref["generation"] = obj.GetGeneration() } diff --git a/internal/controller/internaldns_publisher_test.go b/internal/controller/internaldns_publisher_test.go index a80efe77..7838bb4b 100644 --- a/internal/controller/internaldns_publisher_test.go +++ b/internal/controller/internaldns_publisher_test.go @@ -12,7 +12,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -21,41 +20,26 @@ import ( networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" ) -func internalDNSTestScheme(t *testing.T) *runtime.Scheme { - t.Helper() - s := runtime.NewScheme() - require.NoError(t, computev1alpha.AddToScheme(s)) - require.NoError(t, networkingv1alpha.AddToScheme(s)) - for _, gvk := range []struct { - kind schemaKind - }{ - {schemaKind{dnsManagedNamespaceGVK}}, - {schemaKind{dnsRegistrationGVK}}, - {schemaKind{dnsGrantGVK}}, - {schemaKind{dnsContributionGVK}}, - } { - s.AddKnownTypeWithName(gvk.kind.gvk, &unstructured.Unstructured{}) - s.AddKnownTypeWithName(gvk.kind.gvk.GroupVersion().WithKind(gvk.kind.gvk.Kind+"List"), &unstructured.UnstructuredList{}) - } - return s -} - -type schemaKind struct{ gvk schema.GroupVersionKind } +const ( + internalDNSTestNamespace = "dns-test" + internalDNSTestInstanceName = "api" + internalDNSTestInstanceUID = "dns-instance-uid" +) func TestInstanceDNSAttachmentsMapsTenancyAndAddressFamilies(t *testing.T) { scheme := runtime.NewScheme() require.NoError(t, networkingv1alpha.AddToScheme(scheme)) - network := &networkingv1alpha.Network{ObjectMeta: metav1.ObjectMeta{Name: "prod", Namespace: "default", UID: "vpc-uid"}} + network := &networkingv1alpha.Network{ObjectMeta: metav1.ObjectMeta{Name: "prod", Namespace: internalDNSTestNamespace, UID: "vpc-uid"}} cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(network).Build() instance := &computev1alpha.Instance{ - ObjectMeta: metav1.ObjectMeta{Name: "api", Namespace: "default", UID: "instance-uid"}, - Spec: computev1alpha.InstanceSpec{NetworkInterfaces: []computev1alpha.InstanceNetworkInterface{{Name: "eth0", Network: networkingv1alpha.NetworkRef{Name: "prod"}}}}, + ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestInstanceName, Namespace: internalDNSTestNamespace, UID: internalDNSTestInstanceUID}, + Spec: computev1alpha.InstanceSpec{NetworkInterfaces: []computev1alpha.InstanceNetworkInterface{{Name: defaultInterfaceName, Network: networkingv1alpha.NetworkRef{Name: "prod"}}}}, Status: computev1alpha.InstanceStatus{ // Application readiness is deliberately false. Instance identity is // governed by the interface allocation/programming lifecycle. Conditions: []metav1.Condition{{Type: computev1alpha.InstanceReady, Status: metav1.ConditionFalse}}, NetworkInterfaces: []computev1alpha.InstanceNetworkInterfaceStatus{{ - Name: "eth0", + Name: defaultInterfaceName, Addresses: []computev1alpha.InstanceNetworkInterfaceAddress{ {Address: "10.0.0.8/32"}, {Address: "2001:db8::8/128"}, @@ -69,7 +53,7 @@ func TestInstanceDNSAttachmentsMapsTenancyAndAddressFamilies(t *testing.T) { }, } - attachments, err := instanceDNSAttachments(context.Background(), cl, "default", instance) + attachments, err := instanceDNSAttachments(context.Background(), cl, internalDNSTestNamespace, instance) require.NoError(t, err) require.Len(t, attachments, 1) assert.Equal(t, types.UID("vpc-uid"), attachments[0].VPCUID) @@ -77,7 +61,7 @@ func TestInstanceDNSAttachmentsMapsTenancyAndAddressFamilies(t *testing.T) { assert.Len(t, attachments[0].RecordSets, 2, "A and AAAA host addresses should be retained") instance.Status.NetworkInterfaces[0].Conditions[1].Status = metav1.ConditionFalse - attachments, err = instanceDNSAttachments(context.Background(), cl, "default", instance) + attachments, err = instanceDNSAttachments(context.Background(), cl, internalDNSTestNamespace, instance) require.NoError(t, err) require.Len(t, attachments, 1) assert.False(t, attachments[0].Eligible) @@ -91,14 +75,16 @@ func TestFindManagedNamespaceRequiresProjectAndVPCUID(t *testing.T) { s.AddKnownTypeWithName(dnsManagedNamespaceGVK.GroupVersion().WithKind("DNSManagedNamespaceList"), &unstructured.UnstructuredList{}) wrong := managedNamespaceObject("wrong", "other-project", "vpc-a", true) right := managedNamespaceObject("right", "project-a", "vpc-a", true) - cl := fake.NewClientBuilder().WithScheme(s).WithObjects(wrong, right).Build() + wrongVPC := managedNamespaceObject("wrong-vpc", "project-a", "vpc-b", true) + pending := managedNamespaceObject("pending", "project-a", "vpc-a", false) + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(wrong, wrongVPC, pending, right).Build() - got, err := findManagedNamespace(context.Background(), cl, "default", "project-a", "vpc-a") + got, err := findManagedNamespace(context.Background(), cl, internalDNSTestNamespace, "project-a", "vpc-a") require.NoError(t, err) require.NotNil(t, got) assert.Equal(t, "right", got.GetName()) - got, err = findManagedNamespace(context.Background(), cl, "default", "project-b", "vpc-a") + got, err = findManagedNamespace(context.Background(), cl, internalDNSTestNamespace, "project-b", "vpc-a") require.NoError(t, err) assert.Nil(t, got) } @@ -111,7 +97,7 @@ func TestFindManagedNamespaceRejectsAmbiguityAndIgnoresTerminatingBinding(t *tes second := managedNamespaceObject("second", "project-a", "vpc-a", true) cl := fake.NewClientBuilder().WithScheme(s).WithObjects(first, second).Build() - _, err := findManagedNamespace(context.Background(), cl, "default", "project-a", "vpc-a") + _, err := findManagedNamespace(context.Background(), cl, internalDNSTestNamespace, "project-a", "vpc-a") require.Error(t, err) assert.Contains(t, err.Error(), "multiple accepted") @@ -119,22 +105,22 @@ func TestFindManagedNamespaceRejectsAmbiguityAndIgnoresTerminatingBinding(t *tes second.SetDeletionTimestamp(&now) second.SetFinalizers([]string{"test.example/finalizer"}) cl = fake.NewClientBuilder().WithScheme(s).WithObjects(first, second).Build() - got, err := findManagedNamespace(context.Background(), cl, "default", "project-a", "vpc-a") + got, err := findManagedNamespace(context.Background(), cl, internalDNSTestNamespace, "project-a", "vpc-a") require.NoError(t, err) require.NotNil(t, got) assert.Equal(t, "first", got.GetName()) } func managedNamespaceObject(name, projectUID, vpcUID string, accepted bool) *unstructured.Unstructured { - status := "False" + status := string(metav1.ConditionFalse) if accepted { - status = "True" + status = string(metav1.ConditionTrue) } - obj := newDNSObject(dnsManagedNamespaceGVK, "default", name) - obj.Object["spec"] = map[string]any{"projectUID": projectUID, "vpcRef": map[string]any{"uid": vpcUID}} - obj.Object["status"] = map[string]any{ - "conditions": []any{map[string]any{"type": "Accepted", "status": status}}, - "dnsZoneRef": map[string]any{"name": "zone", "uid": "zone-uid", "generation": int64(1)}, + obj := newDNSObject(dnsManagedNamespaceGVK, internalDNSTestNamespace, name) + obj.Object["spec"] = map[string]any{"projectUID": projectUID, "vpcRef": map[string]any{internalDNSUIDField: vpcUID}} + obj.Object[internalDNSStatusField] = map[string]any{ + "conditions": []any{map[string]any{"type": "Accepted", internalDNSStatusField: status}}, + "dnsZoneRef": map[string]any{internalDNSNameField: "zone", internalDNSUIDField: "zone-uid", "generation": int64(1)}, } return obj } @@ -143,17 +129,17 @@ func TestRefreshObservationPreservesPlatformStatusAndIncrementsFence(t *testing. s := runtime.NewScheme() s.AddKnownTypeWithName(dnsContributionGVK, &unstructured.Unstructured{}) s.AddKnownTypeWithName(dnsContributionGVK.GroupVersion().WithKind("DNSRecordContributionList"), &unstructured.UnstructuredList{}) - contribution := newDNSObject(dnsContributionGVK, "default", "endpoint") + contribution := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, "endpoint") contribution.SetUID("contribution-uid") contribution.SetGeneration(3) contribution.SetResourceVersion("1") contribution.Object["spec"] = map[string]any{"recordSets": []any{}} - contribution.Object["status"] = map[string]any{ + contribution.Object[internalDNSStatusField] = map[string]any{ "writerEpoch": int64(7), "sequence": int64(11), "eligible": false, "publishedRevision": int64(44), - "conditions": []any{map[string]any{"type": "Published", "status": "True"}}, + "conditions": []any{map[string]any{"type": "Published", internalDNSStatusField: string(metav1.ConditionTrue)}}, } cl := fake.NewClientBuilder().WithScheme(s).WithObjects(contribution).WithStatusSubresource(contribution).Build() now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) @@ -169,9 +155,9 @@ func TestRefreshObservationPreservesPlatformStatusAndIncrementsFence(t *testing. var got unstructured.Unstructured got.SetGroupVersionKind(dnsContributionGVK) require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(contribution), &got)) - sequence, _, _ := unstructured.NestedInt64(got.Object, "status", "sequence") - published, _, _ := unstructured.NestedInt64(got.Object, "status", "publishedRevision") - conditions, _, _ := unstructured.NestedSlice(got.Object, "status", "conditions") + sequence, _, _ := unstructured.NestedInt64(got.Object, internalDNSStatusField, "sequence") + published, _, _ := unstructured.NestedInt64(got.Object, internalDNSStatusField, "publishedRevision") + conditions, _, _ := unstructured.NestedSlice(got.Object, internalDNSStatusField, "conditions") assert.Equal(t, int64(12), sequence) assert.Equal(t, int64(44), published) assert.Len(t, conditions, 1) @@ -181,15 +167,15 @@ func TestEnsureDNSObjectRefusesForeignCollision(t *testing.T) { s := runtime.NewScheme() s.AddKnownTypeWithName(dnsRegistrationGVK, &unstructured.Unstructured{}) s.AddKnownTypeWithName(dnsRegistrationGVK.GroupVersion().WithKind("DNSRegistrationList"), &unstructured.UnstructuredList{}) - foreign := newDNSObject(dnsRegistrationGVK, "default", "instance-collision") + foreign := newDNSObject(dnsRegistrationGVK, internalDNSTestNamespace, "instance-collision") foreign.SetUID("foreign") - foreign.Object["spec"] = map[string]any{"name": "someone-else"} + foreign.Object["spec"] = map[string]any{internalDNSNameField: "someone-else"} cl := fake.NewClientBuilder().WithScheme(s).WithObjects(foreign).Build() - owner := metav1.OwnerReference{APIVersion: computev1alpha.GroupVersion.String(), Kind: "Instance", Name: "api", UID: "instance-uid"} + owner := metav1.OwnerReference{APIVersion: computev1alpha.GroupVersion.String(), Kind: internalDNSInstanceKind, Name: internalDNSTestInstanceName, UID: internalDNSTestInstanceUID} - _, _, err := ensureDNSObject(context.Background(), InternalDNSProjectAccess{Reader: cl, Writer: cl}, dnsRegistrationGVK, "default", foreign.GetName(), map[string]string{ - internalDNSManagedBy: internalDNSManager, internalDNSInstanceUID: "instance-uid", - }, owner, map[string]any{"name": "api"}) + _, _, err := ensureDNSObject(context.Background(), InternalDNSProjectAccess{Reader: cl, Writer: cl}, dnsRegistrationGVK, internalDNSTestNamespace, foreign.GetName(), map[string]string{ + internalDNSManagedBy: internalDNSManager, internalDNSInstanceUID: internalDNSTestInstanceUID, + }, owner, map[string]any{internalDNSNameField: internalDNSTestInstanceName}) require.Error(t, err) assert.Contains(t, err.Error(), "refusing to adopt foreign") } @@ -199,11 +185,11 @@ func TestEnsureDNSObjectCreatesProjectInstanceOwnershipWithoutDeleteBlock(t *tes s.AddKnownTypeWithName(dnsRegistrationGVK, &unstructured.Unstructured{}) s.AddKnownTypeWithName(dnsRegistrationGVK.GroupVersion().WithKind("DNSRegistrationList"), &unstructured.UnstructuredList{}) cl := fake.NewClientBuilder().WithScheme(s).Build() - owner := metav1.OwnerReference{APIVersion: computev1alpha.GroupVersion.String(), Kind: "Instance", Name: "api", UID: "project-instance-uid", Controller: ptrBool(true)} + owner := metav1.OwnerReference{APIVersion: computev1alpha.GroupVersion.String(), Kind: internalDNSInstanceKind, Name: internalDNSTestInstanceName, UID: "project-instance-uid", Controller: ptrBool(true)} - created, changed, err := ensureDNSObject(context.Background(), InternalDNSProjectAccess{Reader: cl, Writer: cl}, dnsRegistrationGVK, "default", "instance-owned", map[string]string{ + created, changed, err := ensureDNSObject(context.Background(), InternalDNSProjectAccess{Reader: cl, Writer: cl}, dnsRegistrationGVK, internalDNSTestNamespace, "instance-owned", map[string]string{ internalDNSManagedBy: internalDNSManager, internalDNSInstanceUID: "project-instance-uid", - }, owner, map[string]any{"name": "api"}) + }, owner, map[string]any{internalDNSNameField: internalDNSTestInstanceName}) require.NoError(t, err) assert.True(t, changed) require.Len(t, created.GetOwnerReferences(), 1) diff --git a/test/internaldns/publisher/main.go b/test/internaldns/publisher/main.go index ab5074e3..eb10b616 100644 --- a/test/internaldns/publisher/main.go +++ b/test/internaldns/publisher/main.go @@ -40,7 +40,9 @@ func main() { flags.StringVar(&subject, "subject", "system:serviceaccount:compute-system:dns-publisher", "Authenticated writer") flags.StringVar(&gates, "feature-gates", "", "Compute feature gates") flags.DurationVar(&lease, "lease", 60*time.Second, "Contribution lease") - flags.Parse(os.Args[1:]) + if err := flags.Parse(os.Args[1:]); err != nil { + panic(err) + } ctrl.SetLogger(zap.New(zap.UseDevMode(true))) if err := features.MutableFeatureGate.Set(gates); err != nil { panic(err) From fc8f8b43c7f20c7273f38e840353c5e97896bf45 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Fri, 9 Oct 2026 21:30:33 -0500 Subject: [PATCH 03/33] Align instance DNS publishing with VPC resolver contexts --- api/v1alpha/instance_types.go | 21 ++ api/v1alpha/labels.go | 2 + api/v1alpha/zz_generated.deepcopy.go | 35 +++ cmd/main.go | 40 ++- cmd/main_test.go | 32 +++ .../compute.datumapis.com_instances.yaml | 103 ++++++++ config/components/internal-dns-rbac/role.yaml | 7 +- .../internal-dns-instance-publishing.md | 114 +++++---- internal/config/config.go | 11 + internal/config/config_test.go | 8 + internal/config/zz_generated.deepcopy.go | 21 ++ internal/controller/instance_controller.go | 1 + internal/controller/instance_projector.go | 23 +- .../controller/instance_projector_test.go | 28 +++ .../internaldns_authorization_test.go | 156 ++++++++++++ .../controller/internaldns_observation.go | 196 +++++++++++++++ .../internaldns_observation_test.go | 102 ++++++++ internal/controller/internaldns_publisher.go | 184 +++++++++----- .../controller/internaldns_publisher_test.go | 58 +++-- internal/controller/internaldns_status.go | 85 +++++++ .../controller/internaldns_status_test.go | 68 +++++ internal/features/features.go | 4 +- test/internaldns/README.md | 72 +++--- test/internaldns/fleet_smoke.py | 168 ------------- test/internaldns/publisher/main.go | 21 +- test/internaldns/results/fleet-smoke.json | 235 ------------------ 26 files changed, 1215 insertions(+), 580 deletions(-) create mode 100644 internal/controller/internaldns_authorization_test.go create mode 100644 internal/controller/internaldns_observation.go create mode 100644 internal/controller/internaldns_observation_test.go create mode 100644 internal/controller/internaldns_status.go create mode 100644 internal/controller/internaldns_status_test.go delete mode 100644 test/internaldns/fleet_smoke.py delete mode 100644 test/internaldns/results/fleet-smoke.json diff --git a/api/v1alpha/instance_types.go b/api/v1alpha/instance_types.go index 9d2aa210..195554aa 100644 --- a/api/v1alpha/instance_types.go +++ b/api/v1alpha/instance_types.go @@ -813,6 +813,13 @@ type InstanceStatus struct { // Network interface information NetworkInterfaces []InstanceNetworkInterfaceStatus `json:"networkInterfaces,omitempty"` + // DNS reports assigned private names and publication readiness per network. + // Resolver authorization and publisher leases are managed by platform services. + // +kubebuilder:validation:Optional + // +listType=map + // +listMapKey=networkUID + DNS []InstanceDNSStatus `json:"dns,omitempty"` + // Controller contains status information about the controller managing the instance. // // +kubebuilder:validation:Optional @@ -826,6 +833,20 @@ type InstanceStatus struct { Suspended bool `json:"suspended,omitempty"` } +// InstanceDNSStatus describes private name publication within one network. +type InstanceDNSStatus struct { + // Network identifies the attached network in the owning project. + Network networkingv1alpha.NetworkRef `json:"network"` + // NetworkUID pins the lifetime that owns these names. + NetworkUID string `json:"networkUID"` + // Hostnames contains the fully qualified names allocated for this Instance. + Hostnames []string `json:"hostnames,omitempty"` + // Conditions reports DNS readiness independently of runtime readiness. + // +listType=map + // +listMapKey=type + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + type InstanceControllerStatus struct { // ObservedTemplateHash is the hash of the instance template applied for this instance. // diff --git a/api/v1alpha/labels.go b/api/v1alpha/labels.go index 3aa752df..d27ad2fc 100644 --- a/api/v1alpha/labels.go +++ b/api/v1alpha/labels.go @@ -9,6 +9,8 @@ const ( WorkloadDeploymentUIDLabel = LabelNamespace + "/workload-deployment-uid" InstanceIndexLabel = LabelNamespace + "/instance-index" + // InstanceSourceUIDLabel pins the edge Instance lifetime on its projections. + InstanceSourceUIDLabel = LabelNamespace + "/source-instance-uid" // WorkloadDeploymentNameLabel carries the name of the WorkloadDeployment // that owns an Instance. Stamped at creation and kept current on updates. diff --git a/api/v1alpha/zz_generated.deepcopy.go b/api/v1alpha/zz_generated.deepcopy.go index cae177b8..4d004faa 100644 --- a/api/v1alpha/zz_generated.deepcopy.go +++ b/api/v1alpha/zz_generated.deepcopy.go @@ -513,6 +513,34 @@ func (in *InstanceControllerStatus) DeepCopy() *InstanceControllerStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InstanceDNSStatus) DeepCopyInto(out *InstanceDNSStatus) { + *out = *in + out.Network = in.Network + if in.Hostnames != nil { + in, out := &in.Hostnames, &out.Hostnames + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InstanceDNSStatus. +func (in *InstanceDNSStatus) DeepCopy() *InstanceDNSStatus { + if in == nil { + return nil + } + out := new(InstanceDNSStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InstanceList) DeepCopyInto(out *InstanceList) { *out = *in @@ -811,6 +839,13 @@ func (in *InstanceStatus) DeepCopyInto(out *InstanceStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.DNS != nil { + in, out := &in.DNS, &out.DNS + *out = make([]InstanceDNSStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.Controller != nil { in, out := &in.Controller, &out.Controller *out = new(InstanceControllerStatus) diff --git a/cmd/main.go b/cmd/main.go index 0eeaa829..a22a1e89 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -459,10 +459,16 @@ func main() { setupLog.Error(identityErr, "invalid internal DNS publisher configuration") os.Exit(1) } + observers, observationErr := internalDNSObservationSources(serverConfig.InternalDNS) + if observationErr != nil { + setupLog.Error(observationErr, "invalid internal DNS network observation configuration") + os.Exit(1) + } dnsPublisher := &controller.InternalDNSPublisherReconciler{ - ProjectIdentities: identities, - PrincipalSubject: serverConfig.InternalDNS.PrincipalSubject, - LeaseDuration: serverConfig.InternalDNS.LeaseDuration.Duration, + ProjectIdentities: identities, + PrincipalSubject: serverConfig.InternalDNS.PrincipalSubject, + LeaseDuration: serverConfig.InternalDNS.LeaseDuration.Duration, + ObservationSources: observers, } if err = dnsPublisher.SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "InternalDNSPublisher") @@ -968,6 +974,34 @@ func computeWatchProviderClaims(mode multiclusterproviders.Provider) bool { return mode == multiclusterproviders.ProviderMilo } +func internalDNSObservationSources( + cfg config.InternalDNSConfig, +) (map[string]controller.InternalDNSObservationSource, error) { + sources := make(map[string]controller.InternalDNSObservationSource, len(cfg.ObservationSources)) + for i, source := range cfg.ObservationSources { + if source.Location == "" || source.ClusterUID == "" || source.KubeconfigPath == "" { + return nil, fmt.Errorf("internalDNS.observationSources[%d] requires location, clusterUID, and kubeconfigPath", i) + } + if _, exists := sources[source.Location]; exists { + return nil, fmt.Errorf("duplicate internal DNS observation location %q", source.Location) + } + cfg, err := clientcmd.BuildConfigFromFlags("", source.KubeconfigPath) + if err != nil { + return nil, fmt.Errorf("load network observation credentials for %q: %w", source.Location, err) + } + cfg.Timeout = 5 * time.Second + reader, err := client.New(cfg, client.Options{Scheme: scheme}) + if err != nil { + return nil, fmt.Errorf("create network observation client for %q: %w", source.Location, err) + } + sources[source.Location] = controller.InternalDNSObservationSource{Reader: reader, ClusterUID: source.ClusterUID} + } + if len(sources) == 0 { + return nil, errors.New("internalDNS.observationSources must contain at least one authenticated edge API") + } + return sources, nil +} + func internalDNSProjectIdentities( cfg config.InternalDNSConfig, ) (map[string]controller.InternalDNSProjectIdentity, error) { diff --git a/cmd/main_test.go b/cmd/main_test.go index c42cff40..f64a910f 100644 --- a/cmd/main_test.go +++ b/cmd/main_test.go @@ -48,6 +48,38 @@ func TestInternalDNSProjectIdentities(t *testing.T) { assert.Error(t, err, "a producer must not configure a lease DNS admission rejects") } +func TestInternalDNSObservationSourcesRequireExplicitCredentials(t *testing.T) { + _, err := internalDNSObservationSources(config.InternalDNSConfig{}) + require.Error(t, err) + _, err = internalDNSObservationSources(config.InternalDNSConfig{ + ObservationSources: []config.InternalDNSObservationSource{{Location: "dfw"}}, + }) + require.Error(t, err) + path := filepath.Join(t.TempDir(), "edge.kubeconfig") + require.NoError(t, os.WriteFile(path, []byte(`apiVersion: v1 +kind: Config +clusters: + - name: edge + cluster: + server: https://127.0.0.1:6443 +contexts: + - name: edge + context: + cluster: edge +current-context: edge +`), 0o600)) + source := config.InternalDNSObservationSource{Location: "dfw", ClusterUID: "edge-uid", KubeconfigPath: path} + sources, err := internalDNSObservationSources(config.InternalDNSConfig{ + ObservationSources: []config.InternalDNSObservationSource{source}, + }) + require.NoError(t, err) + require.EqualValues(t, "edge-uid", sources["dfw"].ClusterUID) + _, err = internalDNSObservationSources(config.InternalDNSConfig{ + ObservationSources: []config.InternalDNSObservationSource{source, source}, + }) + require.Error(t, err) +} + // TestComputeWatchProviderClaims is the #171 guard: quota enforcement (and thus // the ResourceClaim watch) is wired only in Milo mode. Single/cluster mode must // stay false, so the manager never engages a ResourceClaim watch against a cell diff --git a/config/base/crd/bases/compute.datumapis.com_instances.yaml b/config/base/crd/bases/compute.datumapis.com_instances.yaml index a7336baf..65ab045b 100644 --- a/config/base/crd/bases/compute.datumapis.com_instances.yaml +++ b/config/base/crd/bases/compute.datumapis.com_instances.yaml @@ -1260,6 +1260,109 @@ spec: required: - observedTemplateHash type: object + dns: + description: |- + DNS reports assigned private names and publication readiness per network. + Resolver authorization and publisher leases are managed by platform services. + items: + description: InstanceDNSStatus describes private name publication + within one network. + properties: + conditions: + description: Conditions reports DNS readiness independently + of runtime readiness. + items: + description: Condition contains details for one aspect of + the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + hostnames: + description: Hostnames contains the fully qualified names allocated + for this Instance. + items: + type: string + type: array + network: + description: Network identifies the attached network in the + owning project. + properties: + name: + description: The network name + type: string + namespace: + description: |- + The network namespace. + + Defaults to the namespace for the type the reference is embedded in. + type: string + required: + - name + type: object + networkUID: + description: NetworkUID pins the lifetime that owns these names. + type: string + required: + - network + - networkUID + type: object + type: array + x-kubernetes-list-map-keys: + - networkUID + x-kubernetes-list-type: map networkInterfaces: description: Network interface information items: diff --git a/config/components/internal-dns-rbac/role.yaml b/config/components/internal-dns-rbac/role.yaml index d4b0057b..afe766b9 100644 --- a/config/components/internal-dns-rbac/role.yaml +++ b/config/components/internal-dns-rbac/role.yaml @@ -4,11 +4,14 @@ metadata: name: compute-internal-dns-publisher rules: - apiGroups: ["dns.networking.miloapis.com"] - resources: ["dnsmanagednamespaces"] + resources: ["dnsresolvercontexts", "dnscontributiongrants"] verbs: ["get", "list"] - apiGroups: ["dns.networking.miloapis.com"] - resources: ["dnsregistrations", "dnscontributiongrants", "dnsrecordcontributions"] + resources: ["dnsregistrations", "dnsrecordcontributions"] verbs: ["get", "list", "create", "update", "patch", "delete"] - apiGroups: ["dns.networking.miloapis.com"] resources: ["dnsrecordcontributions/status"] verbs: ["get", "update", "patch"] + - apiGroups: ["compute.datumapis.com"] + resources: ["instances/status"] + verbs: ["get", "patch"] diff --git a/docs/enhancements/internal-dns-instance-publishing.md b/docs/enhancements/internal-dns-instance-publishing.md index 7fbd2608..2c154e6d 100644 --- a/docs/enhancements/internal-dns-instance-publishing.md +++ b/docs/enhancements/internal-dns-instance-publishing.md @@ -1,65 +1,91 @@ # Internal DNS instance publishing -Compute can publish stable private instance identity names through the internal -DNS API. The integration is disabled by default and supports `Instance` -resources only. It does not infer service discovery intent or publish public -addresses. +Compute publishes private instance names in each attached VPC's `datum.internal` +zone. Users attach a workload to a network; they do not select a DNS zone. The +assigned label combines the Instance name with its immutable lifetime identity. +For example, an instance can receive `web-01-43ce24f81d265cdf.datum.internal`. +Each VPC uses an independent DNS context, including when zone names overlap. -Enable it with `--feature-gates=InternalDNSPublishing=true` and configure the -identity that the DNS control plane trusts for each Milo project: +`Instance.status.dns` reports assigned hostnames and publication readiness per +network. DNS errors do not change runtime readiness. Application readiness does +not remove an instance's identity name. + +```yaml +status: + dns: + - network: + name: application + namespace: production + networkUID: 00000000-0000-0000-0000-000000000020 + hostnames: [web-01-43ce24f81d265cdf.datum.internal] + conditions: + - type: Ready + status: "True" + reason: Published + message: Private name is published + observedGeneration: 1 + lastTransitionTime: "2026-10-09T20:00:00Z" +``` + +Enable the default-off `InternalDNSPublishing` feature gate only after +provisioning the DNS platform, project permissions, and edge observation access. +The management controller requires Milo discovery. ```yaml apiVersion: apiserver.config.datumapis.com/v1alpha1 kind: WorkloadOperator internalDNS: + # Match the authenticated subject used for project DNS writes. principalSubject: system:serviceaccount:compute-system:compute-manager leaseDuration: 60s projects: - name: example-project projectUID: 00000000-0000-0000-0000-000000000001 + # Match the project source API identity trusted by DNS admission. sourceClusterUID: 00000000-0000-0000-0000-000000000010 + observationSources: + - location: dfw + # Pin the edge cluster's kube-system namespace lifetime. + clusterUID: 00000000-0000-0000-0000-000000000030 + # Use separately provisioned read-only edge credentials. + kubeconfigPath: /etc/compute/edge-dfw.kubeconfig ``` -`projectUID` and `sourceClusterUID` must match the trusted project entry in the -DNS control-plane configuration. The principal must be the authenticated -Kubernetes username used for project API writes. These values select and -authorize an existing DNS-owned namespace; they do not authenticate the -publisher by themselves. +Compute selects the ready resolver context whose `consumerID` is +`/` and reads its managed zone. It reserves a name through +`DNSRegistration` and waits for a trusted issuer's scoped `DNSContributionGrant`. +Compute can read grants but cannot create, update, or delete them. Contributions +pin both registration and grant lifetimes and generations. DNS admission checks +the authenticated publisher subject and source identity. -The project API must separately authorize the authenticated principal to read -managed namespaces and to manage registrations, grants, contributions, and -producer-owned contribution status. In Milo this requires the corresponding DNS -`ProtectedResource`/IAM policy (or equivalent project authorization) for the -client certificate or token subject. The -`config/components/internal-dns-rbac` component supplies direct Kubernetes RBAC -for a same-API development environment; applying it only to Compute's deployment -cluster does not grant access through a Milo project control-plane endpoint. -Production rollout therefore requires matching infra/IAM provisioning before -the feature gate is enabled. +The optional `config/components/internal-dns-rbac` component grants DNS access +on a direct Kubernetes API. Milo project authorization must grant equivalent +permissions separately, including Instance status writes. Applying a role in +the deployment cluster does not authorize project API access. -The first integration requires `discovery.mode: milo` and management -controllers. It uses the engaged project client's authenticated connection and -renews leases only from an uncached read of the authoritative project Instance. -An edge or federation copy with a different UID is rejected, preventing an old -copy from publishing into a replacement Instance lifetime. +Edge observation credentials require reads of Instances, NetworkInterfaces, +NetworkInterfaceClaims, NetworkContexts, Pods, Nodes, node leases, and the `kube-system` namespace. +Scope namespaced reads to the supported projects' edge namespaces. Credentials +cannot publish records or issue grants. Cluster-wide reads are limited to nodes +and the pinned namespace. -For each VPC attachment, the publisher reads the project `Network` UID and -selects an accepted `DNSManagedNamespace` whose project and VPC UIDs both -match. It creates a generation-pinned `DNSRegistration`, -`DNSContributionGrant`, and `DNSRecordContribution`. The name combines the -immutable Instance UID with its allocated name, so display-name reuse cannot -adopt an earlier identity. A separate registration is used for every VPC. +Only private IPv4 and IPv6 host addresses from live allocated, programmed +interfaces are eligible. Public addresses and delegated prefixes are excluded. +The observer checks the source Instance lifetime, interface claim, running +runtime Pod, ready Node, and its original heartbeat. Publication cannot outlive +that heartbeat by more than 45 seconds or exceed the configured lease. +Unsupported runtimes, cross-namespace network attachments, ambiguous runtime +Pods, and observation failures are ineligible. Projected positive status cannot +renew a lease. This requires a runtime represented by an Instance-owned Pod and +a node heartbeat lease. -Only allocated and programmed private host addresses are published. IPv4 and -IPv6 are supported; a delegated prefix is not treated as one instance address. -Application readiness does not affect instance identity. Address release or -network deprogramming writes an ineligible observation, and deletion is -garbage-collected through the project-side Instance owner reference. The -publisher refreshes a bounded lease from current Instance/interface state, so a -disconnected publisher cannot retain an address indefinitely. +Interface withdrawal makes the contribution ineligible. Instance deletion and +replacement reclaim records through project-side owner references. DNS retains +ownership of publication status; Compute patches only its observation fields. -DNS reconciliation errors requeue this controller and do not modify Compute -readiness. Publication state remains available on the DNS registration and -contribution resources. With the feature gate disabled, Compute does not -register the publisher, discover DNS resources, make DNS API calls, or require -the optional DNS RBAC component. +Guest resolver application belongs to workload providers. It requires the +access-restricted network services delivery contract described in the +[Compute enhancement](https://github.com/datum-cloud/compute/pull/470). +This publisher does not configure guests, expose authorization leases on public +NetworkInterfaces, or implement peer DNS aliases. Supported guests use +`datum.internal` as their default search domain when resolver delivery is enabled. diff --git a/internal/config/config.go b/internal/config/config.go index 17a8b022..39d77e42 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -71,6 +71,17 @@ type InternalDNSConfig struct { // Projects pins each Milo project name to the trusted project and source API // UIDs configured by the DNS control plane. Projects []InternalDNSProjectIdentity `json:"projects,omitempty"` + + // ObservationSources selects authenticated edge APIs for live interface and + // node observations. Credentials must have read-only observation permissions. + ObservationSources []InternalDNSObservationSource `json:"observationSources,omitempty"` +} + +// +k8s:deepcopy-gen=true +type InternalDNSObservationSource struct { + Location string `json:"location"` + ClusterUID types.UID `json:"clusterUID"` + KubeconfigPath string `json:"kubeconfigPath"` } // +k8s:deepcopy-gen=true diff --git a/internal/config/config_test.go b/internal/config/config_test.go index eab9af75..bb421355 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -38,6 +38,10 @@ metricsServer: internalDNS: principalSubject: system:serviceaccount:compute-system:compute-manager leaseDuration: 60s + observationSources: + - location: dfw + clusterUID: edge-uid + kubeconfigPath: /etc/compute/edge.kubeconfig projects: - name: project-a projectUID: project-uid @@ -48,6 +52,10 @@ internalDNS: } copy := cfg.DeepCopy() copy.InternalDNS.Projects[0].ProjectUID = "changed" + copy.InternalDNS.ObservationSources[0].ClusterUID = "changed" + if cfg.InternalDNS.ObservationSources[0].ClusterUID != "edge-uid" { + t.Fatal("WorkloadOperator DeepCopy aliases observation identities") + } if cfg.InternalDNS.Projects[0].ProjectUID != "project-uid" { t.Fatal("WorkloadOperator DeepCopy aliases internal DNS project identities") } diff --git a/internal/config/zz_generated.deepcopy.go b/internal/config/zz_generated.deepcopy.go index 4cac042d..3b79b828 100644 --- a/internal/config/zz_generated.deepcopy.go +++ b/internal/config/zz_generated.deepcopy.go @@ -69,11 +69,17 @@ func (in *FeatureFlagsConfig) DeepCopy() *FeatureFlagsConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InternalDNSConfig) DeepCopyInto(out *InternalDNSConfig) { *out = *in + out.LeaseDuration = in.LeaseDuration if in.Projects != nil { in, out := &in.Projects, &out.Projects *out = make([]InternalDNSProjectIdentity, len(*in)) copy(*out, *in) } + if in.ObservationSources != nil { + in, out := &in.ObservationSources, &out.ObservationSources + *out = make([]InternalDNSObservationSource, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InternalDNSConfig. @@ -86,6 +92,21 @@ func (in *InternalDNSConfig) DeepCopy() *InternalDNSConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InternalDNSObservationSource) DeepCopyInto(out *InternalDNSObservationSource) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InternalDNSObservationSource. +func (in *InternalDNSObservationSource) DeepCopy() *InternalDNSObservationSource { + if in == nil { + return nil + } + out := new(InternalDNSObservationSource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InternalDNSProjectIdentity) DeepCopyInto(out *InternalDNSProjectIdentity) { *out = *in diff --git a/internal/controller/instance_controller.go b/internal/controller/instance_controller.go index 5ca6c907..5926b49a 100644 --- a/internal/controller/instance_controller.go +++ b/internal/controller/instance_controller.go @@ -1305,6 +1305,7 @@ func (r *InstanceReconciler) writeBackToUpstream(ctx context.Context, instance * computev1alpha.WorkloadUIDLabel: instance.Labels[computev1alpha.WorkloadUIDLabel], computev1alpha.WorkloadDeploymentUIDLabel: instance.Labels[computev1alpha.WorkloadDeploymentUIDLabel], computev1alpha.InstanceIndexLabel: instance.Labels[computev1alpha.InstanceIndexLabel], + computev1alpha.InstanceSourceUIDLabel: string(instance.UID), computev1alpha.WorkloadDeploymentNameLabel: instance.Labels[computev1alpha.WorkloadDeploymentNameLabel], computev1alpha.LocationLabel: instance.Labels[computev1alpha.LocationLabel], computev1alpha.WorkloadNameLabel: instance.Labels[computev1alpha.WorkloadNameLabel], diff --git a/internal/controller/instance_projector.go b/internal/controller/instance_projector.go index 071160f7..a84e7556 100644 --- a/internal/controller/instance_projector.go +++ b/internal/controller/instance_projector.go @@ -5,6 +5,7 @@ package controller import ( "context" "fmt" + "time" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -36,10 +37,9 @@ import ( // so that it is garbage-collected via cascading deletion when the deployment is // removed from the project cluster. // -// The projector never deletes anything. Reclamation belongs to hub ownership: -// hub Instances are owned by their hub WorkloadDeployment, and write-back is -// owner-gated, so an object the projector cannot resolve is not something for it -// to clean up. +// A changed source Instance UID replaces the projection so project-owned +// resources cannot survive into another instance lifetime. Deployment deletion +// reclaims projections through their owner references. // // The controller is registered on the leader-elected local manager so only the // elected replica writes projections, and watches Instances through the @@ -150,6 +150,18 @@ func (r *InstanceProjector) Reconcile(ctx context.Context, req ctrl.Request) (ct }, } + if err := projectCluster.GetAPIReader().Get(ctx, client.ObjectKeyFromObject(projection), projection); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + oldSource := projection.Labels[computev1alpha.InstanceSourceUIDLabel] + newSource := downstreamInstance.Labels[computev1alpha.InstanceSourceUIDLabel] + if oldSource != "" && newSource != "" && oldSource != newSource { + if err := projectClient.Delete(ctx, projection, client.Preconditions{UID: &projection.UID, ResourceVersion: &projection.ResourceVersion}); client.IgnoreNotFound(err) != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: time.Millisecond}, nil + } + operationResult, err := controllerutil.CreateOrUpdate(ctx, projectClient, projection, func() error { // Propagate upstream tracking labels so consumers can filter by origin. if projection.Labels == nil { @@ -173,7 +185,10 @@ func (r *InstanceProjector) Reconcile(ctx context.Context, req ctrl.Request) (ct logger.Info("reconciled Instance projection", "operation", operationResult, "namespace", targetNamespace, "cluster", clusterName) // Status is a separate subresource. + dnsStatus := projection.Status.DNS projection.Status = downstreamInstance.Status + // Private name publication belongs to the project-side DNS publisher. + projection.Status.DNS = dnsStatus if err := projectClient.Status().Update(ctx, projection); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, fmt.Errorf("failed updating Instance projection status: %w", err) } diff --git a/internal/controller/instance_projector_test.go b/internal/controller/instance_projector_test.go index 729a9de5..53db4c33 100644 --- a/internal/controller/instance_projector_test.go +++ b/internal/controller/instance_projector_test.go @@ -438,6 +438,34 @@ func TestInstanceProjector_SpecCopied(t *testing.T) { assert.Equal(t, "test-gate", projection.Spec.Controller.SchedulingGates[0].Name) } +func TestInstanceProjectorPreservesProjectDNSStatus(t *testing.T) { + upstream := projTestKarmadaInstance(map[string]string{computev1alpha.InstanceSourceUIDLabel: "edge-instance"}) + upstream.Status.DNS = []computev1alpha.InstanceDNSStatus{{NetworkUID: "untrusted-edge-status"}} + projection := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: projTestInstanceName, Namespace: projTestProjNS, Labels: map[string]string{computev1alpha.InstanceSourceUIDLabel: "edge-instance"}}, Status: computev1alpha.InstanceStatus{DNS: []computev1alpha.InstanceDNSStatus{{NetworkUID: "project-network", Hostnames: []string{"web.datum.internal"}}}}} + cl := fake.NewClientBuilder().WithScheme(newProjectScheme()).WithObjects(projTestWorkloadDeployment(), projection).WithStatusSubresource(&computev1alpha.Instance{}).Build() + r := newTestProjector(newKarmadaFakeClient(upstream), cl) + _, err := r.Reconcile(context.Background(), projectorRequest()) + require.NoError(t, err) + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(projection), projection)) + require.Equal(t, "project-network", projection.Status.DNS[0].NetworkUID) +} + +func TestInstanceProjectorReplacesDifferentSourceLifetime(t *testing.T) { + upstream := projTestKarmadaInstance(map[string]string{computev1alpha.InstanceSourceUIDLabel: "new-edge-instance"}) + projection := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: projTestInstanceName, Namespace: projTestProjNS, UID: "old-project-instance", Labels: map[string]string{computev1alpha.InstanceSourceUIDLabel: "old-edge-instance"}}} + cl := fake.NewClientBuilder().WithScheme(newProjectScheme()).WithObjects(projTestWorkloadDeployment(), projection).WithStatusSubresource(&computev1alpha.Instance{}).Build() + r := newTestProjector(newKarmadaFakeClient(upstream), cl) + result, err := r.Reconcile(context.Background(), projectorRequest()) + require.NoError(t, err) + require.Positive(t, result.RequeueAfter) + require.True(t, apierrors.IsNotFound(cl.Get(context.Background(), client.ObjectKeyFromObject(projection), projection))) + _, err = r.Reconcile(context.Background(), projectorRequest()) + require.NoError(t, err) + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(projection), projection)) + require.Equal(t, "new-edge-instance", projection.Labels[computev1alpha.InstanceSourceUIDLabel]) + require.Empty(t, projection.Status.DNS) +} + // TestInstanceProjector_NamespaceResolution verifies that the projector resolves // the target project namespace directly from the UpstreamOwnerNamespaceLabel on // the Karmada Instance, landing the projection in the correct namespace. diff --git a/internal/controller/internaldns_authorization_test.go b/internal/controller/internaldns_authorization_test.go new file mode 100644 index 00000000..e91605ad --- /dev/null +++ b/internal/controller/internaldns_authorization_test.go @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + computev1alpha "go.datum.net/compute/api/v1alpha" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +func TestFindContributionGrantRequiresCurrentScopedAuthority(t *testing.T) { + registration := newDNSObject(dnsRegistrationGVK, internalDNSTestNamespace, "web") + registration.SetUID("registration-uid") + registration.SetGeneration(2) + for _, tc := range []struct { + name string + path []string + value any + found bool + }{ + {name: "active matching grant", found: true}, + {name: "different principal", path: []string{"spec", "principal", "subject"}, value: "other"}, + {name: "different source cluster", path: []string{"spec", "principal", "clusterUID"}, value: "other"}, + {name: "different registration lifetime", path: []string{"spec", "registrationRef", "uid"}, value: "other"}, + {name: "stale registration generation", path: []string{"spec", "registrationRef", "generation"}, value: int64(1)}, + {name: "different name scope", path: []string{"spec", "nameScopes"}, value: []any{"other"}}, + {name: "missing record type", path: []string{"spec", "recordTypes"}, value: []any{"A"}}, + {name: "inactive grant", path: []string{"status", "conditions"}, value: []any{map[string]any{"type": "Active", "status": "False", "observedGeneration": int64(1)}}}, + {name: "stale active condition", path: []string{"status", "conditions"}, value: []any{map[string]any{"type": "Active", "status": "True", "observedGeneration": int64(0)}}}, + } { + t.Run(tc.name, func(t *testing.T) { + s := runtime.NewScheme() + s.AddKnownTypeWithName(dnsGrantGVK, &unstructured.Unstructured{}) + s.AddKnownTypeWithName(dnsGrantGVK.GroupVersion().WithKind("DNSContributionGrantList"), &unstructured.UnstructuredList{}) + grant := newDNSObject(dnsGrantGVK, internalDNSTestNamespace, "trusted-grant") + grant.SetUID("grant-uid") + grant.SetGeneration(1) + grant.Object["spec"] = map[string]any{"registrationRef": objectReference(registration), "principal": map[string]any{"clusterUID": "project-api", "subject": "compute"}, "nameScopes": []any{"web"}, "recordTypes": []any{"A", "AAAA"}} + grant.Object["status"] = map[string]any{"conditions": []any{map[string]any{"type": "Active", "status": "True", "observedGeneration": int64(1)}}} + if len(tc.path) > 0 { + require.NoError(t, unstructured.SetNestedField(grant.Object, tc.value, tc.path...)) + } + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(grant).Build() + got, err := findContributionGrant(context.Background(), cl, internalDNSTestNamespace, registration, "project-api", "compute", "web") + require.NoError(t, err) + require.Equal(t, tc.found, got != nil) + }) + } +} + +func TestRefreshObservationCannotExtendSourceDeadline(t *testing.T) { + now := time.Date(2026, 10, 9, 20, 0, 0, 0, time.UTC) + s := runtime.NewScheme() + s.AddKnownTypeWithName(dnsContributionGVK, &unstructured.Unstructured{}) + c := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, "web") + c.SetUID("contribution") + c.SetGeneration(1) + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(c).WithStatusSubresource(c).Build() + r := &InternalDNSPublisherReconciler{Now: func() time.Time { return now }, LeaseDuration: time.Minute} + deadline := now.Add(25 * time.Second) + for _, step := range []time.Duration{0, 10 * time.Second, 26 * time.Second} { + now = time.Date(2026, 10, 9, 20, 0, 0, 0, time.UTC).Add(step) + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(c), c)) + _, err := r.refreshObservation(context.Background(), cl, c, 3, true, deadline) + require.NoError(t, err) + eligible, _, _ := unstructured.NestedBool(c.Object, "status", "eligible") + require.Equal(t, step < 25*time.Second, eligible) + if eligible { + raw, _, _ := unstructured.NestedString(c.Object, "status", "validUntil") + got, err := time.Parse(time.RFC3339Nano, raw) + require.NoError(t, err) + require.True(t, got.Equal(deadline)) + } + } +} + +func TestAllocatedInstanceDNSName(t *testing.T) { + first := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: "Web_01", UID: "first"}} + second := first.DeepCopy() + second.UID = "second" + require.Regexp(t, `^web-01-[a-f0-9]{16}$`, allocatedInstanceDNSName(first)) + require.NotEqual(t, allocatedInstanceDNSName(first), allocatedInstanceDNSName(second)) + require.Equal(t, allocatedInstanceDNSName(first), allocatedInstanceDNSName(first.DeepCopy())) +} + +func TestPublisherWaitsForIssuerAndNeverMutatesGrants(t *testing.T) { + s := runtime.NewScheme() + for _, gvk := range []schema.GroupVersionKind{dnsResolverContextGVK, dnsRegistrationGVK, dnsGrantGVK, dnsContributionGVK} { + s.AddKnownTypeWithName(gvk, &unstructured.Unstructured{}) + s.AddKnownTypeWithName(gvk.GroupVersion().WithKind(gvk.Kind+"List"), &unstructured.UnstructuredList{}) + } + contextObject := resolverContextObject("context", "project-a", "vpc-a", true) + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(contextObject).WithStatusSubresource(newDNSObject(dnsRegistrationGVK, "", ""), newDNSObject(dnsContributionGVK, "", "")).WithInterceptorFuncs(interceptor.Funcs{Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + obj.SetUID(types.UID(obj.GetName() + "-uid")) + obj.SetGeneration(1) + return c.Create(ctx, obj, opts...) + }}).Build() + guard := interceptor.NewClient(cl, interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + require.NotEqual(t, dnsGrantGVK, obj.GetObjectKind().GroupVersionKind()) + return c.Create(ctx, obj, opts...) + }, + Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + require.NotEqual(t, dnsGrantGVK, obj.GetObjectKind().GroupVersionKind()) + return c.Update(ctx, obj, opts...) + }, + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + require.NotEqual(t, dnsGrantGVK, obj.GetObjectKind().GroupVersionKind()) + return c.Delete(ctx, obj, opts...) + }, + }) + now := time.Now().UTC().Truncate(time.Second) + r := &InternalDNSPublisherReconciler{PrincipalSubject: "compute", Now: func() time.Time { return now }} + instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: "web", Namespace: internalDNSTestNamespace, UID: "instance-uid"}} + owner := metav1.OwnerReference{APIVersion: computev1alpha.GroupVersion.String(), Kind: "Instance", Name: instance.Name, UID: instance.UID, Controller: ptrBool(true)} + access := InternalDNSProjectAccess{Reader: guard, Writer: guard} + identity := InternalDNSProjectIdentity{ProjectUID: "project-a", SourceClusterUID: "project-api"} + attachment := instanceDNSAttachment{VPCUID: "vpc-a", Eligible: true, ValidUntil: now.Add(40 * time.Second), RecordSets: []any{map[string]any{"recordType": "A", "records": []any{map[string]any{"name": "", "ttl": int64(30), "a": map[string]any{"content": "10.0.0.8"}}}}}} + ctx := context.Background() + for range 2 { + ready, err := r.reconcileAttachment(ctx, access, identity, internalDNSTestNamespace, instance, owner, attachment, "record") + require.NoError(t, err) + require.False(t, ready) + } + registration := newDNSObject(dnsRegistrationGVK, internalDNSTestNamespace, "record") + require.NoError(t, cl.Get(ctx, client.ObjectKeyFromObject(registration), registration)) + name, _, _ := unstructured.NestedString(registration.Object, "spec", "name") + require.Equal(t, allocatedInstanceDNSName(instance), name) + grant := newDNSObject(dnsGrantGVK, internalDNSTestNamespace, "issuer-grant") + grant.Object["spec"] = map[string]any{"registrationRef": objectReference(registration), "principal": map[string]any{"clusterUID": "project-api", "subject": "compute"}, "nameScopes": []any{name}, "recordTypes": []any{"A", "AAAA"}} + grant.Object["status"] = map[string]any{"conditions": []any{map[string]any{"type": "Active", "status": "True", "observedGeneration": int64(1)}}, "activeWriterEpoch": int64(3), "observedGrantGeneration": int64(1), "observedRegistrationGeneration": int64(1)} + require.NoError(t, cl.Create(ctx, grant)) + ready, err := r.reconcileAttachment(ctx, access, identity, internalDNSTestNamespace, instance, owner, attachment, "record") + require.NoError(t, err) + require.False(t, ready) + contribution := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, "record") + require.NoError(t, cl.Get(ctx, client.ObjectKeyFromObject(contribution), contribution)) + contribution.Object["status"] = map[string]any{"writerEpoch": int64(3)} + require.NoError(t, cl.Status().Update(ctx, contribution)) + ready, err = r.reconcileAttachment(ctx, access, identity, internalDNSTestNamespace, instance, owner, attachment, "record") + require.NoError(t, err) + require.True(t, ready) + require.NoError(t, r.deletePublishedResources(ctx, access, internalDNSTestNamespace, instance.UID, nil)) + require.NoError(t, cl.Get(ctx, client.ObjectKeyFromObject(grant), grant)) +} diff --git a/internal/controller/internaldns_observation.go b/internal/controller/internaldns_observation.go new file mode 100644 index 00000000..6b9c4610 --- /dev/null +++ b/internal/controller/internaldns_observation.go @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "fmt" + "time" + + computev1alpha "go.datum.net/compute/api/v1alpha" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const dnsNodeObservationLifetime = 45 * time.Second + +// InternalDNSObservedInterface is a live interface observation bounded by node liveness. +// The original deadline is preserved when publishing the contribution. +type InternalDNSObservedInterface struct { + Addresses []networkingv1alpha.NetworkInterfaceAddress + Eligible bool + ValidUntil time.Time +} + +type InternalDNSObserveFunc func(context.Context, client.Reader, string, *computev1alpha.Instance, computev1alpha.InstanceNetworkInterface) (InternalDNSObservedInterface, error) + +// InternalDNSObservationSource is an authenticated, uncached edge API reader. +// ClusterUID pins the kube-system namespace UID of the selected edge cluster. +type InternalDNSObservationSource struct { + Reader client.Reader + ClusterUID types.UID +} + +// ObserveInternalDNSInterface validates source identity, interface programming, +// runtime placement, and a current node heartbeat. Projected positive status is +// never sufficient to renew a private address. +func ObserveInternalDNSInterface(ctx context.Context, project client.Reader, namespace string, instance *computev1alpha.Instance, nic computev1alpha.InstanceNetworkInterface, source InternalDNSObservationSource, now time.Time) (InternalDNSObservedInterface, error) { + var result InternalDNSObservedInterface + if nic.Network.Namespace != "" && nic.Network.Namespace != namespace { + return result, fmt.Errorf("interface observations require a network in the Instance namespace") + } + if source.Reader == nil || source.ClusterUID == "" { + return result, fmt.Errorf("no authenticated network observation source is configured for location %q", instance.Labels[computev1alpha.LocationLabel]) + } + var cluster, projectNS corev1.Namespace + if err := source.Reader.Get(ctx, client.ObjectKey{Name: "kube-system"}, &cluster); err != nil { + return result, err + } + if cluster.UID != source.ClusterUID { + return result, fmt.Errorf("network observation source cluster lifetime does not match") + } + if err := project.Get(ctx, client.ObjectKey{Name: namespace}, &projectNS); err != nil { + return result, err + } + if projectNS.UID == "" { + return result, fmt.Errorf("project namespace has no lifetime identity") + } + var network networkingv1alpha.Network + if err := project.Get(ctx, client.ObjectKey{Namespace: namespace, Name: nic.Network.Name}, &network); err != nil { + return result, err + } + if network.UID == "" || !network.DeletionTimestamp.IsZero() { + return result, nil + } + sourceNamespace := "ns-" + string(projectNS.UID) + var current computev1alpha.Instance + if err := source.Reader.Get(ctx, client.ObjectKey{Namespace: sourceNamespace, Name: instance.Name}, ¤t); err != nil { + return result, err + } + if current.UID == "" || string(current.UID) != instance.Labels[computev1alpha.InstanceSourceUIDLabel] || !current.DeletionTimestamp.IsZero() { + return result, fmt.Errorf("source Instance lifetime does not match the project projection") + } + iface, err := observeProgrammedInterface(ctx, source.Reader, ¤t, nic) + if err != nil || iface == nil { + return result, err + } + if err := verifyObservedNetwork(ctx, source.Reader, iface, &network, instance.Labels[computev1alpha.LocationLabel]); err != nil { + return result, err + } + nodeName, err := observeRuntimeNode(ctx, source.Reader, ¤t) + if err != nil || nodeName == "" { + return result, err + } + deadline, err := observeNodeDeadline(ctx, source.Reader, nodeName, now) + if err != nil || deadline.IsZero() { + return result, err + } + result.Addresses = append([]networkingv1alpha.NetworkInterfaceAddress(nil), iface.Spec.Addresses...) + result.Eligible = true + result.ValidUntil = deadline + return result, nil +} + +func verifyObservedNetwork(ctx context.Context, reader client.Reader, iface *networkingv1alpha.NetworkInterface, network *networkingv1alpha.Network, location string) error { + if iface.Status.NetworkContextRef == nil || location == "" { + return fmt.Errorf("interface has no verified network context") + } + var networkContext networkingv1alpha.NetworkContext + if err := reader.Get(ctx, client.ObjectKey{Namespace: iface.Namespace, Name: iface.Status.NetworkContextRef.Name}, &networkContext); err != nil { + return err + } + if networkContext.UID == "" || !networkContext.DeletionTimestamp.IsZero() || networkContext.Labels[networkingv1alpha.NetworkUIDLabel] != string(network.UID) || networkContext.Spec.Network.Name != network.Name || networkContext.Spec.Location.Name != location { + return fmt.Errorf("interface network lifetime or location does not match the project") + } + return nil +} + +func observeProgrammedInterface(ctx context.Context, reader client.Reader, current *computev1alpha.Instance, nic computev1alpha.InstanceNetworkInterface) (*networkingv1alpha.NetworkInterface, error) { + var ifaceName string + for _, status := range current.Status.NetworkInterfaces { + if status.Name == internalDNSInterfaceName(nic) && status.NetworkInterfaceRef != nil { + ifaceName = status.NetworkInterfaceRef.Name + } + } + if ifaceName == "" { + return nil, nil + } + var iface networkingv1alpha.NetworkInterface + if err := reader.Get(ctx, client.ObjectKey{Namespace: current.Namespace, Name: ifaceName}, &iface); err != nil { + return nil, err + } + if iface.UID == "" || !iface.DeletionTimestamp.IsZero() || iface.Spec.Network.Name != nic.Network.Name || iface.Spec.InterfaceName != internalDNSInterfaceName(nic) || iface.Spec.ClaimRef == nil { + return nil, nil + } + for _, kind := range []string{"Allocated", "Programmed"} { + condition := apimeta.FindStatusCondition(iface.Status.Conditions, kind) + if condition == nil || condition.Status != metav1.ConditionTrue || condition.ObservedGeneration != iface.Generation { + return nil, nil + } + } + if iface.Spec.ClaimRef.Name != networkInterfaceClaimName(current.Name, internalDNSInterfaceName(nic)) { + return nil, fmt.Errorf("interface is held by a different instance slot") + } + var claim networkingv1alpha.NetworkInterfaceClaim + if err := reader.Get(ctx, client.ObjectKey{Namespace: current.Namespace, Name: iface.Spec.ClaimRef.Name}, &claim); err != nil { + return nil, err + } + if claim.UID == "" || !claim.DeletionTimestamp.IsZero() || claim.Spec.Network.Name != iface.Spec.Network.Name || claim.Status.NetworkInterfaceRef == nil || claim.Status.NetworkInterfaceRef.Name != iface.Name { + return nil, nil + } + return &iface, nil +} + +func observeRuntimeNode(ctx context.Context, reader client.Reader, current *computev1alpha.Instance) (string, error) { + var pods corev1.PodList + if err := reader.List(ctx, &pods, client.InNamespace(current.Namespace)); err != nil { + return "", err + } + var nodeName string + for i := range pods.Items { + pod := &pods.Items[i] + if metav1.IsControlledBy(pod, current) && pod.DeletionTimestamp.IsZero() && pod.Status.Phase == corev1.PodRunning && pod.Spec.NodeName != "" { + if nodeName != "" { + return "", fmt.Errorf("multiple running runtime Pods own this Instance") + } + nodeName = pod.Spec.NodeName + } + } + return nodeName, nil +} + +func observeNodeDeadline(ctx context.Context, reader client.Reader, nodeName string, now time.Time) (time.Time, error) { + var node corev1.Node + if err := reader.Get(ctx, client.ObjectKey{Name: nodeName}, &node); err != nil { + return time.Time{}, err + } + ready := false + for _, c := range node.Status.Conditions { + ready = ready || (c.Type == corev1.NodeReady && c.Status == corev1.ConditionTrue) + } + if !ready || node.UID == "" || !node.DeletionTimestamp.IsZero() { + return time.Time{}, nil + } + var lease coordinationv1.Lease + if err := reader.Get(ctx, client.ObjectKey{Namespace: "kube-node-lease", Name: node.Name}, &lease); err != nil { + return time.Time{}, err + } + owned := false + for _, owner := range lease.OwnerReferences { + owned = owned || (owner.Kind == "Node" && owner.UID == node.UID && owner.Name == node.Name) + } + if !owned || lease.Spec.RenewTime == nil || !lease.DeletionTimestamp.IsZero() { + return time.Time{}, nil + } + renewed := lease.Spec.RenewTime.Time + deadline := renewed.Add(dnsNodeObservationLifetime) + if renewed.After(now.Add(5*time.Second)) || !deadline.After(now) { + return time.Time{}, nil + } + return deadline, nil +} diff --git a/internal/controller/internaldns_observation_test.go b/internal/controller/internaldns_observation_test.go new file mode 100644 index 00000000..4dec9882 --- /dev/null +++ b/internal/controller/internaldns_observation_test.go @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + computev1alpha "go.datum.net/compute/api/v1alpha" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestObserveInternalDNSInterface(t *testing.T) { + now := time.Date(2026, 10, 9, 20, 0, 0, 0, time.UTC) + for _, tc := range []struct { + name string + mutate func(*computev1alpha.Instance, *networkingv1alpha.NetworkInterface, *corev1.Pod, *corev1.Node, *coordinationv1.Lease) + eligible bool + wantErr bool + networkUID types.UID + }{ + {name: "live network with unready application", eligible: true}, + {name: "replaced network lifetime", networkUID: "replacement-network", wantErr: true}, + {name: "different instance lifetime", mutate: func(i *computev1alpha.Instance, _ *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node, _ *coordinationv1.Lease) { + i.UID = "replacement" + }, wantErr: true}, + {name: "stale programming generation", mutate: func(_ *computev1alpha.Instance, i *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node, _ *coordinationv1.Lease) { + i.Generation++ + }}, + {name: "different network", mutate: func(_ *computev1alpha.Instance, i *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node, _ *coordinationv1.Lease) { + i.Spec.Network.Name = "other" + }}, + {name: "different claim", mutate: func(_ *computev1alpha.Instance, i *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node, _ *coordinationv1.Lease) { + i.Spec.ClaimRef.Name = "other" + }, wantErr: true}, + {name: "runtime exited", mutate: func(_ *computev1alpha.Instance, _ *networkingv1alpha.NetworkInterface, p *corev1.Pod, _ *corev1.Node, _ *coordinationv1.Lease) { + p.Status.Phase = corev1.PodFailed + }}, + {name: "node unavailable", mutate: func(_ *computev1alpha.Instance, _ *networkingv1alpha.NetworkInterface, _ *corev1.Pod, n *corev1.Node, _ *coordinationv1.Lease) { + n.Status.Conditions[0].Status = corev1.ConditionFalse + }}, + {name: "expired heartbeat", mutate: func(_ *computev1alpha.Instance, _ *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node, l *coordinationv1.Lease) { + l.Spec.RenewTime = &metav1.MicroTime{Time: now.Add(-time.Minute)} + }}, + {name: "future heartbeat", mutate: func(_ *computev1alpha.Instance, _ *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node, l *coordinationv1.Lease) { + l.Spec.RenewTime = &metav1.MicroTime{Time: now.Add(time.Minute)} + }}, + {name: "different node lifetime", mutate: func(_ *computev1alpha.Instance, _ *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node, l *coordinationv1.Lease) { + l.OwnerReferences[0].UID = "other-node" + }}, + } { + t.Run(tc.name, func(t *testing.T) { + scheme := runtime.NewScheme() + for _, add := range []func(*runtime.Scheme) error{corev1.AddToScheme, coordinationv1.AddToScheme, computev1alpha.AddToScheme, networkingv1alpha.AddToScheme} { + require.NoError(t, add(scheme)) + } + instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: "web", Namespace: "project", UID: "project-instance", Labels: map[string]string{computev1alpha.InstanceSourceUIDLabel: "edge-instance", computev1alpha.LocationLabel: "dfw"}}, Spec: computev1alpha.InstanceSpec{NetworkInterfaces: []computev1alpha.InstanceNetworkInterface{{Network: networkingv1alpha.NetworkRef{Name: "application"}}}}} + edgeInstance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: instance.Name, Namespace: "ns-project-namespace", UID: "edge-instance"}, Status: computev1alpha.InstanceStatus{NetworkInterfaces: []computev1alpha.InstanceNetworkInterfaceStatus{{Name: "eth0", NetworkInterfaceRef: &networkingv1alpha.LocalNetworkInterfaceRef{Name: "interface"}}}}} + iface := &networkingv1alpha.NetworkInterface{ObjectMeta: metav1.ObjectMeta{Name: "interface", Namespace: edgeInstance.Namespace, UID: "interface-uid", Generation: 2}, Spec: networkingv1alpha.NetworkInterfaceSpec{Network: networkingv1alpha.LocalNetworkRef{Name: "application"}, InterfaceName: "eth0", ClaimRef: &networkingv1alpha.NetworkInterfaceClaimRef{Name: networkInterfaceClaimName("web", "eth0")}, Addresses: []networkingv1alpha.NetworkInterfaceAddress{{Address: "10.20.0.10/32"}}}, Status: networkingv1alpha.NetworkInterfaceStatus{Conditions: []metav1.Condition{{Type: "Allocated", Status: metav1.ConditionTrue, ObservedGeneration: 2}, {Type: "Programmed", Status: metav1.ConditionTrue, ObservedGeneration: 2}}}} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "runtime", Namespace: edgeInstance.Namespace, OwnerReferences: []metav1.OwnerReference{{APIVersion: computev1alpha.GroupVersion.String(), Kind: "Instance", Name: edgeInstance.Name, UID: edgeInstance.UID, Controller: ptrBool(true)}}}, Spec: corev1.PodSpec{NodeName: "worker"}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionFalse}}}} + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "worker", UID: "node-uid"}, Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}}} + lease := &coordinationv1.Lease{ObjectMeta: metav1.ObjectMeta{Name: node.Name, Namespace: "kube-node-lease", OwnerReferences: []metav1.OwnerReference{{Kind: "Node", Name: node.Name, UID: node.UID}}}, Spec: coordinationv1.LeaseSpec{RenewTime: &metav1.MicroTime{Time: now.Add(-10 * time.Second)}}} + if tc.mutate != nil { + tc.mutate(edgeInstance, iface, pod, node, lease) + } + iface.Status.NetworkContextRef = &networkingv1alpha.LocalNetworkContextRef{Name: "context"} + networkUID := types.UID("network-uid") + if tc.networkUID != "" { + networkUID = tc.networkUID + } + network := &networkingv1alpha.Network{ObjectMeta: metav1.ObjectMeta{Name: "application", Namespace: "project", UID: networkUID}} + project := fake.NewClientBuilder().WithScheme(scheme).WithObjects(network, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "project", UID: "project-namespace"}}).Build() + claim := &networkingv1alpha.NetworkInterfaceClaim{ObjectMeta: metav1.ObjectMeta{Name: networkInterfaceClaimName("web", "eth0"), Namespace: edgeInstance.Namespace, UID: "claim-uid"}, Spec: networkingv1alpha.NetworkInterfaceClaimSpec{Network: networkingv1alpha.LocalNetworkRef{Name: "application"}}, Status: networkingv1alpha.NetworkInterfaceClaimStatus{NetworkInterfaceRef: &networkingv1alpha.LocalNetworkInterfaceRef{Name: iface.Name}}} + networkContext := &networkingv1alpha.NetworkContext{ObjectMeta: metav1.ObjectMeta{Name: "context", Namespace: edgeInstance.Namespace, UID: "network-context", Labels: map[string]string{networkingv1alpha.NetworkUIDLabel: "network-uid"}}, Spec: networkingv1alpha.NetworkContextSpec{Network: networkingv1alpha.LocalNetworkRef{Name: "application"}, Location: networkingv1alpha.LocationReference{Name: "dfw"}}} + source := fake.NewClientBuilder().WithScheme(scheme).WithObjects(edgeInstance, iface, claim, networkContext, pod, node, lease, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "kube-system", UID: "edge-cluster"}}).Build() + got, err := ObserveInternalDNSInterface(context.Background(), project, "project", instance, instance.Spec.NetworkInterfaces[0], InternalDNSObservationSource{Reader: source, ClusterUID: "edge-cluster"}, now) + if tc.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + require.Equal(t, tc.eligible, got.Eligible) + if tc.eligible { + require.WithinDuration(t, now.Add(35*time.Second), got.ValidUntil, time.Nanosecond) + require.Equal(t, iface.Spec.Addresses, got.Addresses) + } + }) + } +} + +func TestObserveInternalDNSInterfaceRejectsMissingSource(t *testing.T) { + _, err := ObserveInternalDNSInterface(context.Background(), nil, "project", &computev1alpha.Instance{}, computev1alpha.InstanceNetworkInterface{}, InternalDNSObservationSource{}, time.Now()) + require.Error(t, err) +} diff --git a/internal/controller/internaldns_publisher.go b/internal/controller/internaldns_publisher.go index 39259ef0..d7521752 100644 --- a/internal/controller/internaldns_publisher.go +++ b/internal/controller/internaldns_publisher.go @@ -15,7 +15,6 @@ import ( "time" apierrors "k8s.io/apimachinery/pkg/api/errors" - apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -51,13 +50,14 @@ const ( internalDNSPendingWait = 10 * time.Second internalDNSAPITimeout = 10 * time.Second defaultDNSConcurrency = 4 + defaultInternalDNSDomain = "datum.internal" ) var ( - dnsManagedNamespaceGVK = schema.GroupVersionKind{Group: internalDNSGroup, Version: internalDNSVersion, Kind: "DNSManagedNamespace"} - dnsRegistrationGVK = schema.GroupVersionKind{Group: internalDNSGroup, Version: internalDNSVersion, Kind: "DNSRegistration"} - dnsGrantGVK = schema.GroupVersionKind{Group: internalDNSGroup, Version: internalDNSVersion, Kind: "DNSContributionGrant"} - dnsContributionGVK = schema.GroupVersionKind{Group: internalDNSGroup, Version: internalDNSVersion, Kind: "DNSRecordContribution"} + dnsResolverContextGVK = schema.GroupVersionKind{Group: internalDNSGroup, Version: internalDNSVersion, Kind: "DNSResolverContext"} + dnsRegistrationGVK = schema.GroupVersionKind{Group: internalDNSGroup, Version: internalDNSVersion, Kind: "DNSRegistration"} + dnsGrantGVK = schema.GroupVersionKind{Group: internalDNSGroup, Version: internalDNSVersion, Kind: "DNSContributionGrant"} + dnsContributionGVK = schema.GroupVersionKind{Group: internalDNSGroup, Version: internalDNSVersion, Kind: "DNSRecordContribution"} ) // InternalDNSProjectIdentity is the trusted identity shared with the DNS @@ -101,6 +101,8 @@ type InternalDNSPublisherReconciler struct { LeaseDuration time.Duration MaxConcurrentReconciles int Now func() time.Time + ObservationSources map[string]InternalDNSObservationSource + ObserveInterface InternalDNSObserveFunc } func (r *InternalDNSPublisherReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) { @@ -162,41 +164,50 @@ func (r *InternalDNSPublisherReconciler) Reconcile(ctx context.Context, req mcre return ctrl.Result{}, r.deletePublishedResources(ctx, access, projectNamespace, projectInstance.UID, nil) } - attachments, err := instanceDNSAttachments(ctx, access.Reader, projectNamespace, &projectInstance) + attachments, err := instanceDNSAttachments(ctx, access.Reader, projectNamespace, &projectInstance, r.observeInterface) if err != nil { return ctrl.Result{}, err } desired := make(map[string]struct{}, len(attachments)) + publicationStatus := make([]computev1alpha.InstanceDNSStatus, 0, len(attachments)) + var publicationErr error pending := false for _, attachment := range attachments { resourceName := internalDNSResourceName(projectInstance.UID, attachment.VPCUID) desired[resourceName] = struct{}{} ready, err := r.reconcileAttachment(ctx, access, identity, projectNamespace, &projectInstance, owner, attachment, resourceName) + publication, statusErr := r.publicationStatus(ctx, access.Reader, projectNamespace, &projectInstance, attachment, resourceName, ready, err) + publicationStatus = append(publicationStatus, publication) + if statusErr != nil { + publicationErr = errors.Join(publicationErr, statusErr) + } if err != nil { - return ctrl.Result{}, err + publicationErr = errors.Join(publicationErr, err) } pending = pending || !ready } + if err := r.reportPublication(ctx, access.Writer, &projectInstance, publicationStatus); err != nil { + return ctrl.Result{}, err + } if err := r.deletePublishedResources(ctx, access, projectNamespace, projectInstance.UID, desired); err != nil { return ctrl.Result{}, err } if pending { - return ctrl.Result{RequeueAfter: internalDNSPendingWait}, nil + return ctrl.Result{RequeueAfter: internalDNSPendingWait}, publicationErr } - return ctrl.Result{RequeueAfter: r.leaseDuration() / 2}, nil + return ctrl.Result{RequeueAfter: r.leaseDuration() / 3}, publicationErr } type instanceDNSAttachment struct { - VPCUID types.UID - RecordSets []any - Eligible bool + VPCUID types.UID + Network networkingv1alpha.NetworkRef + RecordSets []any + Eligible bool + ValidUntil time.Time + ObservationError error } -func instanceDNSAttachments(ctx context.Context, reader client.Reader, projectNamespace string, instance *computev1alpha.Instance) ([]instanceDNSAttachment, error) { - statusByName := make(map[string]computev1alpha.InstanceNetworkInterfaceStatus, len(instance.Status.NetworkInterfaces)) - for _, status := range instance.Status.NetworkInterfaces { - statusByName[status.Name] = status - } +func instanceDNSAttachments(ctx context.Context, reader client.Reader, projectNamespace string, instance *computev1alpha.Instance, observe InternalDNSObserveFunc) ([]instanceDNSAttachment, error) { byVPC := map[types.UID]*instanceDNSAttachment{} for _, nic := range instance.Spec.NetworkInterfaces { namespace := nic.Network.Namespace @@ -212,16 +223,27 @@ func instanceDNSAttachments(ctx context.Context, reader client.Reader, projectNa } attachment := byVPC[network.UID] if attachment == nil { - attachment = &instanceDNSAttachment{VPCUID: network.UID, RecordSets: []any{}} + attachment = &instanceDNSAttachment{VPCUID: network.UID, Network: networkingv1alpha.NetworkRef{Name: network.Name, Namespace: namespace}, RecordSets: []any{}} byVPC[network.UID] = attachment } - status, found := statusByName[internalDNSInterfaceName(nic)] - if !found || !interfaceDNSPublishable(status) { + if observe == nil { + attachment.ObservationError = fmt.Errorf("no current interface observation is available") + continue + } + observation, err := observe(ctx, reader, projectNamespace, instance, nic) + if err != nil { + attachment.ObservationError = errors.Join(attachment.ObservationError, err) + continue + } + if !observation.Eligible { continue } attachment.Eligible = true - for _, address := range status.Addresses { + if attachment.ValidUntil.IsZero() || observation.ValidUntil.Before(attachment.ValidUntil) { + attachment.ValidUntil = observation.ValidUntil + } + for _, address := range observation.Addresses { addr, ok := hostAddress(address.Address) if !ok { continue @@ -232,7 +254,7 @@ func instanceDNSAttachments(ctx context.Context, reader client.Reader, projectNa } attachment.RecordSets = append(attachment.RecordSets, map[string]any{ "recordType": rrtype, - "records": []any{map[string]any{internalDNSNameField: "", field: map[string]any{"content": addr.String()}}}, + "records": []any{map[string]any{internalDNSNameField: "", "ttl": int64(30), field: map[string]any{"content": addr.String()}}}, }) } } @@ -256,12 +278,6 @@ func internalDNSInterfaceName(nic computev1alpha.InstanceNetworkInterface) strin return defaultInterfaceName } -func interfaceDNSPublishable(status computev1alpha.InstanceNetworkInterfaceStatus) bool { - allocated := apimeta.FindStatusCondition(status.Conditions, computev1alpha.InstanceNetworkInterfaceAllocated) - programmed := apimeta.FindStatusCondition(status.Conditions, computev1alpha.InstanceNetworkInterfaceProgrammed) - return allocated != nil && allocated.Status == metav1.ConditionTrue && programmed != nil && programmed.Status == metav1.ConditionTrue -} - func hostAddress(raw string) (netip.Addr, bool) { if addr, err := netip.ParseAddr(raw); err == nil { return addr.Unmap(), true @@ -274,14 +290,18 @@ func hostAddress(raw string) (netip.Addr, bool) { } func (r *InternalDNSPublisherReconciler) reconcileAttachment(ctx context.Context, access InternalDNSProjectAccess, identity InternalDNSProjectIdentity, namespace string, instance *computev1alpha.Instance, owner metav1.OwnerReference, attachment instanceDNSAttachment, resourceName string) (bool, error) { - managed, err := findManagedNamespace(ctx, access.Reader, namespace, identity.ProjectUID, attachment.VPCUID) + managed, err := findResolverContext(ctx, access.Reader, namespace, identity.ProjectUID, attachment.VPCUID) if err != nil { return false, err } if managed == nil { return false, nil } - zoneRef, _, _ := unstructured.NestedMap(managed.Object, internalDNSStatusField, "dnsZoneRef") + suffix, _, _ := unstructured.NestedString(managed.Object, internalDNSStatusField, "managedNamespace", "suffix") + if strings.TrimSuffix(strings.ToLower(suffix), ".") != defaultInternalDNSDomain { + return false, fmt.Errorf("DNS context %q must allocate the default domain %q", managed.GetName(), defaultInternalDNSDomain) + } + zoneRef, _, _ := unstructured.NestedMap(managed.Object, internalDNSStatusField, "managedNamespace", "dnsZoneRef") if stringValue(zoneRef, internalDNSNameField) == "" || stringValue(zoneRef, internalDNSUIDField) == "" { return false, nil } @@ -311,21 +331,15 @@ func (r *InternalDNSPublisherReconciler) reconcileAttachment(ctx context.Context if err != nil || changed { return false, err } - registrationRef := objectReference(registration, true) + registrationRef := objectReference(registration) - grantSpec := map[string]any{ - "registrationRef": registrationRef, - "producerID": internalDNSManager, - "principal": map[string]any{ - "clusterUID": identity.SourceClusterUID, - "subject": r.PrincipalSubject, - }, - "recordTypes": []any{"A", internalDNSRecordTypeAAAA}, - } - grant, changed, err := ensureDNSObject(ctx, access, dnsGrantGVK, namespace, resourceName, labels, owner, grantSpec) - if err != nil || changed { + grant, err := findContributionGrant(ctx, access.Reader, namespace, registration, identity.SourceClusterUID, r.PrincipalSubject, ownerName) + if err != nil { return false, err } + if grant == nil { + return false, nil + } writerEpoch, _, _ := unstructured.NestedInt64(grant.Object, internalDNSStatusField, "activeWriterEpoch") observedGrant, _, _ := unstructured.NestedInt64(grant.Object, internalDNSStatusField, "observedGrantGeneration") observedRegistration, _, _ := unstructured.NestedInt64(grant.Object, internalDNSStatusField, "observedRegistrationGeneration") @@ -335,7 +349,7 @@ func (r *InternalDNSPublisherReconciler) reconcileAttachment(ctx context.Context contributionSpec := map[string]any{ "registrationRef": registrationRef, - "grantRef": objectReference(grant, false), + "grantRef": objectReference(grant), "recordSets": attachment.RecordSets, } contribution, changed, err := ensureDNSObject(ctx, access, dnsContributionGVK, namespace, resourceName, labels, owner, contributionSpec) @@ -347,19 +361,27 @@ func (r *InternalDNSPublisherReconciler) reconcileAttachment(ctx context.Context return false, nil } - return r.refreshObservation(ctx, access.Writer, contribution, writerEpoch, attachment.Eligible) + return r.refreshObservation(ctx, access.Writer, contribution, writerEpoch, attachment.Eligible, attachment.ValidUntil) } -func (r *InternalDNSPublisherReconciler) refreshObservation(ctx context.Context, writer client.Client, contribution *unstructured.Unstructured, writerEpoch int64, eligible bool) (bool, error) { +func (r *InternalDNSPublisherReconciler) refreshObservation(ctx context.Context, writer client.Client, contribution *unstructured.Unstructured, writerEpoch int64, eligible bool, deadline time.Time) (bool, error) { now := time.Now() if r.Now != nil { now = r.Now() } + validThrough := now.Add(r.leaseDuration()) + if eligible { + if deadline.IsZero() || !deadline.After(now) { + eligible = false + } else if deadline.Before(validThrough) { + validThrough = deadline + } + } oldEligible, _, _ := unstructured.NestedBool(contribution.Object, internalDNSStatusField, "eligible") observedGeneration, _, _ := unstructured.NestedInt64(contribution.Object, internalDNSStatusField, "observedGeneration") validUntilRaw, _, _ := unstructured.NestedString(contribution.Object, internalDNSStatusField, "validUntil") validUntil, _ := time.Parse(time.RFC3339Nano, validUntilRaw) - if oldEligible == eligible && observedGeneration == contribution.GetGeneration() && validUntil.After(now.Add(r.leaseDuration()/2)) { + if oldEligible == eligible && observedGeneration == contribution.GetGeneration() && validUntil.After(now.Add(r.leaseDuration()/3)) && !validUntil.After(validThrough) { return true, nil } @@ -374,7 +396,7 @@ func (r *InternalDNSPublisherReconciler) refreshObservation(ctx context.Context, {[]string{internalDNSStatusField, "sequence"}, sequence + 1}, {[]string{internalDNSStatusField, "eligible"}, eligible}, {[]string{internalDNSStatusField, internalDNSReasonField}, map[bool]string{true: "AddressAllocatedAndProgrammed", false: "AddressUnavailable"}[eligible]}, - {[]string{internalDNSStatusField, "validUntil"}, now.Add(r.leaseDuration()).UTC().Format(time.RFC3339Nano)}, + {[]string{internalDNSStatusField, "validUntil"}, validThrough.UTC().Format(time.RFC3339Nano)}, } for _, field := range owned { if err := unstructured.SetNestedField(contribution.Object, field.value, field.path...); err != nil { @@ -390,36 +412,74 @@ func (r *InternalDNSPublisherReconciler) refreshObservation(ctx context.Context, return true, nil } -func findManagedNamespace(ctx context.Context, reader client.Reader, namespace string, projectUID, vpcUID types.UID) (*unstructured.Unstructured, error) { +func findResolverContext(ctx context.Context, reader client.Reader, namespace string, projectUID, vpcUID types.UID) (*unstructured.Unstructured, error) { list := &unstructured.UnstructuredList{} - list.SetGroupVersionKind(dnsManagedNamespaceGVK.GroupVersion().WithKind(dnsManagedNamespaceGVK.Kind + "List")) + list.SetGroupVersionKind(dnsResolverContextGVK.GroupVersion().WithKind(dnsResolverContextGVK.Kind + "List")) if err := reader.List(ctx, list, client.InNamespace(namespace)); err != nil { - return nil, fmt.Errorf("list managed DNS namespaces: %w", err) + return nil, fmt.Errorf("list DNS resolver contexts: %w", err) } var match *unstructured.Unstructured for i := range list.Items { item := &list.Items[i] - if !item.GetDeletionTimestamp().IsZero() { + if !item.GetDeletionTimestamp().IsZero() || item.GetUID() == "" || item.GetGeneration() < 1 { continue } - puid, _, _ := unstructured.NestedString(item.Object, "spec", "projectUID") - vid, _, _ := unstructured.NestedString(item.Object, "spec", "vpcRef", internalDNSUIDField) - if puid != string(projectUID) || vid != string(vpcUID) || !unstructuredConditionTrue(item, "Accepted") { + consumerID, _, _ := unstructured.NestedString(item.Object, "spec", "consumerID") + if consumerID != string(projectUID)+"/"+string(vpcUID) || !unstructuredConditionCurrent(item, "Accepted") || !unstructuredConditionCurrent(item, "Ready") { continue } if match != nil { - return nil, fmt.Errorf("multiple accepted managed DNS namespaces match project UID %q and VPC UID %q", projectUID, vpcUID) + return nil, fmt.Errorf("multiple ready DNS resolver contexts match project UID %q and VPC UID %q", projectUID, vpcUID) } match = item.DeepCopy() } return match, nil } -func unstructuredConditionTrue(obj *unstructured.Unstructured, conditionType string) bool { +func unstructuredConditionCurrent(obj *unstructured.Unstructured, conditionType string) bool { conditions, _, _ := unstructured.NestedSlice(obj.Object, internalDNSStatusField, "conditions") for _, raw := range conditions { condition, ok := raw.(map[string]any) - if ok && condition["type"] == conditionType && condition[internalDNSStatusField] == string(metav1.ConditionTrue) { + if ok && condition["type"] == conditionType && condition[internalDNSStatusField] == string(metav1.ConditionTrue) && condition["observedGeneration"] == obj.GetGeneration() { + return true + } + } + return false +} + +func findContributionGrant(ctx context.Context, reader client.Reader, namespace string, registration *unstructured.Unstructured, clusterUID, subject, ownerName string) (*unstructured.Unstructured, error) { + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(dnsGrantGVK.GroupVersion().WithKind(dnsGrantGVK.Kind + "List")) + if err := reader.List(ctx, list, client.InNamespace(namespace)); err != nil { + return nil, fmt.Errorf("list DNS contribution grants: %w", err) + } + var match *unstructured.Unstructured + for i := range list.Items { + grant := &list.Items[i] + if !grant.GetDeletionTimestamp().IsZero() || grant.GetUID() == "" || grant.GetGeneration() < 1 || !unstructuredConditionCurrent(grant, "Active") { + continue + } + ref, _, _ := unstructured.NestedMap(grant.Object, "spec", "registrationRef") + principal, _, _ := unstructured.NestedMap(grant.Object, "spec", "principal") + if !reflect.DeepEqual(ref, objectReference(registration)) || stringValue(principal, "clusterUID") != clusterUID || stringValue(principal, "subject") != subject { + continue + } + scopes, _, _ := unstructured.NestedStringSlice(grant.Object, "spec", "nameScopes") + types, _, _ := unstructured.NestedStringSlice(grant.Object, "spec", "recordTypes") + if !containsDNSString(scopes, ownerName) || !containsDNSString(types, "A") || !containsDNSString(types, "AAAA") { + continue + } + if match != nil { + return nil, fmt.Errorf("multiple contribution grants authorize registration %q", registration.GetName()) + } + match = grant.DeepCopy() + } + return match, nil +} + +func containsDNSString(values []string, value string) bool { + for _, candidate := range values { + if candidate == value { return true } } @@ -487,7 +547,7 @@ func authorityReferencesChanged(gvk schema.GroupVersionKind, current, desired ma func (r *InternalDNSPublisherReconciler) deletePublishedResources(ctx context.Context, access InternalDNSProjectAccess, namespace string, instanceUID types.UID, keep map[string]struct{}) error { selector := client.MatchingLabels{internalDNSInstanceUID: string(instanceUID), internalDNSManagedBy: internalDNSManager} - for _, gvk := range []schema.GroupVersionKind{dnsContributionGVK, dnsGrantGVK, dnsRegistrationGVK} { + for _, gvk := range []schema.GroupVersionKind{dnsContributionGVK, dnsRegistrationGVK} { list := &unstructured.UnstructuredList{} list.SetGroupVersionKind(gvk.GroupVersion().WithKind(gvk.Kind + "List")) if err := access.Reader.List(ctx, list, client.InNamespace(namespace), selector); err != nil { @@ -514,11 +574,9 @@ func newDNSObject(gvk schema.GroupVersionKind, namespace, name string) *unstruct return obj } -func objectReference(obj *unstructured.Unstructured, includeGeneration bool) map[string]any { +func objectReference(obj *unstructured.Unstructured) map[string]any { ref := map[string]any{internalDNSNameField: obj.GetName(), internalDNSUIDField: string(obj.GetUID())} - if includeGeneration { - ref["generation"] = obj.GetGeneration() - } + ref["generation"] = obj.GetGeneration() return ref } @@ -544,7 +602,7 @@ func allocatedInstanceDNSName(instance *computev1alpha.Instance) string { if len(base) > 40 { base = strings.TrimRight(base[:40], "-") } - return base + "-" + shortHash(string(instance.UID)) + ".instances" + return base + "-" + shortHash(string(instance.UID)) } func internalDNSResourceName(instanceUID, vpcUID types.UID) string { diff --git a/internal/controller/internaldns_publisher_test.go b/internal/controller/internaldns_publisher_test.go index 7838bb4b..88f7eafb 100644 --- a/internal/controller/internaldns_publisher_test.go +++ b/internal/controller/internaldns_publisher_test.go @@ -53,15 +53,19 @@ func TestInstanceDNSAttachmentsMapsTenancyAndAddressFamilies(t *testing.T) { }, } - attachments, err := instanceDNSAttachments(context.Background(), cl, internalDNSTestNamespace, instance) + eligible := true + observation := func(context.Context, client.Reader, string, *computev1alpha.Instance, computev1alpha.InstanceNetworkInterface) (InternalDNSObservedInterface, error) { + return InternalDNSObservedInterface{Addresses: []networkingv1alpha.NetworkInterfaceAddress{{Address: "10.0.0.8/32"}, {Address: "2001:db8::8/128"}, {Address: "2001:db8:1::/96"}}, Eligible: eligible, ValidUntil: time.Now().Add(time.Minute)}, nil + } + attachments, err := instanceDNSAttachments(context.Background(), cl, internalDNSTestNamespace, instance, observation) require.NoError(t, err) require.Len(t, attachments, 1) assert.Equal(t, types.UID("vpc-uid"), attachments[0].VPCUID) assert.True(t, attachments[0].Eligible) assert.Len(t, attachments[0].RecordSets, 2, "A and AAAA host addresses should be retained") - instance.Status.NetworkInterfaces[0].Conditions[1].Status = metav1.ConditionFalse - attachments, err = instanceDNSAttachments(context.Background(), cl, internalDNSTestNamespace, instance) + eligible = false + attachments, err = instanceDNSAttachments(context.Background(), cl, internalDNSTestNamespace, instance, observation) require.NoError(t, err) require.Len(t, attachments, 1) assert.False(t, attachments[0].Eligible) @@ -69,58 +73,60 @@ func TestInstanceDNSAttachmentsMapsTenancyAndAddressFamilies(t *testing.T) { assert.Empty(t, attachments[0].RecordSets) } -func TestFindManagedNamespaceRequiresProjectAndVPCUID(t *testing.T) { +func TestFindResolverContextRequiresProjectAndVPCUID(t *testing.T) { s := runtime.NewScheme() - s.AddKnownTypeWithName(dnsManagedNamespaceGVK, &unstructured.Unstructured{}) - s.AddKnownTypeWithName(dnsManagedNamespaceGVK.GroupVersion().WithKind("DNSManagedNamespaceList"), &unstructured.UnstructuredList{}) - wrong := managedNamespaceObject("wrong", "other-project", "vpc-a", true) - right := managedNamespaceObject("right", "project-a", "vpc-a", true) - wrongVPC := managedNamespaceObject("wrong-vpc", "project-a", "vpc-b", true) - pending := managedNamespaceObject("pending", "project-a", "vpc-a", false) + s.AddKnownTypeWithName(dnsResolverContextGVK, &unstructured.Unstructured{}) + s.AddKnownTypeWithName(dnsResolverContextGVK.GroupVersion().WithKind("DNSResolverContextList"), &unstructured.UnstructuredList{}) + wrong := resolverContextObject("wrong", "other-project", "vpc-a", true) + right := resolverContextObject("right", "project-a", "vpc-a", true) + wrongVPC := resolverContextObject("wrong-vpc", "project-a", "vpc-b", true) + pending := resolverContextObject("pending", "project-a", "vpc-a", false) cl := fake.NewClientBuilder().WithScheme(s).WithObjects(wrong, wrongVPC, pending, right).Build() - got, err := findManagedNamespace(context.Background(), cl, internalDNSTestNamespace, "project-a", "vpc-a") + got, err := findResolverContext(context.Background(), cl, internalDNSTestNamespace, "project-a", "vpc-a") require.NoError(t, err) require.NotNil(t, got) assert.Equal(t, "right", got.GetName()) - got, err = findManagedNamespace(context.Background(), cl, internalDNSTestNamespace, "project-b", "vpc-a") + got, err = findResolverContext(context.Background(), cl, internalDNSTestNamespace, "project-b", "vpc-a") require.NoError(t, err) assert.Nil(t, got) } -func TestFindManagedNamespaceRejectsAmbiguityAndIgnoresTerminatingBinding(t *testing.T) { +func TestFindResolverContextRejectsAmbiguityAndIgnoresTerminatingBinding(t *testing.T) { s := runtime.NewScheme() - s.AddKnownTypeWithName(dnsManagedNamespaceGVK, &unstructured.Unstructured{}) - s.AddKnownTypeWithName(dnsManagedNamespaceGVK.GroupVersion().WithKind("DNSManagedNamespaceList"), &unstructured.UnstructuredList{}) - first := managedNamespaceObject("first", "project-a", "vpc-a", true) - second := managedNamespaceObject("second", "project-a", "vpc-a", true) + s.AddKnownTypeWithName(dnsResolverContextGVK, &unstructured.Unstructured{}) + s.AddKnownTypeWithName(dnsResolverContextGVK.GroupVersion().WithKind("DNSResolverContextList"), &unstructured.UnstructuredList{}) + first := resolverContextObject("first", "project-a", "vpc-a", true) + second := resolverContextObject("second", "project-a", "vpc-a", true) cl := fake.NewClientBuilder().WithScheme(s).WithObjects(first, second).Build() - _, err := findManagedNamespace(context.Background(), cl, internalDNSTestNamespace, "project-a", "vpc-a") + _, err := findResolverContext(context.Background(), cl, internalDNSTestNamespace, "project-a", "vpc-a") require.Error(t, err) - assert.Contains(t, err.Error(), "multiple accepted") + assert.Contains(t, err.Error(), "multiple ready") now := metav1.Now() second.SetDeletionTimestamp(&now) second.SetFinalizers([]string{"test.example/finalizer"}) cl = fake.NewClientBuilder().WithScheme(s).WithObjects(first, second).Build() - got, err := findManagedNamespace(context.Background(), cl, internalDNSTestNamespace, "project-a", "vpc-a") + got, err := findResolverContext(context.Background(), cl, internalDNSTestNamespace, "project-a", "vpc-a") require.NoError(t, err) require.NotNil(t, got) assert.Equal(t, "first", got.GetName()) } -func managedNamespaceObject(name, projectUID, vpcUID string, accepted bool) *unstructured.Unstructured { +func resolverContextObject(name, projectUID, vpcUID string, accepted bool) *unstructured.Unstructured { status := string(metav1.ConditionFalse) if accepted { status = string(metav1.ConditionTrue) } - obj := newDNSObject(dnsManagedNamespaceGVK, internalDNSTestNamespace, name) - obj.Object["spec"] = map[string]any{"projectUID": projectUID, "vpcRef": map[string]any{internalDNSUIDField: vpcUID}} + obj := newDNSObject(dnsResolverContextGVK, internalDNSTestNamespace, name) + obj.SetUID(types.UID(name + "-uid")) + obj.SetGeneration(1) + obj.Object["spec"] = map[string]any{"consumerID": projectUID + "/" + vpcUID} obj.Object[internalDNSStatusField] = map[string]any{ - "conditions": []any{map[string]any{"type": "Accepted", internalDNSStatusField: status}}, - "dnsZoneRef": map[string]any{internalDNSNameField: "zone", internalDNSUIDField: "zone-uid", "generation": int64(1)}, + "conditions": []any{map[string]any{"type": "Accepted", internalDNSStatusField: status, "observedGeneration": int64(1)}, map[string]any{"type": "Ready", internalDNSStatusField: status, "observedGeneration": int64(1)}}, + "managedNamespace": map[string]any{"suffix": "datum.internal", "dnsZoneRef": map[string]any{internalDNSNameField: "zone", internalDNSUIDField: "zone-uid"}}, } return obj } @@ -148,7 +154,7 @@ func TestRefreshObservationPreservesPlatformStatusAndIncrementsFence(t *testing. var live unstructured.Unstructured live.SetGroupVersionKind(dnsContributionGVK) require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(contribution), &live)) - ready, err := r.refreshObservation(context.Background(), cl, &live, 7, true) + ready, err := r.refreshObservation(context.Background(), cl, &live, 7, true, now.Add(40*time.Second)) require.NoError(t, err) assert.True(t, ready) diff --git a/internal/controller/internaldns_status.go b/internal/controller/internaldns_status.go new file mode 100644 index 00000000..c8be645a --- /dev/null +++ b/internal/controller/internaldns_status.go @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "fmt" + "reflect" + "strings" + "time" + + computev1alpha "go.datum.net/compute/api/v1alpha" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func (r *InternalDNSPublisherReconciler) now() time.Time { + if r.Now != nil { + return r.Now() + } + return time.Now() +} + +func (r *InternalDNSPublisherReconciler) observeInterface(ctx context.Context, reader client.Reader, namespace string, instance *computev1alpha.Instance, nic computev1alpha.InstanceNetworkInterface) (InternalDNSObservedInterface, error) { + if r.ObserveInterface != nil { + return r.ObserveInterface(ctx, reader, namespace, instance, nic) + } + return ObserveInternalDNSInterface(ctx, reader, namespace, instance, nic, r.ObservationSources[instance.Labels[computev1alpha.LocationLabel]], r.now()) +} + +func (r *InternalDNSPublisherReconciler) publicationStatus(ctx context.Context, reader client.Reader, namespace string, instance *computev1alpha.Instance, attachment instanceDNSAttachment, resourceName string, authorized bool, reconcileErr error) (computev1alpha.InstanceDNSStatus, error) { + result := computev1alpha.InstanceDNSStatus{Network: attachment.Network, NetworkUID: string(attachment.VPCUID)} + for _, previous := range instance.Status.DNS { + if previous.NetworkUID == result.NetworkUID { + result.Conditions = append([]metav1.Condition(nil), previous.Conditions...) + } + } + condition := metav1.Condition{Type: "Ready", Status: metav1.ConditionFalse, Reason: "PendingPublication", Message: "Waiting for DNS publication", ObservedGeneration: instance.Generation} + registration := newDNSObject(dnsRegistrationGVK, namespace, resourceName) + err := reader.Get(ctx, client.ObjectKeyFromObject(registration), registration) + if err != nil && !apierrors.IsNotFound(err) { + return result, err + } + if err == nil { + canonical, _, _ := unstructured.NestedString(registration.Object, "status", "canonicalFQDN") + canonical = strings.TrimSuffix(strings.ToLower(canonical), ".") + if canonical == allocatedInstanceDNSName(instance)+"."+defaultInternalDNSDomain { + result.Hostnames = []string{canonical} + } + contribution := newDNSObject(dnsContributionGVK, namespace, resourceName) + if err := reader.Get(ctx, client.ObjectKeyFromObject(contribution), contribution); err != nil && !apierrors.IsNotFound(err) { + return result, err + } + deadlineRaw, _, _ := unstructured.NestedString(contribution.Object, "status", "validUntil") + deadline, _ := time.Parse(time.RFC3339Nano, deadlineRaw) + if authorized && len(result.Hostnames) > 0 && unstructuredConditionCurrent(registration, "Published") && unstructuredConditionCurrent(contribution, "Published") && deadline.After(r.now()) && attachment.Eligible && attachment.ObservationError == nil { + condition.Status, condition.Reason, condition.Message = metav1.ConditionTrue, "Published", "Private name is published" + } + } + if attachment.ObservationError != nil { + condition.Reason, condition.Message = "NetworkObservationUnavailable", "No current network observation is available" + } else if !attachment.Eligible { + condition.Reason, condition.Message = "AddressUnavailable", "No currently eligible private host address is available" + } + if reconcileErr != nil { + condition.Reason, condition.Message = "PublicationFailed", "DNS publication could not be reconciled" + } + apimeta.SetStatusCondition(&result.Conditions, condition) + return result, nil +} + +func (r *InternalDNSPublisherReconciler) reportPublication(ctx context.Context, writer client.Client, instance *computev1alpha.Instance, status []computev1alpha.InstanceDNSStatus) error { + if reflect.DeepEqual(instance.Status.DNS, status) { + return nil + } + base := instance.DeepCopy() + instance.Status.DNS = status + if err := writer.Status().Patch(ctx, instance, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { + return fmt.Errorf("report Instance DNS publication: %w", err) + } + return nil +} diff --git a/internal/controller/internaldns_status_test.go b/internal/controller/internaldns_status_test.go new file mode 100644 index 00000000..190dd8d0 --- /dev/null +++ b/internal/controller/internaldns_status_test.go @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + computev1alpha "go.datum.net/compute/api/v1alpha" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestPublicationStatusRequiresCurrentUnexpiredPublication(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: "web", UID: "instance-uid", Generation: 2}} + for _, tc := range []struct { + name string + deadline time.Time + current, authorized, wantReady bool + }{ + {name: "published", deadline: now.Add(time.Minute), current: true, authorized: true, wantReady: true}, + {name: "expired", deadline: now.Add(-time.Second), current: true, authorized: true}, + {name: "stale generation", deadline: now.Add(time.Minute), authorized: true}, + {name: "revoked grant", deadline: now.Add(time.Minute), current: true}, + } { + t.Run(tc.name, func(t *testing.T) { + s := runtime.NewScheme() + s.AddKnownTypeWithName(dnsRegistrationGVK, &unstructured.Unstructured{}) + s.AddKnownTypeWithName(dnsContributionGVK, &unstructured.Unstructured{}) + registration := newDNSObject(dnsRegistrationGVK, internalDNSTestNamespace, "web") + registration.SetGeneration(2) + contribution := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, "web") + contribution.SetGeneration(2) + observed := int64(1) + if tc.current { + observed = 2 + } + conditions := []any{map[string]any{"type": "Published", "status": "True", "observedGeneration": observed}} + registration.Object["status"] = map[string]any{"canonicalFQDN": allocatedInstanceDNSName(instance) + ".datum.internal.", "conditions": conditions} + contribution.Object["status"] = map[string]any{"conditions": conditions, "validUntil": tc.deadline.Format(time.RFC3339Nano)} + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(registration, contribution).Build() + r := &InternalDNSPublisherReconciler{Now: func() time.Time { return now }} + got, err := r.publicationStatus(context.Background(), cl, internalDNSTestNamespace, instance, instanceDNSAttachment{VPCUID: "vpc-a", Eligible: true}, "web", tc.authorized, nil) + require.NoError(t, err) + require.Equal(t, []string{allocatedInstanceDNSName(instance) + ".datum.internal"}, got.Hostnames) + require.Equal(t, tc.wantReady, got.Conditions[0].Status == metav1.ConditionTrue) + }) + } +} + +func TestReportPublicationPreservesRuntimeStatus(t *testing.T) { + s := runtime.NewScheme() + require.NoError(t, computev1alpha.AddToScheme(s)) + instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: "web", Namespace: internalDNSTestNamespace}, Status: computev1alpha.InstanceStatus{Conditions: []metav1.Condition{{Type: computev1alpha.InstanceReady, Status: metav1.ConditionTrue}}}} + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(instance).WithStatusSubresource(instance).Build() + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(instance), instance)) + r := &InternalDNSPublisherReconciler{} + require.NoError(t, r.reportPublication(context.Background(), cl, instance, []computev1alpha.InstanceDNSStatus{{NetworkUID: "vpc-a"}})) + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(instance), instance)) + require.Equal(t, metav1.ConditionTrue, instance.Status.Conditions[0].Status) + require.Len(t, instance.Status.DNS, 1) +} diff --git a/internal/features/features.go b/internal/features/features.go index fe058975..bf03cbca 100644 --- a/internal/features/features.go +++ b/internal/features/features.go @@ -110,8 +110,8 @@ const ( InstanceConsoleSessions featuregate.Feature = "InstanceConsoleSessions" // InternalDNSPublishing enables the instance identity publisher. The - // publisher reads DNS-owned managed namespace bindings and writes - // registrations, grants, and leased contributions to project control planes. + // publisher reads DNS contexts and independently issued grants and writes + // registrations and leased contributions to project control planes. // It is deliberately off by default: an installation with the gate disabled // neither discovers DNS API types nor needs DNS RBAC. // diff --git a/test/internaldns/README.md b/test/internaldns/README.md index 2a1877cd..41a629b5 100644 --- a/test/internaldns/README.md +++ b/test/internaldns/README.md @@ -1,38 +1,46 @@ -# Internal DNS publisher smoke test +# Internal DNS publisher validation -This test runs the production Compute publisher against a disposable Kubernetes -API and the shared DNS serving fleet in the sibling `dns-operator` repository. -The DNS checkout must include the internal DNS implementation and its local test -harness. The test requires Go, Docker Compose, Kind, kubectl, OpenSSL, and an -ARM64 Docker runtime. +The publisher uses the shared Kubernetes internal DNS environment in the +`dns-operator` repository. Use that environment's cluster lifecycle, TLS +admission, NATS transport, and shared serving fleet. Select every API through an +explicit kubeconfig. + +Run the Compute contract tests with Kubernetes envtest assets installed: + +```sh +make envtest +bin/setup-envtest use 1.31.0 --bin-dir bin -p path +go test ./internal/controller ./internal/config ./cmd/... +``` + +The tests cover context identity, flat instance names, independently issued +grants, fenced contribution renewal, live edge observations, source lifetime +replacement, and project DNS status ownership. They use fixtures and do not +qualify guest resolution. + +The standalone runner starts the production publisher on an existing project API: ```sh go build -o /tmp/compute-dns-publisher ./test/internaldns/publisher -python3 test/internaldns/fleet_smoke.py \ - --dns-repo ../dns-operator \ - --publisher-binary /tmp/compute-dns-publisher +/tmp/compute-dns-publisher --feature-gates=InternalDNSPublishing=true \ + --kubeconfig /tmp/project-publisher.kubeconfig \ + --namespace production --project-uid PROJECT_UID \ + --source-uid PROJECT_API_UID \ + --subject system:serviceaccount:compute-system:compute-manager \ + --edge-kubeconfig /tmp/edge-observer.kubeconfig \ + --edge-uid EDGE_KUBE_SYSTEM_UID --location dfw ``` -The runner starts only the production publisher controller. Its default feature -gate is off, and the test first checks that it exits without constructing a -Kubernetes client. The enabled phase uses a scoped Compute service account and -the real DNS admission, publication, transport, and serving components. - -The checks cover IPv4 and IPv6 over UDP and TCP, VPC isolation, application -readiness independence, address changes, allocation withdrawal and recovery, -publisher lease expiry and restart, and instance deletion with DNS resource -cleanup. Serving process identities must remain constant. - -The recorded run passed all 23 checks. Both VPCs served their own Compute -records. A query for the other VPC's managed suffix returned `SERVFAIL` with no -private answer because the fixture's public fallback is unavailable. The test -records the response code and checks isolation; it does not qualify public -recursion or require a specific negative response code. - -Instance objects and allocated-address observations are fixtures. This test does -not boot a guest, allocate addresses through NSO, validate Milo IAM deployment, -or test regional replication. The full Compute binary separately requires Milo -discovery and management controllers when the feature gate is enabled. - -The harness removes its own Kind cluster, containers, volumes, and temporary -files. Results and redacted evidence are saved under `test/internaldns/results`. +Provision a resolver context with a `datum.internal` managed zone, the scoped +project publisher role, and a separate trusted grant issuer. Instances must +carry their source lifetime and location labels. Edge namespaces use the project +namespace UID. The observer requires allocated, programmed interfaces, live +claims, Instance-owned runtime Pods, and current node heartbeat leases. +The disabled runner starts without creating Kubernetes clients. + +Release qualification must exercise two VPCs using the same default domain, +real address allocation, guest resolver application, and UDP/TCP queries through +Galactic. Include address changes, withdrawal, publisher outages, grant +revocation, stale replay, source replacement, and controller restarts. Keep this +qualification local and reuse the shared Kubernetes foundation. The publisher +contract tests and runner do not establish that those release criteria pass. diff --git a/test/internaldns/fleet_smoke.py b/test/internaldns/fleet_smoke.py deleted file mode 100644 index 65140d2f..00000000 --- a/test/internaldns/fleet_smoke.py +++ /dev/null @@ -1,168 +0,0 @@ -#!/usr/bin/env python3 -"""Run the production Compute publisher through the disposable DNS serving fleet.""" -import argparse -import hashlib -import importlib.util -import json -from pathlib import Path -import sys - -ROOT = Path(__file__).resolve().parents[2] - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--dns-repo", type=Path, default=ROOT.parent / "dns-operator") - parser.add_argument("--publisher-binary", type=Path, help="Use a prebuilt test runner") - parser.add_argument("--results", type=Path, default=ROOT / "test/internaldns/results/fleet-smoke.json") - args = parser.parse_args() - dns = args.dns_repo.resolve() - spec = importlib.util.spec_from_file_location("dns_fleet", dns / "test/internaldns/run.py") - fleet = importlib.util.module_from_spec(spec) - spec.loader.exec_module(fleet) - h = fleet.Harness(False, args.results) - error = None - - def obj(kind, name): - return json.loads(h.kubectl("-n", fleet.PROJECT_NS, "get", kind, name, "-o", "json").stdout) - - def status(name, allocated=True, address="10.0.1.30", app_ready=False): - condition = lambda typ, value: {"type": typ, "status": str(value), "reason": "TestObservation", "message": "Smoke fixture", "lastTransitionTime": "2026-10-07T00:00:00Z"} - value = {"status": { - "conditions": [condition("Ready", "True" if app_ready else "False")], - "networkInterfaces": [{"name": "eth0", "addresses": [ - {"family": "IPv4", "address": address + "/32"}, - {"family": "IPv6", "address": "2001:db8::30/128"}, - ], "conditions": [condition("Allocated", "True" if allocated else "False"), condition("Programmed", "True")] }], - }} - h.kubectl("-n", fleet.PROJECT_NS, "patch", "instance", name, "--subresource=status", "--type=merge", "-p", json.dumps(value)) - - def query(server, name, rrtype="A", tcp=False): - output = h.compose("exec", "-T", "probe", "dig", "@" + server, name, rrtype, "+noall", "+comments", "+answer", *(["+tcp"] if tcp else [])).stdout - code = next((line.split("status:", 1)[1].split(",", 1)[0].strip() for line in output.splitlines() if "status:" in line), "UNKNOWN") - answers = [line.split()[-1] for line in output.splitlines() if not line.startswith(";") and len(line.split()) >= 5 and line.split()[-2] == rrtype] - return code, answers - - def expect(label, server, name, answers, rrtype="A", tcp=False, absent=False): - observed = {} - def matches(): - code, found = query(server, name, rrtype, tcp) - observed["status"] = code - return (code in ("NOERROR", "NXDOMAIN", "REFUSED", "SERVFAIL") if absent else code == "NOERROR") and found == answers - h.wait(label, matches, timeout=75, interval=0.5) - h.pass_check(label, name=name, recordType=rrtype, answers=answers, tcp=tcp, **observed) - - try: - binary = args.publisher_binary.resolve() if args.publisher_binary else h.work / "compute-dns-publisher" - if not args.publisher_binary: - h.run("go", "build", "-o", str(binary), "./test/internaldns/publisher", cwd=ROOT) - disabled = h.run(str(binary), "--kubeconfig", "/does/not/exist") - if "disabled" not in disabled.stdout: - raise AssertionError("default-off runner tried to start") - h.pass_check("default-off publisher starts without a Kubernetes/DNS configuration") - h.setup() - h.kubectl("apply", "-f", str(ROOT / "config/base/crd/bases/compute.datumapis.com_instances.yaml")) - role = obj("role", "compute-dns-publisher") - role["rules"].extend([ - {"apiGroups": ["compute.datumapis.com"], "resources": ["instances"], "verbs": ["get", "list", "watch"]}, - {"apiGroups": ["networking.datumapis.com"], "resources": ["networks"], "verbs": ["get"]}, - ]) - h.kubectl("replace", "-f", "-", stdin=json.dumps(role)) - configs = h.write_configs() - h.start("control", *h.command("control-plane", configs["control"])) - h.install_webhook() - h.kubectl("apply", "-f", str(dns / "test/internaldns/fixtures/vpc-a.yaml")) - h.create_association("prod-a-vpc-a", "prod-a", "vpc-a") - h.publish("prod-a", "bootstrap", "bootstrap", "10.0.1.1") - bindings = h.sync_addresses(1) - a_uid = obj("network", "vpc-a")["metadata"]["uid"] - vip_a = bindings[a_uid]["spec"]["consumerAddress"] - h.wait_for_bootstrap() - for member in ("node-0", "cluster-0"): - h.start_container(member, configs[member], network="datum-internal-dns_dns") - for member in ("pdns-0", "pdns-1"): - h.start_container(member, configs[member], network="container:datum-internal-dns-cluster-bind-1") - h.wait_for_agent_leases(("node-0", "cluster-0", "pdns-0", "pdns-1")) - for member in ("node-0", "cluster-0", "pdns-0", "pdns-1"): - h.start_container(member + "-watchdog", configs[member + "-watchdog"], network="none", watchdog=True) - h.kubectl("apply", "-f", str(dns / "test/internaldns/fixtures/vpc-b.yaml")) - h.create_association("prod-b-vpc-b", "prod-b", "vpc-b") - bindings = h.sync_addresses(2) - b_uid = obj("network", "vpc-b")["metadata"]["uid"] - vip_b = bindings[b_uid]["spec"]["consumerAddress"] - process_ids = h.container_ids() - h.start("compute", str(binary), "--feature-gates=InternalDNSPublishing=true", "--kubeconfig", str(h.compute_kubeconfig), "--lease=60s") - instance = {"apiVersion": "compute.datumapis.com/v1alpha", "kind": "Instance", "metadata": {"name": "web-01", "namespace": fleet.PROJECT_NS}, "spec": { - "runtime": {"resources": {"instanceType": "datumcloud/d1-standard-2"}, "sandbox": {"containers": [{"name": "web", "image": "docker.io/library/nginx:stable"}]}}, - "networkInterfaces": [{"name": "eth0", "network": {"name": "vpc-a"}}], - }} - h.kubectl("apply", "-f", "-", stdin=json.dumps(instance)) - status("web-01") - - def publication(instance_name): - uid = obj("instance", instance_name)["metadata"]["uid"] - registrations = json.loads(h.kubectl("-n", fleet.PROJECT_NS, "get", "dnsregistrations", "-l", "internal-dns.compute.datumapis.com/instance-uid=" + uid, "-o", "json").stdout)["items"] - if not registrations: - return None - reg = registrations[0] - zone = obj("dnszone", reg["spec"]["dnsZoneRef"]["name"]) - return reg, reg["spec"]["name"] + "." + zone["spec"]["domainName"].rstrip(".") - reg, fqdn = h.wait("production Compute registration", lambda: publication("web-01")) - instance["metadata"]["name"] = "web-02" - instance["spec"]["networkInterfaces"][0]["network"]["name"] = "vpc-b" - h.kubectl("apply", "-f", "-", stdin=json.dumps(instance)) - status("web-02", address="10.0.1.20") - _, fqdn_b = h.wait("second VPC Compute registration", lambda: publication("web-02")) - for tcp in (False, True): - expect("second VPC serves its own Compute IPv4" + (" TCP" if tcp else " UDP"), vip_b, fqdn_b, ["10.0.1.20"], tcp=tcp) - expect("second VPC serves its own Compute IPv6" + (" TCP" if tcp else " UDP"), vip_b, fqdn_b, ["2001:db8::30"], rrtype="AAAA", tcp=tcp) - for tcp in (False, True): - expect("production Compute IPv4" + (" TCP" if tcp else " UDP"), vip_a, fqdn, ["10.0.1.30"], tcp=tcp) - expect("production Compute IPv6" + (" TCP" if tcp else " UDP"), vip_a, fqdn, ["2001:db8::30"], rrtype="AAAA", tcp=tcp) - expect("other VPC cannot resolve instance" + (" TCP" if tcp else " UDP"), vip_b, fqdn, [], tcp=tcp, absent=True) - h.pass_check("application Ready=False leaves instance identity published") - status("web-01", address="10.0.1.31", app_ready=True) - expect("address update reaches DNS", vip_a, fqdn, ["10.0.1.31"]) - status("web-01", allocated=False) - expect("interface deallocation withdraws IPv4", vip_a, fqdn, [], absent=True) - expect("interface deallocation withdraws IPv6", vip_a, fqdn, [], rrtype="AAAA", absent=True) - expect("other VPC retains its Compute record during withdrawal", vip_b, fqdn_b, ["10.0.1.20"]) - status("web-01") - expect("interface recovery republishes", vip_a, fqdn, ["10.0.1.30"]) - h.stop("compute") - expect("stopped publisher lease expires", vip_a, fqdn, [], absent=True) - h.start("compute", str(binary), "--feature-gates=InternalDNSPublishing=true", "--kubeconfig", str(h.compute_kubeconfig), "--lease=60s") - expect("restarted publisher renews observation", vip_a, fqdn, ["10.0.1.30"]) - h.kubectl("-n", fleet.PROJECT_NS, "delete", "instance", "web-01", "--wait=true") - expect("instance deletion withdraws answer", vip_a, fqdn, [], absent=True) - expect("other VPC retains its Compute record during deletion", vip_b, fqdn_b, ["10.0.1.20"]) - h.kubectl("-n", fleet.PROJECT_NS, "delete", "instance", "web-02", "--wait=true") - h.wait("instance DNS objects garbage collected", lambda: all(not json.loads(h.kubectl("-n", fleet.PROJECT_NS, "get", kind, "-l", "internal-dns.compute.datumapis.com/managed-by=compute-instance-publisher", "-o", "json").stdout)["items"] for kind in ("dnsregistrations", "dnscontributiongrants", "dnsrecordcontributions"))) - h.pass_check("instance deletion cleans registration, grant and contribution") - if process_ids != h.container_ids(): - raise AssertionError("serving process count/identity changed") - h.pass_check("shared serving process identities remain constant") - except BaseException as exc: - error = exc - print("FAIL", exc, file=sys.stderr) - for name in ("compute", "control"): - log = h.logs / (name + ".log") - if log.exists(): - print(name + ": " + log.read_text()[-6000:], file=sys.stderr) - finally: - h.write_results(error) - data = json.loads(args.results.read_text()) - data["suite"] = "compute-internal-dns-fleet-smoke" - data["computePublisherSourceSHA256"] = hashlib.sha256((ROOT / "internal/controller/internaldns_publisher.go").read_bytes()).hexdigest() - if binary.is_file(): - data["computePublisherBinarySHA256"] = hashlib.sha256(binary.read_bytes()).hexdigest() - data["limitations"] = ["Instance addressing/status are fixtures; no guest boots or real NSO allocation.", "One local source API and one local broker; no regional HA or Milo IAM deployment validation.", "Uses the production Compute publisher controller, started by a test runner; default-off production wiring is covered separately."] - args.results.write_text(json.dumps(data, indent=2) + "\n") - h.cleanup() - if error: - raise error - print(f"PASS {len(fleet.RESULTS)} full-path checks") - - -if __name__ == "__main__": - main() diff --git a/test/internaldns/publisher/main.go b/test/internaldns/publisher/main.go index eb10b616..5afb8934 100644 --- a/test/internaldns/publisher/main.go +++ b/test/internaldns/publisher/main.go @@ -15,12 +15,14 @@ import ( "go.datum.net/compute/internal/controller" "go.datum.net/compute/internal/features" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/clientcmd" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -31,6 +33,7 @@ import ( func main() { var kubeconfig, namespace, projectUID, sourceUID, subject, gates string + var edgeKubeconfig, edgeUID, location string var lease time.Duration flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError) flags.StringVar(&kubeconfig, "kubeconfig", "", "Disposable project kubeconfig") @@ -40,6 +43,9 @@ func main() { flags.StringVar(&subject, "subject", "system:serviceaccount:compute-system:dns-publisher", "Authenticated writer") flags.StringVar(&gates, "feature-gates", "", "Compute feature gates") flags.DurationVar(&lease, "lease", 60*time.Second, "Contribution lease") + flags.StringVar(&edgeKubeconfig, "edge-kubeconfig", "", "Read-only edge observation kubeconfig") + flags.StringVar(&edgeUID, "edge-uid", "", "Edge kube-system namespace UID") + flags.StringVar(&location, "location", "", "Instance location label") if err := flags.Parse(os.Args[1:]); err != nil { panic(err) } @@ -51,17 +57,29 @@ func main() { fmt.Println("InternalDNSPublishing is disabled; no DNS client or controller started") return } + if edgeKubeconfig == "" || edgeUID == "" || location == "" { + panic("edge-kubeconfig, edge-uid, and location are required") + } cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig) if err != nil { panic(err) } cfg.Timeout = 10 * time.Second scheme := runtime.NewScheme() - for _, add := range []func(*runtime.Scheme) error{corev1.AddToScheme, computev1alpha.AddToScheme, networkingv1alpha.AddToScheme} { + for _, add := range []func(*runtime.Scheme) error{corev1.AddToScheme, coordinationv1.AddToScheme, computev1alpha.AddToScheme, networkingv1alpha.AddToScheme} { if err := add(scheme); err != nil { panic(err) } } + edgeConfig, err := clientcmd.BuildConfigFromFlags("", edgeKubeconfig) + if err != nil { + panic(err) + } + edgeConfig.Timeout = 5 * time.Second + edge, err := client.New(edgeConfig, client.Options{Scheme: scheme}) + if err != nil { + panic(err) + } cl, err := cluster.New(cfg, func(o *cluster.Options) { o.Scheme = scheme o.Cache.DefaultNamespaces = map[string]cache.Config{namespace: {}} @@ -77,6 +95,7 @@ func main() { panic(err) } publisher := &controller.InternalDNSPublisherReconciler{ + ObservationSources: map[string]controller.InternalDNSObservationSource{location: {Reader: edge, ClusterUID: types.UID(edgeUID)}}, ProjectIdentities: map[string]controller.InternalDNSProjectIdentity{"single": { ProjectName: "single", ProjectUID: types.UID(projectUID), SourceClusterUID: sourceUID, }}, PrincipalSubject: subject, LeaseDuration: lease, diff --git a/test/internaldns/results/fleet-smoke.json b/test/internaldns/results/fleet-smoke.json deleted file mode 100644 index 888c69a5..00000000 --- a/test/internaldns/results/fleet-smoke.json +++ /dev/null @@ -1,235 +0,0 @@ -{ - "passed": true, - "error": null, - "checks": [ - { - "label": "default-off publisher starts without a Kubernetes/DNS configuration", - "passed": true - }, - { - "label": "second VPC serves its own Compute IPv4 UDP", - "passed": true, - "name": "web-02-1333f06e881549a7.instances.53992e2b22a3a5f1bb360315.managed.internal", - "recordType": "A", - "answers": [ - "10.0.1.20" - ], - "tcp": false, - "status": "NOERROR" - }, - { - "label": "second VPC serves its own Compute IPv6 UDP", - "passed": true, - "name": "web-02-1333f06e881549a7.instances.53992e2b22a3a5f1bb360315.managed.internal", - "recordType": "AAAA", - "answers": [ - "2001:db8::30" - ], - "tcp": false, - "status": "NOERROR" - }, - { - "label": "second VPC serves its own Compute IPv4 TCP", - "passed": true, - "name": "web-02-1333f06e881549a7.instances.53992e2b22a3a5f1bb360315.managed.internal", - "recordType": "A", - "answers": [ - "10.0.1.20" - ], - "tcp": true, - "status": "NOERROR" - }, - { - "label": "second VPC serves its own Compute IPv6 TCP", - "passed": true, - "name": "web-02-1333f06e881549a7.instances.53992e2b22a3a5f1bb360315.managed.internal", - "recordType": "AAAA", - "answers": [ - "2001:db8::30" - ], - "tcp": true, - "status": "NOERROR" - }, - { - "label": "production Compute IPv4 UDP", - "passed": true, - "name": "web-01-7411484c38c1f2a7.instances.56a4888365a3749b85e6f47c.managed.internal", - "recordType": "A", - "answers": [ - "10.0.1.30" - ], - "tcp": false, - "status": "NOERROR" - }, - { - "label": "production Compute IPv6 UDP", - "passed": true, - "name": "web-01-7411484c38c1f2a7.instances.56a4888365a3749b85e6f47c.managed.internal", - "recordType": "AAAA", - "answers": [ - "2001:db8::30" - ], - "tcp": false, - "status": "NOERROR" - }, - { - "label": "other VPC cannot resolve instance UDP", - "passed": true, - "name": "web-01-7411484c38c1f2a7.instances.56a4888365a3749b85e6f47c.managed.internal", - "recordType": "A", - "answers": [], - "tcp": false, - "status": "SERVFAIL" - }, - { - "label": "production Compute IPv4 TCP", - "passed": true, - "name": "web-01-7411484c38c1f2a7.instances.56a4888365a3749b85e6f47c.managed.internal", - "recordType": "A", - "answers": [ - "10.0.1.30" - ], - "tcp": true, - "status": "NOERROR" - }, - { - "label": "production Compute IPv6 TCP", - "passed": true, - "name": "web-01-7411484c38c1f2a7.instances.56a4888365a3749b85e6f47c.managed.internal", - "recordType": "AAAA", - "answers": [ - "2001:db8::30" - ], - "tcp": true, - "status": "NOERROR" - }, - { - "label": "other VPC cannot resolve instance TCP", - "passed": true, - "name": "web-01-7411484c38c1f2a7.instances.56a4888365a3749b85e6f47c.managed.internal", - "recordType": "A", - "answers": [], - "tcp": true, - "status": "SERVFAIL" - }, - { - "label": "application Ready=False leaves instance identity published", - "passed": true - }, - { - "label": "address update reaches DNS", - "passed": true, - "name": "web-01-7411484c38c1f2a7.instances.56a4888365a3749b85e6f47c.managed.internal", - "recordType": "A", - "answers": [ - "10.0.1.31" - ], - "tcp": false, - "status": "NOERROR" - }, - { - "label": "interface deallocation withdraws IPv4", - "passed": true, - "name": "web-01-7411484c38c1f2a7.instances.56a4888365a3749b85e6f47c.managed.internal", - "recordType": "A", - "answers": [], - "tcp": false, - "status": "NOERROR" - }, - { - "label": "interface deallocation withdraws IPv6", - "passed": true, - "name": "web-01-7411484c38c1f2a7.instances.56a4888365a3749b85e6f47c.managed.internal", - "recordType": "AAAA", - "answers": [], - "tcp": false, - "status": "NOERROR" - }, - { - "label": "other VPC retains its Compute record during withdrawal", - "passed": true, - "name": "web-02-1333f06e881549a7.instances.53992e2b22a3a5f1bb360315.managed.internal", - "recordType": "A", - "answers": [ - "10.0.1.20" - ], - "tcp": false, - "status": "NOERROR" - }, - { - "label": "interface recovery republishes", - "passed": true, - "name": "web-01-7411484c38c1f2a7.instances.56a4888365a3749b85e6f47c.managed.internal", - "recordType": "A", - "answers": [ - "10.0.1.30" - ], - "tcp": false, - "status": "NOERROR" - }, - { - "label": "stopped publisher lease expires", - "passed": true, - "name": "web-01-7411484c38c1f2a7.instances.56a4888365a3749b85e6f47c.managed.internal", - "recordType": "A", - "answers": [], - "tcp": false, - "status": "NOERROR" - }, - { - "label": "restarted publisher renews observation", - "passed": true, - "name": "web-01-7411484c38c1f2a7.instances.56a4888365a3749b85e6f47c.managed.internal", - "recordType": "A", - "answers": [ - "10.0.1.30" - ], - "tcp": false, - "status": "NOERROR" - }, - { - "label": "instance deletion withdraws answer", - "passed": true, - "name": "web-01-7411484c38c1f2a7.instances.56a4888365a3749b85e6f47c.managed.internal", - "recordType": "A", - "answers": [], - "tcp": false, - "status": "NXDOMAIN" - }, - { - "label": "other VPC retains its Compute record during deletion", - "passed": true, - "name": "web-02-1333f06e881549a7.instances.53992e2b22a3a5f1bb360315.managed.internal", - "recordType": "A", - "answers": [ - "10.0.1.20" - ], - "tcp": false, - "status": "NOERROR" - }, - { - "label": "instance deletion cleans registration, grant and contribution", - "passed": true - }, - { - "label": "shared serving process identities remain constant", - "passed": true - } - ], - "sourceSHA256": "46a42735831e53140feaba8b891aee6702329fe0ecf920872e1bdad694d5d6ab", - "images": { - "dnsdist": "powerdns/dnsdist-20@sha256:4ed9af56729c7021795dc478421237ab9cf6c7ceec8734677405e7617f06d52f", - "bind": "internetsystemsconsortium/bind9@sha256:071465f88068854d0ceadd9b985fb1acae4071e80754e545cd811fde57541ad0", - "powerdns": "powerdns/pdns-auth-51@sha256:bb5b1c133bcca1dd455075321de7d55db4945a8d7f2ba23339e3c7bbe416b205" - }, - "suite": "compute-internal-dns-fleet-smoke", - "computePublisherSourceSHA256": "47e2cff09e92c46e53a9664e3f4b226002a87961c0c9ce24f09b47474f508bbf", - "computePublisherBinarySHA256": "073c047232db7f5f022e569c14f73c7bee0b90b677f178948855e8a58831e455", - "limitations": [ - "Instance addressing/status are fixtures; no guest boots or real NSO allocation.", - "One local source API and one local broker; no regional HA or Milo IAM deployment validation.", - "Uses the production Compute publisher controller, started by a test runner; default-off production wiring is covered separately.", - "Unassociated managed suffixes return SERVFAIL with no private answers in this fixture; public recursion is not qualified." - ], - "buildBaseCommit": "cd312926e3370288bac62ba642cceb9fc15cfb40" -} From dac11565a59b7001003197cbf503752868ff3702 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Fri, 9 Oct 2026 21:37:55 -0500 Subject: [PATCH 04/33] Share DNS protocol constants for CI lint checks --- .../controller/instance_projector_test.go | 4 +- .../internaldns_authorization_test.go | 50 ++++----- .../internaldns_observation_test.go | 24 ++--- internal/controller/internaldns_publisher.go | 101 ++++++++++-------- .../controller/internaldns_publisher_test.go | 53 +++++---- internal/controller/internaldns_status.go | 8 +- .../controller/internaldns_status_test.go | 18 ++-- 7 files changed, 140 insertions(+), 118 deletions(-) diff --git a/internal/controller/instance_projector_test.go b/internal/controller/instance_projector_test.go index 53db4c33..e9377c44 100644 --- a/internal/controller/instance_projector_test.go +++ b/internal/controller/instance_projector_test.go @@ -439,9 +439,9 @@ func TestInstanceProjector_SpecCopied(t *testing.T) { } func TestInstanceProjectorPreservesProjectDNSStatus(t *testing.T) { - upstream := projTestKarmadaInstance(map[string]string{computev1alpha.InstanceSourceUIDLabel: "edge-instance"}) + upstream := projTestKarmadaInstance(map[string]string{computev1alpha.InstanceSourceUIDLabel: internalDNSTestEdgeUID}) upstream.Status.DNS = []computev1alpha.InstanceDNSStatus{{NetworkUID: "untrusted-edge-status"}} - projection := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: projTestInstanceName, Namespace: projTestProjNS, Labels: map[string]string{computev1alpha.InstanceSourceUIDLabel: "edge-instance"}}, Status: computev1alpha.InstanceStatus{DNS: []computev1alpha.InstanceDNSStatus{{NetworkUID: "project-network", Hostnames: []string{"web.datum.internal"}}}}} + projection := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: projTestInstanceName, Namespace: projTestProjNS, Labels: map[string]string{computev1alpha.InstanceSourceUIDLabel: internalDNSTestEdgeUID}}, Status: computev1alpha.InstanceStatus{DNS: []computev1alpha.InstanceDNSStatus{{NetworkUID: "project-network", Hostnames: []string{"web.datum.internal"}}}}} cl := fake.NewClientBuilder().WithScheme(newProjectScheme()).WithObjects(projTestWorkloadDeployment(), projection).WithStatusSubresource(&computev1alpha.Instance{}).Build() r := newTestProjector(newKarmadaFakeClient(upstream), cl) _, err := r.Reconcile(context.Background(), projectorRequest()) diff --git a/internal/controller/internaldns_authorization_test.go b/internal/controller/internaldns_authorization_test.go index e91605ad..c1c602c8 100644 --- a/internal/controller/internaldns_authorization_test.go +++ b/internal/controller/internaldns_authorization_test.go @@ -20,7 +20,7 @@ import ( ) func TestFindContributionGrantRequiresCurrentScopedAuthority(t *testing.T) { - registration := newDNSObject(dnsRegistrationGVK, internalDNSTestNamespace, "web") + registration := newDNSObject(dnsRegistrationGVK, internalDNSTestNamespace, internalDNSTestInstanceName) registration.SetUID("registration-uid") registration.SetGeneration(2) for _, tc := range []struct { @@ -30,14 +30,14 @@ func TestFindContributionGrantRequiresCurrentScopedAuthority(t *testing.T) { found bool }{ {name: "active matching grant", found: true}, - {name: "different principal", path: []string{"spec", "principal", "subject"}, value: "other"}, - {name: "different source cluster", path: []string{"spec", "principal", "clusterUID"}, value: "other"}, - {name: "different registration lifetime", path: []string{"spec", "registrationRef", "uid"}, value: "other"}, - {name: "stale registration generation", path: []string{"spec", "registrationRef", "generation"}, value: int64(1)}, - {name: "different name scope", path: []string{"spec", "nameScopes"}, value: []any{"other"}}, - {name: "missing record type", path: []string{"spec", "recordTypes"}, value: []any{"A"}}, - {name: "inactive grant", path: []string{"status", "conditions"}, value: []any{map[string]any{"type": "Active", "status": "False", "observedGeneration": int64(1)}}}, - {name: "stale active condition", path: []string{"status", "conditions"}, value: []any{map[string]any{"type": "Active", "status": "True", "observedGeneration": int64(0)}}}, + {name: "different principal", path: []string{internalDNSSpecField, internalDNSPrincipalField, internalDNSSubjectField}, value: internalDNSTestForeignValue}, + {name: "different source cluster", path: []string{internalDNSSpecField, internalDNSPrincipalField, internalDNSClusterUIDField}, value: internalDNSTestForeignValue}, + {name: "different registration lifetime", path: []string{internalDNSSpecField, internalDNSRegistrationRefField, "uid"}, value: internalDNSTestForeignValue}, + {name: "stale registration generation", path: []string{internalDNSSpecField, internalDNSRegistrationRefField, "generation"}, value: int64(1)}, + {name: "different name scope", path: []string{internalDNSSpecField, internalDNSNameScopesField}, value: []any{internalDNSTestForeignValue}}, + {name: "missing record type", path: []string{internalDNSSpecField, internalDNSRecordTypesField}, value: []any{"A"}}, + {name: "inactive grant", path: []string{internalDNSStatusField, internalDNSConditionsField}, value: []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionActive, internalDNSStatusField: string(metav1.ConditionFalse), internalDNSObservedGenerationField: int64(1)}}}, + {name: "stale active condition", path: []string{internalDNSStatusField, internalDNSConditionsField}, value: []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionActive, internalDNSStatusField: string(metav1.ConditionTrue), internalDNSObservedGenerationField: int64(0)}}}, } { t.Run(tc.name, func(t *testing.T) { s := runtime.NewScheme() @@ -46,13 +46,13 @@ func TestFindContributionGrantRequiresCurrentScopedAuthority(t *testing.T) { grant := newDNSObject(dnsGrantGVK, internalDNSTestNamespace, "trusted-grant") grant.SetUID("grant-uid") grant.SetGeneration(1) - grant.Object["spec"] = map[string]any{"registrationRef": objectReference(registration), "principal": map[string]any{"clusterUID": "project-api", "subject": "compute"}, "nameScopes": []any{"web"}, "recordTypes": []any{"A", "AAAA"}} - grant.Object["status"] = map[string]any{"conditions": []any{map[string]any{"type": "Active", "status": "True", "observedGeneration": int64(1)}}} + grant.Object[internalDNSSpecField] = map[string]any{internalDNSRegistrationRefField: objectReference(registration), internalDNSPrincipalField: map[string]any{internalDNSClusterUIDField: internalDNSTestSourceAPIUID, internalDNSSubjectField: internalDNSTestSubject}, internalDNSNameScopesField: []any{internalDNSTestInstanceName}, internalDNSRecordTypesField: []any{"A", "AAAA"}} + grant.Object[internalDNSStatusField] = map[string]any{internalDNSConditionsField: []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionActive, internalDNSStatusField: string(metav1.ConditionTrue), internalDNSObservedGenerationField: int64(1)}}} if len(tc.path) > 0 { require.NoError(t, unstructured.SetNestedField(grant.Object, tc.value, tc.path...)) } cl := fake.NewClientBuilder().WithScheme(s).WithObjects(grant).Build() - got, err := findContributionGrant(context.Background(), cl, internalDNSTestNamespace, registration, "project-api", "compute", "web") + got, err := findContributionGrant(context.Background(), cl, internalDNSTestNamespace, registration, internalDNSTestSourceAPIUID, internalDNSTestSubject, internalDNSTestInstanceName) require.NoError(t, err) require.Equal(t, tc.found, got != nil) }) @@ -63,7 +63,7 @@ func TestRefreshObservationCannotExtendSourceDeadline(t *testing.T) { now := time.Date(2026, 10, 9, 20, 0, 0, 0, time.UTC) s := runtime.NewScheme() s.AddKnownTypeWithName(dnsContributionGVK, &unstructured.Unstructured{}) - c := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, "web") + c := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, internalDNSTestInstanceName) c.SetUID("contribution") c.SetGeneration(1) cl := fake.NewClientBuilder().WithScheme(s).WithObjects(c).WithStatusSubresource(c).Build() @@ -74,10 +74,10 @@ func TestRefreshObservationCannotExtendSourceDeadline(t *testing.T) { require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(c), c)) _, err := r.refreshObservation(context.Background(), cl, c, 3, true, deadline) require.NoError(t, err) - eligible, _, _ := unstructured.NestedBool(c.Object, "status", "eligible") + eligible, _, _ := unstructured.NestedBool(c.Object, internalDNSStatusField, "eligible") require.Equal(t, step < 25*time.Second, eligible) if eligible { - raw, _, _ := unstructured.NestedString(c.Object, "status", "validUntil") + raw, _, _ := unstructured.NestedString(c.Object, internalDNSStatusField, "validUntil") got, err := time.Parse(time.RFC3339Nano, raw) require.NoError(t, err) require.True(t, got.Equal(deadline)) @@ -100,7 +100,7 @@ func TestPublisherWaitsForIssuerAndNeverMutatesGrants(t *testing.T) { s.AddKnownTypeWithName(gvk, &unstructured.Unstructured{}) s.AddKnownTypeWithName(gvk.GroupVersion().WithKind(gvk.Kind+"List"), &unstructured.UnstructuredList{}) } - contextObject := resolverContextObject("context", "project-a", "vpc-a", true) + contextObject := resolverContextObject("context", "project-a", internalDNSTestVPCUID, true) cl := fake.NewClientBuilder().WithScheme(s).WithObjects(contextObject).WithStatusSubresource(newDNSObject(dnsRegistrationGVK, "", ""), newDNSObject(dnsContributionGVK, "", "")).WithInterceptorFuncs(interceptor.Funcs{Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { obj.SetUID(types.UID(obj.GetName() + "-uid")) obj.SetGeneration(1) @@ -121,12 +121,12 @@ func TestPublisherWaitsForIssuerAndNeverMutatesGrants(t *testing.T) { }, }) now := time.Now().UTC().Truncate(time.Second) - r := &InternalDNSPublisherReconciler{PrincipalSubject: "compute", Now: func() time.Time { return now }} - instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: "web", Namespace: internalDNSTestNamespace, UID: "instance-uid"}} - owner := metav1.OwnerReference{APIVersion: computev1alpha.GroupVersion.String(), Kind: "Instance", Name: instance.Name, UID: instance.UID, Controller: ptrBool(true)} + r := &InternalDNSPublisherReconciler{PrincipalSubject: internalDNSTestSubject, Now: func() time.Time { return now }} + instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestInstanceName, Namespace: internalDNSTestNamespace, UID: internalDNSTestInstanceUID}} + owner := metav1.OwnerReference{APIVersion: computev1alpha.GroupVersion.String(), Kind: internalDNSInstanceKind, Name: instance.Name, UID: instance.UID, Controller: ptrBool(true)} access := InternalDNSProjectAccess{Reader: guard, Writer: guard} - identity := InternalDNSProjectIdentity{ProjectUID: "project-a", SourceClusterUID: "project-api"} - attachment := instanceDNSAttachment{VPCUID: "vpc-a", Eligible: true, ValidUntil: now.Add(40 * time.Second), RecordSets: []any{map[string]any{"recordType": "A", "records": []any{map[string]any{"name": "", "ttl": int64(30), "a": map[string]any{"content": "10.0.0.8"}}}}}} + identity := InternalDNSProjectIdentity{ProjectUID: "project-a", SourceClusterUID: internalDNSTestSourceAPIUID} + attachment := instanceDNSAttachment{VPCUID: internalDNSTestVPCUID, Eligible: true, ValidUntil: now.Add(40 * time.Second), RecordSets: []any{map[string]any{"recordType": "A", "records": []any{map[string]any{"name": "", "ttl": int64(30), "a": map[string]any{"content": "10.0.0.8"}}}}}} ctx := context.Background() for range 2 { ready, err := r.reconcileAttachment(ctx, access, identity, internalDNSTestNamespace, instance, owner, attachment, "record") @@ -135,18 +135,18 @@ func TestPublisherWaitsForIssuerAndNeverMutatesGrants(t *testing.T) { } registration := newDNSObject(dnsRegistrationGVK, internalDNSTestNamespace, "record") require.NoError(t, cl.Get(ctx, client.ObjectKeyFromObject(registration), registration)) - name, _, _ := unstructured.NestedString(registration.Object, "spec", "name") + name, _, _ := unstructured.NestedString(registration.Object, internalDNSSpecField, "name") require.Equal(t, allocatedInstanceDNSName(instance), name) grant := newDNSObject(dnsGrantGVK, internalDNSTestNamespace, "issuer-grant") - grant.Object["spec"] = map[string]any{"registrationRef": objectReference(registration), "principal": map[string]any{"clusterUID": "project-api", "subject": "compute"}, "nameScopes": []any{name}, "recordTypes": []any{"A", "AAAA"}} - grant.Object["status"] = map[string]any{"conditions": []any{map[string]any{"type": "Active", "status": "True", "observedGeneration": int64(1)}}, "activeWriterEpoch": int64(3), "observedGrantGeneration": int64(1), "observedRegistrationGeneration": int64(1)} + grant.Object[internalDNSSpecField] = map[string]any{internalDNSRegistrationRefField: objectReference(registration), internalDNSPrincipalField: map[string]any{internalDNSClusterUIDField: internalDNSTestSourceAPIUID, internalDNSSubjectField: internalDNSTestSubject}, internalDNSNameScopesField: []any{name}, internalDNSRecordTypesField: []any{"A", "AAAA"}} + grant.Object[internalDNSStatusField] = map[string]any{internalDNSConditionsField: []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionActive, internalDNSStatusField: string(metav1.ConditionTrue), internalDNSObservedGenerationField: int64(1)}}, "activeWriterEpoch": int64(3), "observedGrantGeneration": int64(1), "observedRegistrationGeneration": int64(1)} require.NoError(t, cl.Create(ctx, grant)) ready, err := r.reconcileAttachment(ctx, access, identity, internalDNSTestNamespace, instance, owner, attachment, "record") require.NoError(t, err) require.False(t, ready) contribution := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, "record") require.NoError(t, cl.Get(ctx, client.ObjectKeyFromObject(contribution), contribution)) - contribution.Object["status"] = map[string]any{"writerEpoch": int64(3)} + contribution.Object[internalDNSStatusField] = map[string]any{internalDNSWriterEpochField: int64(3)} require.NoError(t, cl.Status().Update(ctx, contribution)) ready, err = r.reconcileAttachment(ctx, access, identity, internalDNSTestNamespace, instance, owner, attachment, "record") require.NoError(t, err) diff --git a/internal/controller/internaldns_observation_test.go b/internal/controller/internaldns_observation_test.go index 4dec9882..e46419b3 100644 --- a/internal/controller/internaldns_observation_test.go +++ b/internal/controller/internaldns_observation_test.go @@ -36,10 +36,10 @@ func TestObserveInternalDNSInterface(t *testing.T) { i.Generation++ }}, {name: "different network", mutate: func(_ *computev1alpha.Instance, i *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node, _ *coordinationv1.Lease) { - i.Spec.Network.Name = "other" + i.Spec.Network.Name = internalDNSTestForeignValue }}, {name: "different claim", mutate: func(_ *computev1alpha.Instance, i *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node, _ *coordinationv1.Lease) { - i.Spec.ClaimRef.Name = "other" + i.Spec.ClaimRef.Name = internalDNSTestForeignValue }, wantErr: true}, {name: "runtime exited", mutate: func(_ *computev1alpha.Instance, _ *networkingv1alpha.NetworkInterface, p *corev1.Pod, _ *corev1.Node, _ *coordinationv1.Lease) { p.Status.Phase = corev1.PodFailed @@ -62,10 +62,10 @@ func TestObserveInternalDNSInterface(t *testing.T) { for _, add := range []func(*runtime.Scheme) error{corev1.AddToScheme, coordinationv1.AddToScheme, computev1alpha.AddToScheme, networkingv1alpha.AddToScheme} { require.NoError(t, add(scheme)) } - instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: "web", Namespace: "project", UID: "project-instance", Labels: map[string]string{computev1alpha.InstanceSourceUIDLabel: "edge-instance", computev1alpha.LocationLabel: "dfw"}}, Spec: computev1alpha.InstanceSpec{NetworkInterfaces: []computev1alpha.InstanceNetworkInterface{{Network: networkingv1alpha.NetworkRef{Name: "application"}}}}} - edgeInstance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: instance.Name, Namespace: "ns-project-namespace", UID: "edge-instance"}, Status: computev1alpha.InstanceStatus{NetworkInterfaces: []computev1alpha.InstanceNetworkInterfaceStatus{{Name: "eth0", NetworkInterfaceRef: &networkingv1alpha.LocalNetworkInterfaceRef{Name: "interface"}}}}} - iface := &networkingv1alpha.NetworkInterface{ObjectMeta: metav1.ObjectMeta{Name: "interface", Namespace: edgeInstance.Namespace, UID: "interface-uid", Generation: 2}, Spec: networkingv1alpha.NetworkInterfaceSpec{Network: networkingv1alpha.LocalNetworkRef{Name: "application"}, InterfaceName: "eth0", ClaimRef: &networkingv1alpha.NetworkInterfaceClaimRef{Name: networkInterfaceClaimName("web", "eth0")}, Addresses: []networkingv1alpha.NetworkInterfaceAddress{{Address: "10.20.0.10/32"}}}, Status: networkingv1alpha.NetworkInterfaceStatus{Conditions: []metav1.Condition{{Type: "Allocated", Status: metav1.ConditionTrue, ObservedGeneration: 2}, {Type: "Programmed", Status: metav1.ConditionTrue, ObservedGeneration: 2}}}} - pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "runtime", Namespace: edgeInstance.Namespace, OwnerReferences: []metav1.OwnerReference{{APIVersion: computev1alpha.GroupVersion.String(), Kind: "Instance", Name: edgeInstance.Name, UID: edgeInstance.UID, Controller: ptrBool(true)}}}, Spec: corev1.PodSpec{NodeName: "worker"}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionFalse}}}} + instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestInstanceName, Namespace: internalDNSTestObservationNamespace, UID: "project-instance", Labels: map[string]string{computev1alpha.InstanceSourceUIDLabel: internalDNSTestEdgeUID, computev1alpha.LocationLabel: "dfw"}}, Spec: computev1alpha.InstanceSpec{NetworkInterfaces: []computev1alpha.InstanceNetworkInterface{{Network: networkingv1alpha.NetworkRef{Name: internalDNSTestNetworkName}}}}} + edgeInstance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: instance.Name, Namespace: "ns-project-namespace", UID: internalDNSTestEdgeUID}, Status: computev1alpha.InstanceStatus{NetworkInterfaces: []computev1alpha.InstanceNetworkInterfaceStatus{{Name: defaultInterfaceName, NetworkInterfaceRef: &networkingv1alpha.LocalNetworkInterfaceRef{Name: "interface"}}}}} + iface := &networkingv1alpha.NetworkInterface{ObjectMeta: metav1.ObjectMeta{Name: "interface", Namespace: edgeInstance.Namespace, UID: "interface-uid", Generation: 2}, Spec: networkingv1alpha.NetworkInterfaceSpec{Network: networkingv1alpha.LocalNetworkRef{Name: internalDNSTestNetworkName}, InterfaceName: defaultInterfaceName, ClaimRef: &networkingv1alpha.NetworkInterfaceClaimRef{Name: networkInterfaceClaimName(internalDNSTestInstanceName, defaultInterfaceName)}, Addresses: []networkingv1alpha.NetworkInterfaceAddress{{Address: "10.20.0.10/32"}}}, Status: networkingv1alpha.NetworkInterfaceStatus{Conditions: []metav1.Condition{{Type: "Allocated", Status: metav1.ConditionTrue, ObservedGeneration: 2}, {Type: "Programmed", Status: metav1.ConditionTrue, ObservedGeneration: 2}}}} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "runtime", Namespace: edgeInstance.Namespace, OwnerReferences: []metav1.OwnerReference{{APIVersion: computev1alpha.GroupVersion.String(), Kind: internalDNSInstanceKind, Name: edgeInstance.Name, UID: edgeInstance.UID, Controller: ptrBool(true)}}}, Spec: corev1.PodSpec{NodeName: "worker"}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionFalse}}}} node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "worker", UID: "node-uid"}, Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}}} lease := &coordinationv1.Lease{ObjectMeta: metav1.ObjectMeta{Name: node.Name, Namespace: "kube-node-lease", OwnerReferences: []metav1.OwnerReference{{Kind: "Node", Name: node.Name, UID: node.UID}}}, Spec: coordinationv1.LeaseSpec{RenewTime: &metav1.MicroTime{Time: now.Add(-10 * time.Second)}}} if tc.mutate != nil { @@ -76,12 +76,12 @@ func TestObserveInternalDNSInterface(t *testing.T) { if tc.networkUID != "" { networkUID = tc.networkUID } - network := &networkingv1alpha.Network{ObjectMeta: metav1.ObjectMeta{Name: "application", Namespace: "project", UID: networkUID}} - project := fake.NewClientBuilder().WithScheme(scheme).WithObjects(network, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "project", UID: "project-namespace"}}).Build() - claim := &networkingv1alpha.NetworkInterfaceClaim{ObjectMeta: metav1.ObjectMeta{Name: networkInterfaceClaimName("web", "eth0"), Namespace: edgeInstance.Namespace, UID: "claim-uid"}, Spec: networkingv1alpha.NetworkInterfaceClaimSpec{Network: networkingv1alpha.LocalNetworkRef{Name: "application"}}, Status: networkingv1alpha.NetworkInterfaceClaimStatus{NetworkInterfaceRef: &networkingv1alpha.LocalNetworkInterfaceRef{Name: iface.Name}}} - networkContext := &networkingv1alpha.NetworkContext{ObjectMeta: metav1.ObjectMeta{Name: "context", Namespace: edgeInstance.Namespace, UID: "network-context", Labels: map[string]string{networkingv1alpha.NetworkUIDLabel: "network-uid"}}, Spec: networkingv1alpha.NetworkContextSpec{Network: networkingv1alpha.LocalNetworkRef{Name: "application"}, Location: networkingv1alpha.LocationReference{Name: "dfw"}}} + network := &networkingv1alpha.Network{ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestNetworkName, Namespace: internalDNSTestObservationNamespace, UID: networkUID}} + project := fake.NewClientBuilder().WithScheme(scheme).WithObjects(network, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestObservationNamespace, UID: "project-namespace"}}).Build() + claim := &networkingv1alpha.NetworkInterfaceClaim{ObjectMeta: metav1.ObjectMeta{Name: networkInterfaceClaimName(internalDNSTestInstanceName, defaultInterfaceName), Namespace: edgeInstance.Namespace, UID: "claim-uid"}, Spec: networkingv1alpha.NetworkInterfaceClaimSpec{Network: networkingv1alpha.LocalNetworkRef{Name: internalDNSTestNetworkName}}, Status: networkingv1alpha.NetworkInterfaceClaimStatus{NetworkInterfaceRef: &networkingv1alpha.LocalNetworkInterfaceRef{Name: iface.Name}}} + networkContext := &networkingv1alpha.NetworkContext{ObjectMeta: metav1.ObjectMeta{Name: "context", Namespace: edgeInstance.Namespace, UID: "network-context", Labels: map[string]string{networkingv1alpha.NetworkUIDLabel: "network-uid"}}, Spec: networkingv1alpha.NetworkContextSpec{Network: networkingv1alpha.LocalNetworkRef{Name: internalDNSTestNetworkName}, Location: networkingv1alpha.LocationReference{Name: "dfw"}}} source := fake.NewClientBuilder().WithScheme(scheme).WithObjects(edgeInstance, iface, claim, networkContext, pod, node, lease, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "kube-system", UID: "edge-cluster"}}).Build() - got, err := ObserveInternalDNSInterface(context.Background(), project, "project", instance, instance.Spec.NetworkInterfaces[0], InternalDNSObservationSource{Reader: source, ClusterUID: "edge-cluster"}, now) + got, err := ObserveInternalDNSInterface(context.Background(), project, internalDNSTestObservationNamespace, instance, instance.Spec.NetworkInterfaces[0], InternalDNSObservationSource{Reader: source, ClusterUID: "edge-cluster"}, now) if tc.wantErr { require.Error(t, err) } else { @@ -97,6 +97,6 @@ func TestObserveInternalDNSInterface(t *testing.T) { } func TestObserveInternalDNSInterfaceRejectsMissingSource(t *testing.T) { - _, err := ObserveInternalDNSInterface(context.Background(), nil, "project", &computev1alpha.Instance{}, computev1alpha.InstanceNetworkInterface{}, InternalDNSObservationSource{}, time.Now()) + _, err := ObserveInternalDNSInterface(context.Background(), nil, internalDNSTestObservationNamespace, &computev1alpha.Instance{}, computev1alpha.InstanceNetworkInterface{}, InternalDNSObservationSource{}, time.Now()) require.Error(t, err) } diff --git a/internal/controller/internaldns_publisher.go b/internal/controller/internaldns_publisher.go index d7521752..cafb0951 100644 --- a/internal/controller/internaldns_publisher.go +++ b/internal/controller/internaldns_publisher.go @@ -33,24 +33,39 @@ import ( ) const ( - internalDNSInstanceKind = "Instance" - internalDNSRecordTypeAAAA = "AAAA" - internalDNSNameField = "name" - internalDNSStatusField = "status" - internalDNSReasonField = "reason" - internalDNSUIDField = "uid" - internalDNSGroup = "dns.networking.miloapis.com" - internalDNSVersion = "v1alpha1" - internalDNSInstanceUID = "internal-dns.compute.datumapis.com/instance-uid" - internalDNSVPCUID = "internal-dns.compute.datumapis.com/vpc-uid" - internalDNSManagedBy = "internal-dns.compute.datumapis.com/managed-by" - internalDNSManager = "compute-instance-publisher" - defaultDNSLease = 60 * time.Second - maxDNSLease = 90 * time.Second - internalDNSPendingWait = 10 * time.Second - internalDNSAPITimeout = 10 * time.Second - defaultDNSConcurrency = 4 - defaultInternalDNSDomain = "datum.internal" + internalDNSInstanceKind = "Instance" + internalDNSRecordTypeAAAA = "AAAA" + internalDNSNameField = "name" + internalDNSStatusField = "status" + internalDNSReasonField = "reason" + internalDNSUIDField = "uid" + internalDNSGroup = "dns.networking.miloapis.com" + internalDNSVersion = "v1alpha1" + internalDNSInstanceUID = "internal-dns.compute.datumapis.com/instance-uid" + internalDNSVPCUID = "internal-dns.compute.datumapis.com/vpc-uid" + internalDNSManagedBy = "internal-dns.compute.datumapis.com/managed-by" + internalDNSManager = "compute-instance-publisher" + defaultDNSLease = 60 * time.Second + maxDNSLease = 90 * time.Second + internalDNSPendingWait = 10 * time.Second + internalDNSAPITimeout = 10 * time.Second + defaultDNSConcurrency = 4 + defaultInternalDNSDomain = "datum.internal" + internalDNSConditionsField = "conditions" + internalDNSObservedGenerationField = "observedGeneration" + internalDNSWriterEpochField = "writerEpoch" + internalDNSRegistrationRefField = "registrationRef" + internalDNSRecordTypesField = "recordTypes" + internalDNSNameScopesField = "nameScopes" + internalDNSClusterUIDField = "clusterUID" + internalDNSConditionActive = "Active" + internalDNSConditionPublished = "Published" + internalDNSPrincipalField = "principal" + internalDNSConditionTypeField = "type" + internalDNSSpecField = "spec" + internalDNSSubjectField = "subject" + internalDNSGrantRefField = "grantRef" + internalDNSRecordSetsField = "recordSets" ) var ( @@ -321,11 +336,11 @@ func (r *InternalDNSPublisherReconciler) reconcileAttachment(ctx context.Context } registrationSpec := map[string]any{ - "dnsZoneRef": zoneRef, - internalDNSNameField: ownerName, - "recordTypes": []any{"A", internalDNSRecordTypeAAAA}, - "publicationPolicy": "EligibleContributions", - "ttlSeconds": int64(30), + "dnsZoneRef": zoneRef, + internalDNSNameField: ownerName, + internalDNSRecordTypesField: []any{"A", internalDNSRecordTypeAAAA}, + "publicationPolicy": "EligibleContributions", + "ttlSeconds": int64(30), } registration, changed, err := ensureDNSObject(ctx, access, dnsRegistrationGVK, namespace, resourceName, labels, owner, registrationSpec) if err != nil || changed { @@ -348,15 +363,15 @@ func (r *InternalDNSPublisherReconciler) reconcileAttachment(ctx context.Context } contributionSpec := map[string]any{ - "registrationRef": registrationRef, - "grantRef": objectReference(grant), - "recordSets": attachment.RecordSets, + internalDNSRegistrationRefField: registrationRef, + internalDNSGrantRefField: objectReference(grant), + internalDNSRecordSetsField: attachment.RecordSets, } contribution, changed, err := ensureDNSObject(ctx, access, dnsContributionGVK, namespace, resourceName, labels, owner, contributionSpec) if err != nil || changed { return false, err } - boundEpoch, _, _ := unstructured.NestedInt64(contribution.Object, internalDNSStatusField, "writerEpoch") + boundEpoch, _, _ := unstructured.NestedInt64(contribution.Object, internalDNSStatusField, internalDNSWriterEpochField) if boundEpoch != writerEpoch { return false, nil } @@ -378,7 +393,7 @@ func (r *InternalDNSPublisherReconciler) refreshObservation(ctx context.Context, } } oldEligible, _, _ := unstructured.NestedBool(contribution.Object, internalDNSStatusField, "eligible") - observedGeneration, _, _ := unstructured.NestedInt64(contribution.Object, internalDNSStatusField, "observedGeneration") + observedGeneration, _, _ := unstructured.NestedInt64(contribution.Object, internalDNSStatusField, internalDNSObservedGenerationField) validUntilRaw, _, _ := unstructured.NestedString(contribution.Object, internalDNSStatusField, "validUntil") validUntil, _ := time.Parse(time.RFC3339Nano, validUntilRaw) if oldEligible == eligible && observedGeneration == contribution.GetGeneration() && validUntil.After(now.Add(r.leaseDuration()/3)) && !validUntil.After(validThrough) { @@ -391,8 +406,8 @@ func (r *InternalDNSPublisherReconciler) refreshObservation(ctx context.Context, path []string value any }{ - {[]string{internalDNSStatusField, "observedGeneration"}, contribution.GetGeneration()}, - {[]string{internalDNSStatusField, "writerEpoch"}, writerEpoch}, + {[]string{internalDNSStatusField, internalDNSObservedGenerationField}, contribution.GetGeneration()}, + {[]string{internalDNSStatusField, internalDNSWriterEpochField}, writerEpoch}, {[]string{internalDNSStatusField, "sequence"}, sequence + 1}, {[]string{internalDNSStatusField, "eligible"}, eligible}, {[]string{internalDNSStatusField, internalDNSReasonField}, map[bool]string{true: "AddressAllocatedAndProgrammed", false: "AddressUnavailable"}[eligible]}, @@ -424,7 +439,7 @@ func findResolverContext(ctx context.Context, reader client.Reader, namespace st if !item.GetDeletionTimestamp().IsZero() || item.GetUID() == "" || item.GetGeneration() < 1 { continue } - consumerID, _, _ := unstructured.NestedString(item.Object, "spec", "consumerID") + consumerID, _, _ := unstructured.NestedString(item.Object, internalDNSSpecField, "consumerID") if consumerID != string(projectUID)+"/"+string(vpcUID) || !unstructuredConditionCurrent(item, "Accepted") || !unstructuredConditionCurrent(item, "Ready") { continue } @@ -437,10 +452,10 @@ func findResolverContext(ctx context.Context, reader client.Reader, namespace st } func unstructuredConditionCurrent(obj *unstructured.Unstructured, conditionType string) bool { - conditions, _, _ := unstructured.NestedSlice(obj.Object, internalDNSStatusField, "conditions") + conditions, _, _ := unstructured.NestedSlice(obj.Object, internalDNSStatusField, internalDNSConditionsField) for _, raw := range conditions { condition, ok := raw.(map[string]any) - if ok && condition["type"] == conditionType && condition[internalDNSStatusField] == string(metav1.ConditionTrue) && condition["observedGeneration"] == obj.GetGeneration() { + if ok && condition[internalDNSConditionTypeField] == conditionType && condition[internalDNSStatusField] == string(metav1.ConditionTrue) && condition[internalDNSObservedGenerationField] == obj.GetGeneration() { return true } } @@ -456,16 +471,16 @@ func findContributionGrant(ctx context.Context, reader client.Reader, namespace var match *unstructured.Unstructured for i := range list.Items { grant := &list.Items[i] - if !grant.GetDeletionTimestamp().IsZero() || grant.GetUID() == "" || grant.GetGeneration() < 1 || !unstructuredConditionCurrent(grant, "Active") { + if !grant.GetDeletionTimestamp().IsZero() || grant.GetUID() == "" || grant.GetGeneration() < 1 || !unstructuredConditionCurrent(grant, internalDNSConditionActive) { continue } - ref, _, _ := unstructured.NestedMap(grant.Object, "spec", "registrationRef") - principal, _, _ := unstructured.NestedMap(grant.Object, "spec", "principal") - if !reflect.DeepEqual(ref, objectReference(registration)) || stringValue(principal, "clusterUID") != clusterUID || stringValue(principal, "subject") != subject { + ref, _, _ := unstructured.NestedMap(grant.Object, internalDNSSpecField, internalDNSRegistrationRefField) + principal, _, _ := unstructured.NestedMap(grant.Object, internalDNSSpecField, internalDNSPrincipalField) + if !reflect.DeepEqual(ref, objectReference(registration)) || stringValue(principal, internalDNSClusterUIDField) != clusterUID || stringValue(principal, internalDNSSubjectField) != subject { continue } - scopes, _, _ := unstructured.NestedStringSlice(grant.Object, "spec", "nameScopes") - types, _, _ := unstructured.NestedStringSlice(grant.Object, "spec", "recordTypes") + scopes, _, _ := unstructured.NestedStringSlice(grant.Object, internalDNSSpecField, internalDNSNameScopesField) + types, _, _ := unstructured.NestedStringSlice(grant.Object, internalDNSSpecField, internalDNSRecordTypesField) if !containsDNSString(scopes, ownerName) || !containsDNSString(types, "A") || !containsDNSString(types, "AAAA") { continue } @@ -490,7 +505,7 @@ func ensureDNSObject(ctx context.Context, access InternalDNSProjectAccess, gvk s desired := newDNSObject(gvk, namespace, name) desired.SetLabels(labels) desired.SetOwnerReferences([]metav1.OwnerReference{owner}) - desired.Object["spec"] = spec + desired.Object[internalDNSSpecField] = spec current := newDNSObject(gvk, namespace, name) err := access.Reader.Get(ctx, client.ObjectKeyFromObject(desired), current) if apierrors.IsNotFound(err) { @@ -508,7 +523,7 @@ func ensureDNSObject(ctx context.Context, access InternalDNSProjectAccess, gvk s if current.GetLabels()[internalDNSManagedBy] != internalDNSManager || current.GetLabels()[internalDNSInstanceUID] != labels[internalDNSInstanceUID] || !ownedBy(current, owner) { return nil, false, fmt.Errorf("refusing to adopt foreign %s %s/%s", gvk.Kind, namespace, name) } - currentSpec, _, _ := unstructured.NestedMap(current.Object, "spec") + currentSpec, _, _ := unstructured.NestedMap(current.Object, internalDNSSpecField) if authorityReferencesChanged(gvk, currentSpec, spec) { if err := access.Writer.Delete(ctx, current); client.IgnoreNotFound(err) != nil { return nil, false, err @@ -516,7 +531,7 @@ func ensureDNSObject(ctx context.Context, access InternalDNSProjectAccess, gvk s return current, true, nil } if !reflect.DeepEqual(currentSpec, spec) || !reflect.DeepEqual(current.GetLabels(), labels) || !reflect.DeepEqual(current.GetOwnerReferences(), []metav1.OwnerReference{owner}) { - current.Object["spec"] = spec + current.Object[internalDNSSpecField] = spec current.SetLabels(labels) current.SetOwnerReferences([]metav1.OwnerReference{owner}) if err := access.Writer.Update(ctx, current); err != nil { @@ -542,7 +557,7 @@ func authorityReferencesChanged(gvk schema.GroupVersionKind, current, desired ma if gvk != dnsContributionGVK { return false } - return !reflect.DeepEqual(current["registrationRef"], desired["registrationRef"]) || !reflect.DeepEqual(current["grantRef"], desired["grantRef"]) + return !reflect.DeepEqual(current[internalDNSRegistrationRefField], desired[internalDNSRegistrationRefField]) || !reflect.DeepEqual(current[internalDNSGrantRefField], desired[internalDNSGrantRefField]) } func (r *InternalDNSPublisherReconciler) deletePublishedResources(ctx context.Context, access InternalDNSProjectAccess, namespace string, instanceUID types.UID, keep map[string]struct{}) error { diff --git a/internal/controller/internaldns_publisher_test.go b/internal/controller/internaldns_publisher_test.go index 88f7eafb..6a882852 100644 --- a/internal/controller/internaldns_publisher_test.go +++ b/internal/controller/internaldns_publisher_test.go @@ -21,9 +21,16 @@ import ( ) const ( - internalDNSTestNamespace = "dns-test" - internalDNSTestInstanceName = "api" - internalDNSTestInstanceUID = "dns-instance-uid" + internalDNSTestNamespace = "dns-test" + internalDNSTestInstanceName = "api" + internalDNSTestInstanceUID = "dns-instance-uid" + internalDNSTestEdgeUID = "dns-edge-instance" + internalDNSTestForeignValue = "dns-unauthorized" + internalDNSTestSubject = "system:serviceaccount:dns-test:publisher" + internalDNSTestObservationNamespace = "dns-observation-project" + internalDNSTestNetworkName = "dns-application" + internalDNSTestVPCUID = "dns-test-vpc" + internalDNSTestSourceAPIUID = "dns-project-api" ) func TestInstanceDNSAttachmentsMapsTenancyAndAddressFamilies(t *testing.T) { @@ -77,18 +84,18 @@ func TestFindResolverContextRequiresProjectAndVPCUID(t *testing.T) { s := runtime.NewScheme() s.AddKnownTypeWithName(dnsResolverContextGVK, &unstructured.Unstructured{}) s.AddKnownTypeWithName(dnsResolverContextGVK.GroupVersion().WithKind("DNSResolverContextList"), &unstructured.UnstructuredList{}) - wrong := resolverContextObject("wrong", "other-project", "vpc-a", true) - right := resolverContextObject("right", "project-a", "vpc-a", true) + wrong := resolverContextObject("wrong", "other-project", internalDNSTestVPCUID, true) + right := resolverContextObject("right", "project-a", internalDNSTestVPCUID, true) wrongVPC := resolverContextObject("wrong-vpc", "project-a", "vpc-b", true) - pending := resolverContextObject("pending", "project-a", "vpc-a", false) + pending := resolverContextObject("pending", "project-a", internalDNSTestVPCUID, false) cl := fake.NewClientBuilder().WithScheme(s).WithObjects(wrong, wrongVPC, pending, right).Build() - got, err := findResolverContext(context.Background(), cl, internalDNSTestNamespace, "project-a", "vpc-a") + got, err := findResolverContext(context.Background(), cl, internalDNSTestNamespace, "project-a", internalDNSTestVPCUID) require.NoError(t, err) require.NotNil(t, got) assert.Equal(t, "right", got.GetName()) - got, err = findResolverContext(context.Background(), cl, internalDNSTestNamespace, "project-b", "vpc-a") + got, err = findResolverContext(context.Background(), cl, internalDNSTestNamespace, "project-b", internalDNSTestVPCUID) require.NoError(t, err) assert.Nil(t, got) } @@ -97,11 +104,11 @@ func TestFindResolverContextRejectsAmbiguityAndIgnoresTerminatingBinding(t *test s := runtime.NewScheme() s.AddKnownTypeWithName(dnsResolverContextGVK, &unstructured.Unstructured{}) s.AddKnownTypeWithName(dnsResolverContextGVK.GroupVersion().WithKind("DNSResolverContextList"), &unstructured.UnstructuredList{}) - first := resolverContextObject("first", "project-a", "vpc-a", true) - second := resolverContextObject("second", "project-a", "vpc-a", true) + first := resolverContextObject("first", "project-a", internalDNSTestVPCUID, true) + second := resolverContextObject("second", "project-a", internalDNSTestVPCUID, true) cl := fake.NewClientBuilder().WithScheme(s).WithObjects(first, second).Build() - _, err := findResolverContext(context.Background(), cl, internalDNSTestNamespace, "project-a", "vpc-a") + _, err := findResolverContext(context.Background(), cl, internalDNSTestNamespace, "project-a", internalDNSTestVPCUID) require.Error(t, err) assert.Contains(t, err.Error(), "multiple ready") @@ -109,7 +116,7 @@ func TestFindResolverContextRejectsAmbiguityAndIgnoresTerminatingBinding(t *test second.SetDeletionTimestamp(&now) second.SetFinalizers([]string{"test.example/finalizer"}) cl = fake.NewClientBuilder().WithScheme(s).WithObjects(first, second).Build() - got, err := findResolverContext(context.Background(), cl, internalDNSTestNamespace, "project-a", "vpc-a") + got, err := findResolverContext(context.Background(), cl, internalDNSTestNamespace, "project-a", internalDNSTestVPCUID) require.NoError(t, err) require.NotNil(t, got) assert.Equal(t, "first", got.GetName()) @@ -123,10 +130,10 @@ func resolverContextObject(name, projectUID, vpcUID string, accepted bool) *unst obj := newDNSObject(dnsResolverContextGVK, internalDNSTestNamespace, name) obj.SetUID(types.UID(name + "-uid")) obj.SetGeneration(1) - obj.Object["spec"] = map[string]any{"consumerID": projectUID + "/" + vpcUID} + obj.Object[internalDNSSpecField] = map[string]any{"consumerID": projectUID + "/" + vpcUID} obj.Object[internalDNSStatusField] = map[string]any{ - "conditions": []any{map[string]any{"type": "Accepted", internalDNSStatusField: status, "observedGeneration": int64(1)}, map[string]any{"type": "Ready", internalDNSStatusField: status, "observedGeneration": int64(1)}}, - "managedNamespace": map[string]any{"suffix": "datum.internal", "dnsZoneRef": map[string]any{internalDNSNameField: "zone", internalDNSUIDField: "zone-uid"}}, + internalDNSConditionsField: []any{map[string]any{internalDNSConditionTypeField: "Accepted", internalDNSStatusField: status, internalDNSObservedGenerationField: int64(1)}, map[string]any{internalDNSConditionTypeField: "Ready", internalDNSStatusField: status, internalDNSObservedGenerationField: int64(1)}}, + "managedNamespace": map[string]any{"suffix": "datum.internal", "dnsZoneRef": map[string]any{internalDNSNameField: "zone", internalDNSUIDField: "zone-uid"}}, } return obj } @@ -139,13 +146,13 @@ func TestRefreshObservationPreservesPlatformStatusAndIncrementsFence(t *testing. contribution.SetUID("contribution-uid") contribution.SetGeneration(3) contribution.SetResourceVersion("1") - contribution.Object["spec"] = map[string]any{"recordSets": []any{}} + contribution.Object[internalDNSSpecField] = map[string]any{internalDNSRecordSetsField: []any{}} contribution.Object[internalDNSStatusField] = map[string]any{ - "writerEpoch": int64(7), - "sequence": int64(11), - "eligible": false, - "publishedRevision": int64(44), - "conditions": []any{map[string]any{"type": "Published", internalDNSStatusField: string(metav1.ConditionTrue)}}, + internalDNSWriterEpochField: int64(7), + "sequence": int64(11), + "eligible": false, + "publishedRevision": int64(44), + internalDNSConditionsField: []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionPublished, internalDNSStatusField: string(metav1.ConditionTrue)}}, } cl := fake.NewClientBuilder().WithScheme(s).WithObjects(contribution).WithStatusSubresource(contribution).Build() now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) @@ -163,7 +170,7 @@ func TestRefreshObservationPreservesPlatformStatusAndIncrementsFence(t *testing. require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(contribution), &got)) sequence, _, _ := unstructured.NestedInt64(got.Object, internalDNSStatusField, "sequence") published, _, _ := unstructured.NestedInt64(got.Object, internalDNSStatusField, "publishedRevision") - conditions, _, _ := unstructured.NestedSlice(got.Object, internalDNSStatusField, "conditions") + conditions, _, _ := unstructured.NestedSlice(got.Object, internalDNSStatusField, internalDNSConditionsField) assert.Equal(t, int64(12), sequence) assert.Equal(t, int64(44), published) assert.Len(t, conditions, 1) @@ -175,7 +182,7 @@ func TestEnsureDNSObjectRefusesForeignCollision(t *testing.T) { s.AddKnownTypeWithName(dnsRegistrationGVK.GroupVersion().WithKind("DNSRegistrationList"), &unstructured.UnstructuredList{}) foreign := newDNSObject(dnsRegistrationGVK, internalDNSTestNamespace, "instance-collision") foreign.SetUID("foreign") - foreign.Object["spec"] = map[string]any{internalDNSNameField: "someone-else"} + foreign.Object[internalDNSSpecField] = map[string]any{internalDNSNameField: "someone-else"} cl := fake.NewClientBuilder().WithScheme(s).WithObjects(foreign).Build() owner := metav1.OwnerReference{APIVersion: computev1alpha.GroupVersion.String(), Kind: internalDNSInstanceKind, Name: internalDNSTestInstanceName, UID: internalDNSTestInstanceUID} diff --git a/internal/controller/internaldns_status.go b/internal/controller/internaldns_status.go index c8be645a..e1c4343a 100644 --- a/internal/controller/internaldns_status.go +++ b/internal/controller/internaldns_status.go @@ -45,7 +45,7 @@ func (r *InternalDNSPublisherReconciler) publicationStatus(ctx context.Context, return result, err } if err == nil { - canonical, _, _ := unstructured.NestedString(registration.Object, "status", "canonicalFQDN") + canonical, _, _ := unstructured.NestedString(registration.Object, internalDNSStatusField, "canonicalFQDN") canonical = strings.TrimSuffix(strings.ToLower(canonical), ".") if canonical == allocatedInstanceDNSName(instance)+"."+defaultInternalDNSDomain { result.Hostnames = []string{canonical} @@ -54,10 +54,10 @@ func (r *InternalDNSPublisherReconciler) publicationStatus(ctx context.Context, if err := reader.Get(ctx, client.ObjectKeyFromObject(contribution), contribution); err != nil && !apierrors.IsNotFound(err) { return result, err } - deadlineRaw, _, _ := unstructured.NestedString(contribution.Object, "status", "validUntil") + deadlineRaw, _, _ := unstructured.NestedString(contribution.Object, internalDNSStatusField, "validUntil") deadline, _ := time.Parse(time.RFC3339Nano, deadlineRaw) - if authorized && len(result.Hostnames) > 0 && unstructuredConditionCurrent(registration, "Published") && unstructuredConditionCurrent(contribution, "Published") && deadline.After(r.now()) && attachment.Eligible && attachment.ObservationError == nil { - condition.Status, condition.Reason, condition.Message = metav1.ConditionTrue, "Published", "Private name is published" + if authorized && len(result.Hostnames) > 0 && unstructuredConditionCurrent(registration, internalDNSConditionPublished) && unstructuredConditionCurrent(contribution, internalDNSConditionPublished) && deadline.After(r.now()) && attachment.Eligible && attachment.ObservationError == nil { + condition.Status, condition.Reason, condition.Message = metav1.ConditionTrue, internalDNSConditionPublished, "Private name is published" } } if attachment.ObservationError != nil { diff --git a/internal/controller/internaldns_status_test.go b/internal/controller/internaldns_status_test.go index 190dd8d0..aa2b359e 100644 --- a/internal/controller/internaldns_status_test.go +++ b/internal/controller/internaldns_status_test.go @@ -18,7 +18,7 @@ import ( func TestPublicationStatusRequiresCurrentUnexpiredPublication(t *testing.T) { now := time.Now().UTC().Truncate(time.Second) - instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: "web", UID: "instance-uid", Generation: 2}} + instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestInstanceName, UID: internalDNSTestInstanceUID, Generation: 2}} for _, tc := range []struct { name string deadline time.Time @@ -33,20 +33,20 @@ func TestPublicationStatusRequiresCurrentUnexpiredPublication(t *testing.T) { s := runtime.NewScheme() s.AddKnownTypeWithName(dnsRegistrationGVK, &unstructured.Unstructured{}) s.AddKnownTypeWithName(dnsContributionGVK, &unstructured.Unstructured{}) - registration := newDNSObject(dnsRegistrationGVK, internalDNSTestNamespace, "web") + registration := newDNSObject(dnsRegistrationGVK, internalDNSTestNamespace, internalDNSTestInstanceName) registration.SetGeneration(2) - contribution := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, "web") + contribution := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, internalDNSTestInstanceName) contribution.SetGeneration(2) observed := int64(1) if tc.current { observed = 2 } - conditions := []any{map[string]any{"type": "Published", "status": "True", "observedGeneration": observed}} - registration.Object["status"] = map[string]any{"canonicalFQDN": allocatedInstanceDNSName(instance) + ".datum.internal.", "conditions": conditions} - contribution.Object["status"] = map[string]any{"conditions": conditions, "validUntil": tc.deadline.Format(time.RFC3339Nano)} + conditions := []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionPublished, internalDNSStatusField: string(metav1.ConditionTrue), internalDNSObservedGenerationField: observed}} + registration.Object[internalDNSStatusField] = map[string]any{"canonicalFQDN": allocatedInstanceDNSName(instance) + ".datum.internal.", internalDNSConditionsField: conditions} + contribution.Object[internalDNSStatusField] = map[string]any{internalDNSConditionsField: conditions, "validUntil": tc.deadline.Format(time.RFC3339Nano)} cl := fake.NewClientBuilder().WithScheme(s).WithObjects(registration, contribution).Build() r := &InternalDNSPublisherReconciler{Now: func() time.Time { return now }} - got, err := r.publicationStatus(context.Background(), cl, internalDNSTestNamespace, instance, instanceDNSAttachment{VPCUID: "vpc-a", Eligible: true}, "web", tc.authorized, nil) + got, err := r.publicationStatus(context.Background(), cl, internalDNSTestNamespace, instance, instanceDNSAttachment{VPCUID: internalDNSTestVPCUID, Eligible: true}, internalDNSTestInstanceName, tc.authorized, nil) require.NoError(t, err) require.Equal(t, []string{allocatedInstanceDNSName(instance) + ".datum.internal"}, got.Hostnames) require.Equal(t, tc.wantReady, got.Conditions[0].Status == metav1.ConditionTrue) @@ -57,11 +57,11 @@ func TestPublicationStatusRequiresCurrentUnexpiredPublication(t *testing.T) { func TestReportPublicationPreservesRuntimeStatus(t *testing.T) { s := runtime.NewScheme() require.NoError(t, computev1alpha.AddToScheme(s)) - instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: "web", Namespace: internalDNSTestNamespace}, Status: computev1alpha.InstanceStatus{Conditions: []metav1.Condition{{Type: computev1alpha.InstanceReady, Status: metav1.ConditionTrue}}}} + instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestInstanceName, Namespace: internalDNSTestNamespace}, Status: computev1alpha.InstanceStatus{Conditions: []metav1.Condition{{Type: computev1alpha.InstanceReady, Status: metav1.ConditionTrue}}}} cl := fake.NewClientBuilder().WithScheme(s).WithObjects(instance).WithStatusSubresource(instance).Build() require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(instance), instance)) r := &InternalDNSPublisherReconciler{} - require.NoError(t, r.reportPublication(context.Background(), cl, instance, []computev1alpha.InstanceDNSStatus{{NetworkUID: "vpc-a"}})) + require.NoError(t, r.reportPublication(context.Background(), cl, instance, []computev1alpha.InstanceDNSStatus{{NetworkUID: internalDNSTestVPCUID}})) require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(instance), instance)) require.Equal(t, metav1.ConditionTrue, instance.Status.Conditions[0].Status) require.Len(t, instance.Status.DNS, 1) From ade5d4575a315dedc15a1c42267eeccc33994d2d Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Fri, 9 Oct 2026 21:40:47 -0500 Subject: [PATCH 05/33] Scope runtime observations to the instance slot --- internal/controller/internaldns_observation.go | 10 +++++++++- internal/controller/internaldns_observation_test.go | 5 +++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/internal/controller/internaldns_observation.go b/internal/controller/internaldns_observation.go index 6b9c4610..41519264 100644 --- a/internal/controller/internaldns_observation.go +++ b/internal/controller/internaldns_observation.go @@ -148,7 +148,15 @@ func observeProgrammedInterface(ctx context.Context, reader client.Reader, curre func observeRuntimeNode(ctx context.Context, reader client.Reader, current *computev1alpha.Instance) (string, error) { var pods corev1.PodList - if err := reader.List(ctx, &pods, client.InNamespace(current.Namespace)); err != nil { + options := []client.ListOption{client.InNamespace(current.Namespace)} + deployment, index := current.Labels[computev1alpha.WorkloadDeploymentUIDLabel], current.Labels[computev1alpha.InstanceIndexLabel] + if deployment != "" && index != "" { + options = append(options, client.MatchingLabels{ + computev1alpha.WorkloadDeploymentUIDLabel: deployment, + computev1alpha.InstanceIndexLabel: index, + }) + } + if err := reader.List(ctx, &pods, options...); err != nil { return "", err } var nodeName string diff --git a/internal/controller/internaldns_observation_test.go b/internal/controller/internaldns_observation_test.go index e46419b3..d8b7ca2d 100644 --- a/internal/controller/internaldns_observation_test.go +++ b/internal/controller/internaldns_observation_test.go @@ -44,6 +44,9 @@ func TestObserveInternalDNSInterface(t *testing.T) { {name: "runtime exited", mutate: func(_ *computev1alpha.Instance, _ *networkingv1alpha.NetworkInterface, p *corev1.Pod, _ *corev1.Node, _ *coordinationv1.Lease) { p.Status.Phase = corev1.PodFailed }}, + {name: "different runtime slot", mutate: func(_ *computev1alpha.Instance, _ *networkingv1alpha.NetworkInterface, p *corev1.Pod, _ *corev1.Node, _ *coordinationv1.Lease) { + p.Labels[computev1alpha.InstanceIndexLabel] = "different-slot" + }}, {name: "node unavailable", mutate: func(_ *computev1alpha.Instance, _ *networkingv1alpha.NetworkInterface, _ *corev1.Pod, n *corev1.Node, _ *coordinationv1.Lease) { n.Status.Conditions[0].Status = corev1.ConditionFalse }}, @@ -68,6 +71,8 @@ func TestObserveInternalDNSInterface(t *testing.T) { pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "runtime", Namespace: edgeInstance.Namespace, OwnerReferences: []metav1.OwnerReference{{APIVersion: computev1alpha.GroupVersion.String(), Kind: internalDNSInstanceKind, Name: edgeInstance.Name, UID: edgeInstance.UID, Controller: ptrBool(true)}}}, Spec: corev1.PodSpec{NodeName: "worker"}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionFalse}}}} node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "worker", UID: "node-uid"}, Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}}} lease := &coordinationv1.Lease{ObjectMeta: metav1.ObjectMeta{Name: node.Name, Namespace: "kube-node-lease", OwnerReferences: []metav1.OwnerReference{{Kind: "Node", Name: node.Name, UID: node.UID}}}, Spec: coordinationv1.LeaseSpec{RenewTime: &metav1.MicroTime{Time: now.Add(-10 * time.Second)}}} + edgeInstance.Labels = map[string]string{computev1alpha.WorkloadDeploymentUIDLabel: "dns-deployment", computev1alpha.InstanceIndexLabel: "0"} + pod.Labels = map[string]string{computev1alpha.WorkloadDeploymentUIDLabel: "dns-deployment", computev1alpha.InstanceIndexLabel: "0"} if tc.mutate != nil { tc.mutate(edgeInstance, iface, pod, node, lease) } From 850ca823050bc125497298059a0baf78d0e11e41 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Sun, 11 Oct 2026 14:06:09 -0500 Subject: [PATCH 06/33] Publish instance names using the allocated DNS namespace --- .../internal-dns-instance-publishing.md | 6 ++-- .../internaldns_authorization_test.go | 1 + internal/controller/internaldns_publisher.go | 29 ++++++++++++------- .../controller/internaldns_publisher_test.go | 4 +-- internal/controller/internaldns_status.go | 21 ++++++++++++-- .../controller/internaldns_status_test.go | 28 +++++++++++++++--- 6 files changed, 68 insertions(+), 21 deletions(-) diff --git a/docs/enhancements/internal-dns-instance-publishing.md b/docs/enhancements/internal-dns-instance-publishing.md index 2c154e6d..4bbd43fd 100644 --- a/docs/enhancements/internal-dns-instance-publishing.md +++ b/docs/enhancements/internal-dns-instance-publishing.md @@ -1,7 +1,9 @@ # Internal DNS instance publishing -Compute publishes private instance names in each attached VPC's `datum.internal` -zone. Users attach a workload to a network; they do not select a DNS zone. The +Compute publishes private instance names in the zone allocated for each attached +VPC. Network services selects `datum.internal` as the default domain and requests +the zone from DNS. Compute reads the allocated suffix. Users attach a workload +to a network; they do not select a DNS zone. The assigned label combines the Instance name with its immutable lifetime identity. For example, an instance can receive `web-01-43ce24f81d265cdf.datum.internal`. Each VPC uses an independent DNS context, including when zone names overlap. diff --git a/internal/controller/internaldns_authorization_test.go b/internal/controller/internaldns_authorization_test.go index c1c602c8..c1cb7677 100644 --- a/internal/controller/internaldns_authorization_test.go +++ b/internal/controller/internaldns_authorization_test.go @@ -101,6 +101,7 @@ func TestPublisherWaitsForIssuerAndNeverMutatesGrants(t *testing.T) { s.AddKnownTypeWithName(gvk.GroupVersion().WithKind(gvk.Kind+"List"), &unstructured.UnstructuredList{}) } contextObject := resolverContextObject("context", "project-a", internalDNSTestVPCUID, true) + require.NoError(t, unstructured.SetNestedField(contextObject.Object, "private.example.net", internalDNSStatusField, "managedNamespace", "suffix")) cl := fake.NewClientBuilder().WithScheme(s).WithObjects(contextObject).WithStatusSubresource(newDNSObject(dnsRegistrationGVK, "", ""), newDNSObject(dnsContributionGVK, "", "")).WithInterceptorFuncs(interceptor.Funcs{Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { obj.SetUID(types.UID(obj.GetName() + "-uid")) obj.SetGeneration(1) diff --git a/internal/controller/internaldns_publisher.go b/internal/controller/internaldns_publisher.go index cafb0951..808ff9d0 100644 --- a/internal/controller/internaldns_publisher.go +++ b/internal/controller/internaldns_publisher.go @@ -19,6 +19,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" ctrl "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -39,6 +40,7 @@ const ( internalDNSStatusField = "status" internalDNSReasonField = "reason" internalDNSUIDField = "uid" + internalDNSZoneRefField = "dnsZoneRef" internalDNSGroup = "dns.networking.miloapis.com" internalDNSVersion = "v1alpha1" internalDNSInstanceUID = "internal-dns.compute.datumapis.com/instance-uid" @@ -50,7 +52,6 @@ const ( internalDNSPendingWait = 10 * time.Second internalDNSAPITimeout = 10 * time.Second defaultDNSConcurrency = 4 - defaultInternalDNSDomain = "datum.internal" internalDNSConditionsField = "conditions" internalDNSObservedGenerationField = "observedGeneration" internalDNSWriterEpochField = "writerEpoch" @@ -191,7 +192,7 @@ func (r *InternalDNSPublisherReconciler) Reconcile(ctx context.Context, req mcre resourceName := internalDNSResourceName(projectInstance.UID, attachment.VPCUID) desired[resourceName] = struct{}{} ready, err := r.reconcileAttachment(ctx, access, identity, projectNamespace, &projectInstance, owner, attachment, resourceName) - publication, statusErr := r.publicationStatus(ctx, access.Reader, projectNamespace, &projectInstance, attachment, resourceName, ready, err) + publication, statusErr := r.publicationStatus(ctx, access.Reader, identity, projectNamespace, &projectInstance, attachment, resourceName, ready, err) publicationStatus = append(publicationStatus, publication) if statusErr != nil { publicationErr = errors.Join(publicationErr, statusErr) @@ -312,13 +313,9 @@ func (r *InternalDNSPublisherReconciler) reconcileAttachment(ctx context.Context if managed == nil { return false, nil } - suffix, _, _ := unstructured.NestedString(managed.Object, internalDNSStatusField, "managedNamespace", "suffix") - if strings.TrimSuffix(strings.ToLower(suffix), ".") != defaultInternalDNSDomain { - return false, fmt.Errorf("DNS context %q must allocate the default domain %q", managed.GetName(), defaultInternalDNSDomain) - } - zoneRef, _, _ := unstructured.NestedMap(managed.Object, internalDNSStatusField, "managedNamespace", "dnsZoneRef") - if stringValue(zoneRef, internalDNSNameField) == "" || stringValue(zoneRef, internalDNSUIDField) == "" { - return false, nil + _, zoneRef, err := allocatedDNSNamespace(managed) + if err != nil { + return false, err } labels := map[string]string{ @@ -336,7 +333,7 @@ func (r *InternalDNSPublisherReconciler) reconcileAttachment(ctx context.Context } registrationSpec := map[string]any{ - "dnsZoneRef": zoneRef, + internalDNSZoneRefField: zoneRef, internalDNSNameField: ownerName, internalDNSRecordTypesField: []any{"A", internalDNSRecordTypeAAAA}, "publicationPolicy": "EligibleContributions", @@ -440,7 +437,7 @@ func findResolverContext(ctx context.Context, reader client.Reader, namespace st continue } consumerID, _, _ := unstructured.NestedString(item.Object, internalDNSSpecField, "consumerID") - if consumerID != string(projectUID)+"/"+string(vpcUID) || !unstructuredConditionCurrent(item, "Accepted") || !unstructuredConditionCurrent(item, "Ready") { + if consumerID != string(projectUID)+"/"+string(vpcUID) || !unstructuredConditionCurrent(item, "Ready") { continue } if match != nil { @@ -451,6 +448,16 @@ func findResolverContext(ctx context.Context, reader client.Reader, namespace st return match, nil } +func allocatedDNSNamespace(resolver *unstructured.Unstructured) (string, map[string]any, error) { + suffix, _, _ := unstructured.NestedString(resolver.Object, internalDNSStatusField, "managedNamespace", "suffix") + suffix = strings.TrimSuffix(strings.ToLower(suffix), ".") + zoneRef, _, _ := unstructured.NestedMap(resolver.Object, internalDNSStatusField, "managedNamespace", internalDNSZoneRefField) + if len(validation.IsDNS1123Subdomain(suffix)) > 0 || stringValue(zoneRef, internalDNSNameField) == "" || stringValue(zoneRef, internalDNSUIDField) == "" { + return "", nil, fmt.Errorf("DNS context %q has no valid allocated namespace", resolver.GetName()) + } + return suffix, zoneRef, nil +} + func unstructuredConditionCurrent(obj *unstructured.Unstructured, conditionType string) bool { conditions, _, _ := unstructured.NestedSlice(obj.Object, internalDNSStatusField, internalDNSConditionsField) for _, raw := range conditions { diff --git a/internal/controller/internaldns_publisher_test.go b/internal/controller/internaldns_publisher_test.go index 6a882852..edd4ffc0 100644 --- a/internal/controller/internaldns_publisher_test.go +++ b/internal/controller/internaldns_publisher_test.go @@ -132,8 +132,8 @@ func resolverContextObject(name, projectUID, vpcUID string, accepted bool) *unst obj.SetGeneration(1) obj.Object[internalDNSSpecField] = map[string]any{"consumerID": projectUID + "/" + vpcUID} obj.Object[internalDNSStatusField] = map[string]any{ - internalDNSConditionsField: []any{map[string]any{internalDNSConditionTypeField: "Accepted", internalDNSStatusField: status, internalDNSObservedGenerationField: int64(1)}, map[string]any{internalDNSConditionTypeField: "Ready", internalDNSStatusField: status, internalDNSObservedGenerationField: int64(1)}}, - "managedNamespace": map[string]any{"suffix": "datum.internal", "dnsZoneRef": map[string]any{internalDNSNameField: "zone", internalDNSUIDField: "zone-uid"}}, + internalDNSConditionsField: []any{map[string]any{internalDNSConditionTypeField: "Ready", internalDNSStatusField: status, internalDNSObservedGenerationField: int64(1)}}, + "managedNamespace": map[string]any{"suffix": "datum.internal", internalDNSZoneRefField: map[string]any{internalDNSNameField: "zone", internalDNSUIDField: "zone-uid"}}, } return obj } diff --git a/internal/controller/internaldns_status.go b/internal/controller/internaldns_status.go index e1c4343a..85e2fa81 100644 --- a/internal/controller/internaldns_status.go +++ b/internal/controller/internaldns_status.go @@ -31,7 +31,7 @@ func (r *InternalDNSPublisherReconciler) observeInterface(ctx context.Context, r return ObserveInternalDNSInterface(ctx, reader, namespace, instance, nic, r.ObservationSources[instance.Labels[computev1alpha.LocationLabel]], r.now()) } -func (r *InternalDNSPublisherReconciler) publicationStatus(ctx context.Context, reader client.Reader, namespace string, instance *computev1alpha.Instance, attachment instanceDNSAttachment, resourceName string, authorized bool, reconcileErr error) (computev1alpha.InstanceDNSStatus, error) { +func (r *InternalDNSPublisherReconciler) publicationStatus(ctx context.Context, reader client.Reader, identity InternalDNSProjectIdentity, namespace string, instance *computev1alpha.Instance, attachment instanceDNSAttachment, resourceName string, authorized bool, reconcileErr error) (computev1alpha.InstanceDNSStatus, error) { result := computev1alpha.InstanceDNSStatus{Network: attachment.Network, NetworkUID: string(attachment.VPCUID)} for _, previous := range instance.Status.DNS { if previous.NetworkUID == result.NetworkUID { @@ -47,7 +47,11 @@ func (r *InternalDNSPublisherReconciler) publicationStatus(ctx context.Context, if err == nil { canonical, _, _ := unstructured.NestedString(registration.Object, internalDNSStatusField, "canonicalFQDN") canonical = strings.TrimSuffix(strings.ToLower(canonical), ".") - if canonical == allocatedInstanceDNSName(instance)+"."+defaultInternalDNSDomain { + matches, err := publicationMatchesContext(ctx, reader, identity, namespace, instance, attachment, registration, canonical) + if err != nil { + return result, err + } + if matches { result.Hostnames = []string{canonical} } contribution := newDNSObject(dnsContributionGVK, namespace, resourceName) @@ -72,6 +76,19 @@ func (r *InternalDNSPublisherReconciler) publicationStatus(ctx context.Context, return result, nil } +func publicationMatchesContext(ctx context.Context, reader client.Reader, identity InternalDNSProjectIdentity, namespace string, instance *computev1alpha.Instance, attachment instanceDNSAttachment, registration *unstructured.Unstructured, canonical string) (bool, error) { + resolver, err := findResolverContext(ctx, reader, namespace, identity.ProjectUID, attachment.VPCUID) + if err != nil || resolver == nil { + return false, err + } + suffix, zone, err := allocatedDNSNamespace(resolver) + if err != nil { + return false, err + } + registeredZone, _, _ := unstructured.NestedMap(registration.Object, internalDNSSpecField, internalDNSZoneRefField) + return canonical == allocatedInstanceDNSName(instance)+"."+suffix && stringValue(registeredZone, internalDNSNameField) == stringValue(zone, internalDNSNameField) && stringValue(registeredZone, internalDNSUIDField) == stringValue(zone, internalDNSUIDField), nil +} + func (r *InternalDNSPublisherReconciler) reportPublication(ctx context.Context, writer client.Client, instance *computev1alpha.Instance, status []computev1alpha.InstanceDNSStatus) error { if reflect.DeepEqual(instance.Status.DNS, status) { return nil diff --git a/internal/controller/internaldns_status_test.go b/internal/controller/internaldns_status_test.go index aa2b359e..2699e18d 100644 --- a/internal/controller/internaldns_status_test.go +++ b/internal/controller/internaldns_status_test.go @@ -21,16 +21,27 @@ func TestPublicationStatusRequiresCurrentUnexpiredPublication(t *testing.T) { instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestInstanceName, UID: internalDNSTestInstanceUID, Generation: 2}} for _, tc := range []struct { name string + domain string + foreignZone bool deadline time.Time current, authorized, wantReady bool }{ {name: "published", deadline: now.Add(time.Minute), current: true, authorized: true, wantReady: true}, + {name: "custom domain", domain: "private.example.net", deadline: now.Add(time.Minute), current: true, authorized: true, wantReady: true}, + {name: "different zone lifetime", foreignZone: true, deadline: now.Add(time.Minute), current: true, authorized: true}, {name: "expired", deadline: now.Add(-time.Second), current: true, authorized: true}, {name: "stale generation", deadline: now.Add(time.Minute), authorized: true}, {name: "revoked grant", deadline: now.Add(time.Minute), current: true}, } { t.Run(tc.name, func(t *testing.T) { + if tc.domain == "" { + tc.domain = "datum.internal" + } s := runtime.NewScheme() + s.AddKnownTypeWithName(dnsResolverContextGVK, &unstructured.Unstructured{}) + s.AddKnownTypeWithName(dnsResolverContextGVK.GroupVersion().WithKind("DNSResolverContextList"), &unstructured.UnstructuredList{}) + resolver := resolverContextObject("context", "project-a", internalDNSTestVPCUID, true) + require.NoError(t, unstructured.SetNestedField(resolver.Object, tc.domain, internalDNSStatusField, "managedNamespace", "suffix")) s.AddKnownTypeWithName(dnsRegistrationGVK, &unstructured.Unstructured{}) s.AddKnownTypeWithName(dnsContributionGVK, &unstructured.Unstructured{}) registration := newDNSObject(dnsRegistrationGVK, internalDNSTestNamespace, internalDNSTestInstanceName) @@ -42,13 +53,22 @@ func TestPublicationStatusRequiresCurrentUnexpiredPublication(t *testing.T) { observed = 2 } conditions := []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionPublished, internalDNSStatusField: string(metav1.ConditionTrue), internalDNSObservedGenerationField: observed}} - registration.Object[internalDNSStatusField] = map[string]any{"canonicalFQDN": allocatedInstanceDNSName(instance) + ".datum.internal.", internalDNSConditionsField: conditions} + registration.Object[internalDNSStatusField] = map[string]any{"canonicalFQDN": allocatedInstanceDNSName(instance) + "." + tc.domain + ".", internalDNSConditionsField: conditions} + zoneUID := "zone-uid" + if tc.foreignZone { + zoneUID = internalDNSTestForeignValue + } + registration.Object[internalDNSSpecField] = map[string]any{internalDNSZoneRefField: map[string]any{internalDNSNameField: "zone", internalDNSUIDField: zoneUID}} contribution.Object[internalDNSStatusField] = map[string]any{internalDNSConditionsField: conditions, "validUntil": tc.deadline.Format(time.RFC3339Nano)} - cl := fake.NewClientBuilder().WithScheme(s).WithObjects(registration, contribution).Build() + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(resolver, registration, contribution).Build() r := &InternalDNSPublisherReconciler{Now: func() time.Time { return now }} - got, err := r.publicationStatus(context.Background(), cl, internalDNSTestNamespace, instance, instanceDNSAttachment{VPCUID: internalDNSTestVPCUID, Eligible: true}, internalDNSTestInstanceName, tc.authorized, nil) + got, err := r.publicationStatus(context.Background(), cl, InternalDNSProjectIdentity{ProjectUID: "project-a"}, internalDNSTestNamespace, instance, instanceDNSAttachment{VPCUID: internalDNSTestVPCUID, Eligible: true}, internalDNSTestInstanceName, tc.authorized, nil) require.NoError(t, err) - require.Equal(t, []string{allocatedInstanceDNSName(instance) + ".datum.internal"}, got.Hostnames) + if tc.foreignZone { + require.Empty(t, got.Hostnames) + } else { + require.Equal(t, []string{allocatedInstanceDNSName(instance) + "." + tc.domain}, got.Hostnames) + } require.Equal(t, tc.wantReady, got.Conditions[0].Status == metav1.ConditionTrue) }) } From 55018abbc0ac8438c67de427967e167511b03b59 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 12:27:20 -0500 Subject: [PATCH 07/33] test: capture multi-location dependency readiness failures --- .../chainsaw-test.yaml | 8 ++++++ .../diagnose-referenced-data.sh | 27 +++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 test/e2e/referenced-data-multi-location/diagnose-referenced-data.sh diff --git a/test/e2e/referenced-data-multi-location/chainsaw-test.yaml b/test/e2e/referenced-data-multi-location/chainsaw-test.yaml index 4802fce3..9129b696 100644 --- a/test/e2e/referenced-data-multi-location/chainsaw-test.yaml +++ b/test/e2e/referenced-data-multi-location/chainsaw-test.yaml @@ -444,6 +444,10 @@ spec: status: "True" reason: "Ready" + catch: + - script: + content: sh diagnose-referenced-data.sh + - name: assert-instance-ready-on-pop-ord description: | Assert the ord cell did the same. Under the retired delivery path this is @@ -471,6 +475,10 @@ spec: status: "True" reason: "Ready" + catch: + - script: + content: sh diagnose-referenced-data.sh + - name: assert-referenced-data-gate-cleared-on-both-cells description: | Assert the gate itself is gone from both Instances, not just that the diff --git a/test/e2e/referenced-data-multi-location/diagnose-referenced-data.sh b/test/e2e/referenced-data-multi-location/diagnose-referenced-data.sh new file mode 100644 index 00000000..19849d9e --- /dev/null +++ b/test/e2e/referenced-data-multi-location/diagnose-referenced-data.sh @@ -0,0 +1,27 @@ +#!/bin/sh +set -eu + +KDIR=../../../tmp/e2e/kubeconfigs +DOWNSTREAM_NS=$(kubectl --kubeconfig="$KDIR/control-plane.yaml" \ + get namespace "$NAMESPACE" -o template='{{printf "ns-%s" .metadata.uid}}') + +for PLANE in control-plane karmada pop-dfw pop-ord; do + NS="$DOWNSTREAM_NS" + if [ "$PLANE" = control-plane ]; then + NS="$NAMESPACE" + fi + kubectl --kubeconfig="$KDIR/$PLANE.yaml" -n "$NS" \ + get workloaddeployments -o json | jq --arg plane "$PLANE" \ + '.items[] | {plane: $plane, name: .metadata.name, + expected: .metadata.annotations["compute.datumapis.com/expected-referenced-data"], + conditions: .status.conditions}' +done + +for CELL in pop-dfw pop-ord; do + kubectl --kubeconfig="$KDIR/$CELL.yaml" -n "$DOWNSTREAM_NS" \ + get instances -o json | jq --arg cell "$CELL" \ + '.items[] | {cell: $cell, name: .metadata.name, + owner: .metadata.ownerReferences, + gates: .spec.controller.schedulingGates, + conditions: .status.conditions}' +done From 05c0d7edc9aedd9d4ddae77a138c31f0f4ead34c Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 12:29:20 -0500 Subject: [PATCH 08/33] fix: share the DNS readiness condition name --- internal/controller/internaldns_publisher.go | 3 ++- internal/controller/internaldns_publisher_test.go | 2 +- internal/controller/internaldns_status.go | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/controller/internaldns_publisher.go b/internal/controller/internaldns_publisher.go index 808ff9d0..9db20ce8 100644 --- a/internal/controller/internaldns_publisher.go +++ b/internal/controller/internaldns_publisher.go @@ -60,6 +60,7 @@ const ( internalDNSNameScopesField = "nameScopes" internalDNSClusterUIDField = "clusterUID" internalDNSConditionActive = "Active" + internalDNSConditionReady = "Ready" internalDNSConditionPublished = "Published" internalDNSPrincipalField = "principal" internalDNSConditionTypeField = "type" @@ -437,7 +438,7 @@ func findResolverContext(ctx context.Context, reader client.Reader, namespace st continue } consumerID, _, _ := unstructured.NestedString(item.Object, internalDNSSpecField, "consumerID") - if consumerID != string(projectUID)+"/"+string(vpcUID) || !unstructuredConditionCurrent(item, "Ready") { + if consumerID != string(projectUID)+"/"+string(vpcUID) || !unstructuredConditionCurrent(item, internalDNSConditionReady) { continue } if match != nil { diff --git a/internal/controller/internaldns_publisher_test.go b/internal/controller/internaldns_publisher_test.go index edd4ffc0..522ad9aa 100644 --- a/internal/controller/internaldns_publisher_test.go +++ b/internal/controller/internaldns_publisher_test.go @@ -132,7 +132,7 @@ func resolverContextObject(name, projectUID, vpcUID string, accepted bool) *unst obj.SetGeneration(1) obj.Object[internalDNSSpecField] = map[string]any{"consumerID": projectUID + "/" + vpcUID} obj.Object[internalDNSStatusField] = map[string]any{ - internalDNSConditionsField: []any{map[string]any{internalDNSConditionTypeField: "Ready", internalDNSStatusField: status, internalDNSObservedGenerationField: int64(1)}}, + internalDNSConditionsField: []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionReady, internalDNSStatusField: status, internalDNSObservedGenerationField: int64(1)}}, "managedNamespace": map[string]any{"suffix": "datum.internal", internalDNSZoneRefField: map[string]any{internalDNSNameField: "zone", internalDNSUIDField: "zone-uid"}}, } return obj diff --git a/internal/controller/internaldns_status.go b/internal/controller/internaldns_status.go index 85e2fa81..6b4762e3 100644 --- a/internal/controller/internaldns_status.go +++ b/internal/controller/internaldns_status.go @@ -38,7 +38,7 @@ func (r *InternalDNSPublisherReconciler) publicationStatus(ctx context.Context, result.Conditions = append([]metav1.Condition(nil), previous.Conditions...) } } - condition := metav1.Condition{Type: "Ready", Status: metav1.ConditionFalse, Reason: "PendingPublication", Message: "Waiting for DNS publication", ObservedGeneration: instance.Generation} + condition := metav1.Condition{Type: internalDNSConditionReady, Status: metav1.ConditionFalse, Reason: "PendingPublication", Message: "Waiting for DNS publication", ObservedGeneration: instance.Generation} registration := newDNSObject(dnsRegistrationGVK, namespace, resourceName) err := reader.Get(ctx, client.ObjectKeyFromObject(registration), registration) if err != nil && !apierrors.IsNotFound(err) { From 0327f40afd8d2605350c3dd4c6552c2856bdc9ec Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 18:30:41 -0500 Subject: [PATCH 09/33] fix: Preserve DNS through observation outages --- api/v1alpha/instance_types.go | 2 +- cmd/main.go | 21 ++--- cmd/main_test.go | 22 +---- ...datumapis.com_instanceconsolesessions.yaml | 2 +- .../compute.datumapis.com_instances.yaml | 4 +- .../compute.datumapis.com_instancetypes.yaml | 2 +- .../compute.datumapis.com_runtimeclasses.yaml | 2 +- ...ute.datumapis.com_workloaddeployments.yaml | 2 +- .../compute.datumapis.com_workloads.yaml | 2 +- .../internal-dns-instance-publishing.md | 92 +++++++------------ internal/config/config.go | 17 +--- internal/config/config_test.go | 11 +-- internal/config/zz_generated.deepcopy.go | 1 - .../internaldns_authorization_test.go | 46 +++++----- .../controller/internaldns_observation.go | 79 ++++++---------- .../internaldns_observation_test.go | 41 +++------ internal/controller/internaldns_publisher.go | 80 ++++------------ .../controller/internaldns_publisher_test.go | 6 +- internal/controller/internaldns_status.go | 7 +- .../controller/internaldns_status_test.go | 14 ++- test/internaldns/README.md | 11 +-- test/internaldns/publisher/main.go | 18 ++-- 22 files changed, 166 insertions(+), 316 deletions(-) diff --git a/api/v1alpha/instance_types.go b/api/v1alpha/instance_types.go index 195554aa..1c8bf82c 100644 --- a/api/v1alpha/instance_types.go +++ b/api/v1alpha/instance_types.go @@ -814,7 +814,7 @@ type InstanceStatus struct { NetworkInterfaces []InstanceNetworkInterfaceStatus `json:"networkInterfaces,omitempty"` // DNS reports assigned private names and publication readiness per network. - // Resolver authorization and publisher leases are managed by platform services. + // Resolver access and record eligibility are managed by platform services. // +kubebuilder:validation:Optional // +listType=map // +listMapKey=networkUID diff --git a/cmd/main.go b/cmd/main.go index a22a1e89..e2b19c6d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -467,7 +467,6 @@ func main() { dnsPublisher := &controller.InternalDNSPublisherReconciler{ ProjectIdentities: identities, PrincipalSubject: serverConfig.InternalDNS.PrincipalSubject, - LeaseDuration: serverConfig.InternalDNS.LeaseDuration.Duration, ObservationSources: observers, } if err = dnsPublisher.SetupWithManager(mgr); err != nil { @@ -979,8 +978,8 @@ func internalDNSObservationSources( ) (map[string]controller.InternalDNSObservationSource, error) { sources := make(map[string]controller.InternalDNSObservationSource, len(cfg.ObservationSources)) for i, source := range cfg.ObservationSources { - if source.Location == "" || source.ClusterUID == "" || source.KubeconfigPath == "" { - return nil, fmt.Errorf("internalDNS.observationSources[%d] requires location, clusterUID, and kubeconfigPath", i) + if source.Location == "" || source.KubeconfigPath == "" { + return nil, fmt.Errorf("internalDNS.observationSources[%d] requires location and kubeconfigPath", i) } if _, exists := sources[source.Location]; exists { return nil, fmt.Errorf("duplicate internal DNS observation location %q", source.Location) @@ -994,7 +993,7 @@ func internalDNSObservationSources( if err != nil { return nil, fmt.Errorf("create network observation client for %q: %w", source.Location, err) } - sources[source.Location] = controller.InternalDNSObservationSource{Reader: reader, ClusterUID: source.ClusterUID} + sources[source.Location] = controller.InternalDNSObservationSource{Reader: reader} } if len(sources) == 0 { return nil, errors.New("internalDNS.observationSources must contain at least one authenticated edge API") @@ -1010,24 +1009,20 @@ func internalDNSProjectIdentities( } identities := make(map[string]controller.InternalDNSProjectIdentity, len(cfg.Projects)) for i, project := range cfg.Projects { - if project.Name == "" || project.ProjectUID == "" || project.SourceClusterUID == "" { - return nil, fmt.Errorf("internalDNS.projects[%d] requires name, projectUID, and sourceClusterUID", i) + if project.Name == "" || project.ProjectUID == "" { + return nil, fmt.Errorf("internalDNS.projects[%d] requires name and projectUID", i) } if _, duplicate := identities[project.Name]; duplicate { return nil, fmt.Errorf("internalDNS.projects contains duplicate project %q", project.Name) } identities[project.Name] = controller.InternalDNSProjectIdentity{ - ProjectName: project.Name, - ProjectUID: project.ProjectUID, - SourceClusterUID: project.SourceClusterUID, + ProjectName: project.Name, + ProjectUID: project.ProjectUID, } } if len(identities) == 0 { return nil, errors.New("internalDNS.projects must contain at least one project") } - if cfg.LeaseDuration.Duration > 90*time.Second { - return nil, fmt.Errorf( - "internalDNS.leaseDuration %s exceeds the DNS platform maximum 90s", cfg.LeaseDuration.Duration) - } + return identities, nil } diff --git a/cmd/main_test.go b/cmd/main_test.go index f64a910f..75f64ad3 100644 --- a/cmd/main_test.go +++ b/cmd/main_test.go @@ -28,24 +28,12 @@ import ( func TestInternalDNSProjectIdentities(t *testing.T) { identities, err := internalDNSProjectIdentities(config.InternalDNSConfig{ PrincipalSubject: "system:serviceaccount:compute-system:compute-manager", - LeaseDuration: metav1.Duration{Duration: 60 * time.Second}, - Projects: []config.InternalDNSProjectIdentity{{ - Name: "project-a", ProjectUID: "project-uid", SourceClusterUID: "source-uid", - }}, + Projects: []config.InternalDNSProjectIdentity{{Name: "project-a", ProjectUID: "project-uid"}}, }) require.NoError(t, err) - assert.Equal(t, "source-uid", identities["project-a"].SourceClusterUID) - + assert.EqualValues(t, "project-uid", identities["project-a"].ProjectUID) _, err = internalDNSProjectIdentities(config.InternalDNSConfig{PrincipalSubject: "subject"}) - assert.Error(t, err, "enabled publishing requires at least one pinned project identity") - _, err = internalDNSProjectIdentities(config.InternalDNSConfig{ - PrincipalSubject: "subject", - LeaseDuration: metav1.Duration{Duration: 91 * time.Second}, - Projects: []config.InternalDNSProjectIdentity{{ - Name: "project-a", ProjectUID: "project-uid", SourceClusterUID: "source-uid", - }}, - }) - assert.Error(t, err, "a producer must not configure a lease DNS admission rejects") + assert.Error(t, err) } func TestInternalDNSObservationSourcesRequireExplicitCredentials(t *testing.T) { @@ -68,12 +56,12 @@ contexts: cluster: edge current-context: edge `), 0o600)) - source := config.InternalDNSObservationSource{Location: "dfw", ClusterUID: "edge-uid", KubeconfigPath: path} + source := config.InternalDNSObservationSource{Location: "dfw", KubeconfigPath: path} sources, err := internalDNSObservationSources(config.InternalDNSConfig{ ObservationSources: []config.InternalDNSObservationSource{source}, }) require.NoError(t, err) - require.EqualValues(t, "edge-uid", sources["dfw"].ClusterUID) + require.NotNil(t, sources["dfw"].Reader) _, err = internalDNSObservationSources(config.InternalDNSConfig{ ObservationSources: []config.InternalDNSObservationSource{source, source}, }) diff --git a/config/base/crd/bases/compute.datumapis.com_instanceconsolesessions.yaml b/config/base/crd/bases/compute.datumapis.com_instanceconsolesessions.yaml index 1f0347b8..0cc7ace1 100644 --- a/config/base/crd/bases/compute.datumapis.com_instanceconsolesessions.yaml +++ b/config/base/crd/bases/compute.datumapis.com_instanceconsolesessions.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.4 + controller-gen.kubebuilder.io/version: v0.19.0 discovery.miloapis.com/parent-contexts: Project name: instanceconsolesessions.compute.datumapis.com spec: diff --git a/config/base/crd/bases/compute.datumapis.com_instances.yaml b/config/base/crd/bases/compute.datumapis.com_instances.yaml index 65ab045b..7fe00402 100644 --- a/config/base/crd/bases/compute.datumapis.com_instances.yaml +++ b/config/base/crd/bases/compute.datumapis.com_instances.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.4 + controller-gen.kubebuilder.io/version: v0.19.0 discovery.miloapis.com/parent-contexts: Project name: instances.compute.datumapis.com spec: @@ -1263,7 +1263,7 @@ spec: dns: description: |- DNS reports assigned private names and publication readiness per network. - Resolver authorization and publisher leases are managed by platform services. + Resolver access and record eligibility are managed by platform services. items: description: InstanceDNSStatus describes private name publication within one network. diff --git a/config/base/crd/bases/compute.datumapis.com_instancetypes.yaml b/config/base/crd/bases/compute.datumapis.com_instancetypes.yaml index ccf11e3b..235c4dff 100644 --- a/config/base/crd/bases/compute.datumapis.com_instancetypes.yaml +++ b/config/base/crd/bases/compute.datumapis.com_instancetypes.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.4 + controller-gen.kubebuilder.io/version: v0.19.0 discovery.miloapis.com/parent-contexts: Platform,Project name: instancetypes.compute.datumapis.com spec: diff --git a/config/base/crd/bases/compute.datumapis.com_runtimeclasses.yaml b/config/base/crd/bases/compute.datumapis.com_runtimeclasses.yaml index 0002298a..a1e8d12e 100644 --- a/config/base/crd/bases/compute.datumapis.com_runtimeclasses.yaml +++ b/config/base/crd/bases/compute.datumapis.com_runtimeclasses.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.4 + controller-gen.kubebuilder.io/version: v0.19.0 discovery.miloapis.com/parent-contexts: Platform,Project name: runtimeclasses.compute.datumapis.com spec: diff --git a/config/base/crd/bases/compute.datumapis.com_workloaddeployments.yaml b/config/base/crd/bases/compute.datumapis.com_workloaddeployments.yaml index 6b74d7fb..488bdfcb 100644 --- a/config/base/crd/bases/compute.datumapis.com_workloaddeployments.yaml +++ b/config/base/crd/bases/compute.datumapis.com_workloaddeployments.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.4 + controller-gen.kubebuilder.io/version: v0.19.0 discovery.miloapis.com/parent-contexts: Project name: workloaddeployments.compute.datumapis.com spec: diff --git a/config/base/crd/bases/compute.datumapis.com_workloads.yaml b/config/base/crd/bases/compute.datumapis.com_workloads.yaml index 325a8306..a169e646 100644 --- a/config/base/crd/bases/compute.datumapis.com_workloads.yaml +++ b/config/base/crd/bases/compute.datumapis.com_workloads.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.4 + controller-gen.kubebuilder.io/version: v0.19.0 discovery.miloapis.com/parent-contexts: Project name: workloads.compute.datumapis.com spec: diff --git a/docs/enhancements/internal-dns-instance-publishing.md b/docs/enhancements/internal-dns-instance-publishing.md index 4bbd43fd..bf320f2e 100644 --- a/docs/enhancements/internal-dns-instance-publishing.md +++ b/docs/enhancements/internal-dns-instance-publishing.md @@ -1,16 +1,14 @@ # Internal DNS instance publishing -Compute publishes private instance names in the zone allocated for each attached -VPC. Network services selects `datum.internal` as the default domain and requests -the zone from DNS. Compute reads the allocated suffix. Users attach a workload -to a network; they do not select a DNS zone. The -assigned label combines the Instance name with its immutable lifetime identity. -For example, an instance can receive `web-01-43ce24f81d265cdf.datum.internal`. -Each VPC uses an independent DNS context, including when zone names overlap. +Compute publishes private instance names in each attached network's managed +zone. Network services requests `datum.internal` by default. Compute reads the +allocated suffix, so users attach workloads to networks without selecting zones. +An instance named `web-01` receives `web-01.datum.internal`. Overlapping domains +remain isolated by the network's DNS context. -`Instance.status.dns` reports assigned hostnames and publication readiness per -network. DNS errors do not change runtime readiness. Application readiness does -not remove an instance's identity name. +`Instance.status.dns` reports hostnames and publication readiness per network. +DNS failures do not affect runtime readiness. Application readiness does not +remove an instance identity name. ```yaml status: @@ -18,20 +16,16 @@ status: - network: name: application namespace: production - networkUID: 00000000-0000-0000-0000-000000000020 - hostnames: [web-01-43ce24f81d265cdf.datum.internal] + hostnames: [web-01.datum.internal] conditions: - type: Ready status: "True" reason: Published message: Private name is published - observedGeneration: 1 - lastTransitionTime: "2026-10-09T20:00:00Z" ``` -Enable the default-off `InternalDNSPublishing` feature gate only after -provisioning the DNS platform, project permissions, and edge observation access. -The management controller requires Milo discovery. +Enable the default-off `InternalDNSPublishing` feature gate after provisioning +DNS, project permissions, and read-only edge observation credentials. ```yaml apiVersion: apiserver.config.datumapis.com/v1alpha1 @@ -39,55 +33,31 @@ kind: WorkloadOperator internalDNS: # Match the authenticated subject used for project DNS writes. principalSubject: system:serviceaccount:compute-system:compute-manager - leaseDuration: 60s projects: - name: example-project projectUID: 00000000-0000-0000-0000-000000000001 - # Match the project source API identity trusted by DNS admission. - sourceClusterUID: 00000000-0000-0000-0000-000000000010 observationSources: - location: dfw - # Pin the edge cluster's kube-system namespace lifetime. - clusterUID: 00000000-0000-0000-0000-000000000030 - # Use separately provisioned read-only edge credentials. + # Authenticate to the edge API that owns instance observations. kubeconfigPath: /etc/compute/edge-dfw.kubeconfig ``` -Compute selects the ready resolver context whose `consumerID` is -`/` and reads its managed zone. It reserves a name through -`DNSRegistration` and waits for a trusted issuer's scoped `DNSContributionGrant`. -Compute can read grants but cannot create, update, or delete them. Contributions -pin both registration and grant lifetimes and generations. DNS admission checks -the authenticated publisher subject and source identity. - -The optional `config/components/internal-dns-rbac` component grants DNS access -on a direct Kubernetes API. Milo project authorization must grant equivalent -permissions separately, including Instance status writes. Applying a role in -the deployment cluster does not authorize project API access. - -Edge observation credentials require reads of Instances, NetworkInterfaces, -NetworkInterfaceClaims, NetworkContexts, Pods, Nodes, node leases, and the `kube-system` namespace. -Scope namespaced reads to the supported projects' edge namespaces. Credentials -cannot publish records or issue grants. Cluster-wide reads are limited to nodes -and the pinned namespace. - -Only private IPv4 and IPv6 host addresses from live allocated, programmed -interfaces are eligible. Public addresses and delegated prefixes are excluded. -The observer checks the source Instance lifetime, interface claim, running -runtime Pod, ready Node, and its original heartbeat. Publication cannot outlive -that heartbeat by more than 45 seconds or exceed the configured lease. -Unsupported runtimes, cross-namespace network attachments, ambiguous runtime -Pods, and observation failures are ineligible. Projected positive status cannot -renew a lease. This requires a runtime represented by an Instance-owned Pod and -a node heartbeat lease. - -Interface withdrawal makes the contribution ineligible. Instance deletion and -replacement reclaim records through project-side owner references. DNS retains -ownership of publication status; Compute patches only its observation fields. - -Guest resolver application belongs to workload providers. It requires the -access-restricted network services delivery contract described in the -[Compute enhancement](https://github.com/datum-cloud/compute/pull/470). -This publisher does not configure guests, expose authorization leases on public -NetworkInterfaces, or implement peer DNS aliases. Supported guests use -`datum.internal` as their default search domain when resolver delivery is enabled. +Compute selects the network's resolver context and reads its managed zone. It +reserves a name through `DNSRegistration`, reads an independently issued scoped +`DNSContributionGrant`, and publishes `DNSRecordContribution` resources. Project +RBAC authorizes writes. Object lifetime references and writer versions prevent +stale publishers from changing replacement resources. + +Only allocated, programmed private host addresses are eligible. Confirmed +interface withdrawal, stopped runtime, or unhealthy node withdraws addresses. +An unavailable observation source or unknown node health leaves published records +unchanged and reports unknown observation health. Records have no renewal +requirement. Instance deletion reclaims records through project owner references. + +Observation credentials read Instances, NetworkInterfaces, NetworkInterfaceClaims, +NetworkContexts, Pods, Nodes, and project namespaces. Scope namespaced reads to +supported edge namespaces. These credentials cannot publish records or issue grants. + +Workload providers apply the network's resolver address and allocated search +domain to guests. Resolver delivery belongs to network services. The publisher +contract does not expose access deadlines on public NetworkInterfaces. diff --git a/internal/config/config.go b/internal/config/config.go index 39d77e42..68dfeafd 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -64,12 +64,7 @@ type InternalDNSConfig struct { // is not an authentication substitute. PrincipalSubject string `json:"principalSubject,omitempty"` - // LeaseDuration bounds how long an instance address can remain published - // without a fresh observation. DNS admission currently permits at most 90s. - LeaseDuration metav1.Duration `json:"leaseDuration,omitempty"` - - // Projects pins each Milo project name to the trusted project and source API - // UIDs configured by the DNS control plane. + // Projects identifies the consumer project APIs. Projects []InternalDNSProjectIdentity `json:"projects,omitempty"` // ObservationSources selects authenticated edge APIs for live interface and @@ -79,16 +74,14 @@ type InternalDNSConfig struct { // +k8s:deepcopy-gen=true type InternalDNSObservationSource struct { - Location string `json:"location"` - ClusterUID types.UID `json:"clusterUID"` - KubeconfigPath string `json:"kubeconfigPath"` + Location string `json:"location"` + KubeconfigPath string `json:"kubeconfigPath"` } // +k8s:deepcopy-gen=true type InternalDNSProjectIdentity struct { - Name string `json:"name"` - ProjectUID types.UID `json:"projectUID"` - SourceClusterUID string `json:"sourceClusterUID"` + Name string `json:"name"` + ProjectUID types.UID `json:"projectUID"` } func SetDefaults_WorkloadOperator(obj *WorkloadOperator) { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index bb421355..981e7c1e 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -4,7 +4,6 @@ import ( "os" "path/filepath" "testing" - "time" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -37,23 +36,17 @@ metricsServer: bindAddress: "0" internalDNS: principalSubject: system:serviceaccount:compute-system:compute-manager - leaseDuration: 60s observationSources: - location: dfw - clusterUID: edge-uid kubeconfigPath: /etc/compute/edge.kubeconfig projects: - name: project-a projectUID: project-uid - sourceClusterUID: source-uid `) - if cfg.InternalDNS.LeaseDuration.Duration != 60*time.Second { - t.Fatalf("leaseDuration = %s, want 60s", cfg.InternalDNS.LeaseDuration.Duration) - } copy := cfg.DeepCopy() copy.InternalDNS.Projects[0].ProjectUID = "changed" - copy.InternalDNS.ObservationSources[0].ClusterUID = "changed" - if cfg.InternalDNS.ObservationSources[0].ClusterUID != "edge-uid" { + copy.InternalDNS.ObservationSources[0].Location = "changed" + if cfg.InternalDNS.ObservationSources[0].Location != "dfw" { t.Fatal("WorkloadOperator DeepCopy aliases observation identities") } if cfg.InternalDNS.Projects[0].ProjectUID != "project-uid" { diff --git a/internal/config/zz_generated.deepcopy.go b/internal/config/zz_generated.deepcopy.go index 3b79b828..5928a9ad 100644 --- a/internal/config/zz_generated.deepcopy.go +++ b/internal/config/zz_generated.deepcopy.go @@ -69,7 +69,6 @@ func (in *FeatureFlagsConfig) DeepCopy() *FeatureFlagsConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InternalDNSConfig) DeepCopyInto(out *InternalDNSConfig) { *out = *in - out.LeaseDuration = in.LeaseDuration if in.Projects != nil { in, out := &in.Projects, &out.Projects *out = make([]InternalDNSProjectIdentity, len(*in)) diff --git a/internal/controller/internaldns_authorization_test.go b/internal/controller/internaldns_authorization_test.go index c1cb7677..a2449852 100644 --- a/internal/controller/internaldns_authorization_test.go +++ b/internal/controller/internaldns_authorization_test.go @@ -31,7 +31,6 @@ func TestFindContributionGrantRequiresCurrentScopedAuthority(t *testing.T) { }{ {name: "active matching grant", found: true}, {name: "different principal", path: []string{internalDNSSpecField, internalDNSPrincipalField, internalDNSSubjectField}, value: internalDNSTestForeignValue}, - {name: "different source cluster", path: []string{internalDNSSpecField, internalDNSPrincipalField, internalDNSClusterUIDField}, value: internalDNSTestForeignValue}, {name: "different registration lifetime", path: []string{internalDNSSpecField, internalDNSRegistrationRefField, "uid"}, value: internalDNSTestForeignValue}, {name: "stale registration generation", path: []string{internalDNSSpecField, internalDNSRegistrationRefField, "generation"}, value: int64(1)}, {name: "different name scope", path: []string{internalDNSSpecField, internalDNSNameScopesField}, value: []any{internalDNSTestForeignValue}}, @@ -46,51 +45,52 @@ func TestFindContributionGrantRequiresCurrentScopedAuthority(t *testing.T) { grant := newDNSObject(dnsGrantGVK, internalDNSTestNamespace, "trusted-grant") grant.SetUID("grant-uid") grant.SetGeneration(1) - grant.Object[internalDNSSpecField] = map[string]any{internalDNSRegistrationRefField: objectReference(registration), internalDNSPrincipalField: map[string]any{internalDNSClusterUIDField: internalDNSTestSourceAPIUID, internalDNSSubjectField: internalDNSTestSubject}, internalDNSNameScopesField: []any{internalDNSTestInstanceName}, internalDNSRecordTypesField: []any{"A", "AAAA"}} + grant.Object[internalDNSSpecField] = map[string]any{internalDNSRegistrationRefField: objectReference(registration), internalDNSPrincipalField: map[string]any{internalDNSSubjectField: internalDNSTestSubject}, internalDNSNameScopesField: []any{internalDNSTestInstanceName}, internalDNSRecordTypesField: []any{"A", "AAAA"}} grant.Object[internalDNSStatusField] = map[string]any{internalDNSConditionsField: []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionActive, internalDNSStatusField: string(metav1.ConditionTrue), internalDNSObservedGenerationField: int64(1)}}} if len(tc.path) > 0 { require.NoError(t, unstructured.SetNestedField(grant.Object, tc.value, tc.path...)) } cl := fake.NewClientBuilder().WithScheme(s).WithObjects(grant).Build() - got, err := findContributionGrant(context.Background(), cl, internalDNSTestNamespace, registration, internalDNSTestSourceAPIUID, internalDNSTestSubject, internalDNSTestInstanceName) + got, err := findContributionGrant(context.Background(), cl, internalDNSTestNamespace, registration, internalDNSTestSubject, internalDNSTestInstanceName) require.NoError(t, err) require.Equal(t, tc.found, got != nil) }) } } -func TestRefreshObservationCannotExtendSourceDeadline(t *testing.T) { - now := time.Date(2026, 10, 9, 20, 0, 0, 0, time.UTC) +func TestObservationPersistsUntilExplicitWithdrawal(t *testing.T) { s := runtime.NewScheme() s.AddKnownTypeWithName(dnsContributionGVK, &unstructured.Unstructured{}) c := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, internalDNSTestInstanceName) c.SetUID("contribution") c.SetGeneration(1) cl := fake.NewClientBuilder().WithScheme(s).WithObjects(c).WithStatusSubresource(c).Build() - r := &InternalDNSPublisherReconciler{Now: func() time.Time { return now }, LeaseDuration: time.Minute} - deadline := now.Add(25 * time.Second) - for _, step := range []time.Duration{0, 10 * time.Second, 26 * time.Second} { - now = time.Date(2026, 10, 9, 20, 0, 0, 0, time.UTC).Add(step) + r := &InternalDNSPublisherReconciler{} + for _, eligible := range []bool{true, true, false} { require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(c), c)) - _, err := r.refreshObservation(context.Background(), cl, c, 3, true, deadline) + _, err := r.refreshObservation(context.Background(), cl, c, 3, eligible) require.NoError(t, err) - eligible, _, _ := unstructured.NestedBool(c.Object, internalDNSStatusField, "eligible") - require.Equal(t, step < 25*time.Second, eligible) - if eligible { - raw, _, _ := unstructured.NestedString(c.Object, internalDNSStatusField, "validUntil") - got, err := time.Parse(time.RFC3339Nano, raw) - require.NoError(t, err) - require.True(t, got.Equal(deadline)) - } + got, _, _ := unstructured.NestedBool(c.Object, internalDNSStatusField, "eligible") + require.Equal(t, eligible, got) + _, found, _ := unstructured.NestedString(c.Object, internalDNSStatusField, "validUntil") + require.False(t, found) } } +func TestUnavailableObservationLeavesPublishedStateUntouched(t *testing.T) { + s := runtime.NewScheme() + cl := fake.NewClientBuilder().WithScheme(s).Build() + r := &InternalDNSPublisherReconciler{} + _, err := r.reconcileAttachment(context.Background(), InternalDNSProjectAccess{Reader: cl, Writer: cl}, InternalDNSProjectIdentity{}, internalDNSTestNamespace, nil, metav1.OwnerReference{}, instanceDNSAttachment{ObservationError: context.DeadlineExceeded}, "record") + require.ErrorIs(t, err, context.DeadlineExceeded) +} + func TestAllocatedInstanceDNSName(t *testing.T) { first := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: "Web_01", UID: "first"}} second := first.DeepCopy() second.UID = "second" - require.Regexp(t, `^web-01-[a-f0-9]{16}$`, allocatedInstanceDNSName(first)) - require.NotEqual(t, allocatedInstanceDNSName(first), allocatedInstanceDNSName(second)) + require.Equal(t, "web-01", allocatedInstanceDNSName(first)) + require.Equal(t, allocatedInstanceDNSName(first), allocatedInstanceDNSName(second)) require.Equal(t, allocatedInstanceDNSName(first), allocatedInstanceDNSName(first.DeepCopy())) } @@ -126,8 +126,8 @@ func TestPublisherWaitsForIssuerAndNeverMutatesGrants(t *testing.T) { instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestInstanceName, Namespace: internalDNSTestNamespace, UID: internalDNSTestInstanceUID}} owner := metav1.OwnerReference{APIVersion: computev1alpha.GroupVersion.String(), Kind: internalDNSInstanceKind, Name: instance.Name, UID: instance.UID, Controller: ptrBool(true)} access := InternalDNSProjectAccess{Reader: guard, Writer: guard} - identity := InternalDNSProjectIdentity{ProjectUID: "project-a", SourceClusterUID: internalDNSTestSourceAPIUID} - attachment := instanceDNSAttachment{VPCUID: internalDNSTestVPCUID, Eligible: true, ValidUntil: now.Add(40 * time.Second), RecordSets: []any{map[string]any{"recordType": "A", "records": []any{map[string]any{"name": "", "ttl": int64(30), "a": map[string]any{"content": "10.0.0.8"}}}}}} + identity := InternalDNSProjectIdentity{ProjectUID: "project-a"} + attachment := instanceDNSAttachment{VPCUID: internalDNSTestVPCUID, Eligible: true, RecordSets: []any{map[string]any{"recordType": "A", "records": []any{map[string]any{"name": "", "ttl": int64(30), "a": map[string]any{"content": "10.0.0.8"}}}}}} ctx := context.Background() for range 2 { ready, err := r.reconcileAttachment(ctx, access, identity, internalDNSTestNamespace, instance, owner, attachment, "record") @@ -139,7 +139,7 @@ func TestPublisherWaitsForIssuerAndNeverMutatesGrants(t *testing.T) { name, _, _ := unstructured.NestedString(registration.Object, internalDNSSpecField, "name") require.Equal(t, allocatedInstanceDNSName(instance), name) grant := newDNSObject(dnsGrantGVK, internalDNSTestNamespace, "issuer-grant") - grant.Object[internalDNSSpecField] = map[string]any{internalDNSRegistrationRefField: objectReference(registration), internalDNSPrincipalField: map[string]any{internalDNSClusterUIDField: internalDNSTestSourceAPIUID, internalDNSSubjectField: internalDNSTestSubject}, internalDNSNameScopesField: []any{name}, internalDNSRecordTypesField: []any{"A", "AAAA"}} + grant.Object[internalDNSSpecField] = map[string]any{internalDNSRegistrationRefField: objectReference(registration), internalDNSPrincipalField: map[string]any{internalDNSSubjectField: internalDNSTestSubject}, internalDNSNameScopesField: []any{name}, internalDNSRecordTypesField: []any{"A", "AAAA"}} grant.Object[internalDNSStatusField] = map[string]any{internalDNSConditionsField: []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionActive, internalDNSStatusField: string(metav1.ConditionTrue), internalDNSObservedGenerationField: int64(1)}}, "activeWriterEpoch": int64(3), "observedGrantGeneration": int64(1), "observedRegistrationGeneration": int64(1)} require.NoError(t, cl.Create(ctx, grant)) ready, err := r.reconcileAttachment(ctx, access, identity, internalDNSTestNamespace, instance, owner, attachment, "record") diff --git a/internal/controller/internaldns_observation.go b/internal/controller/internaldns_observation.go index 41519264..2365cd2e 100644 --- a/internal/controller/internaldns_observation.go +++ b/internal/controller/internaldns_observation.go @@ -5,55 +5,38 @@ package controller import ( "context" "fmt" - "time" computev1alpha "go.datum.net/compute/api/v1alpha" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" - coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) -const dnsNodeObservationLifetime = 45 * time.Second - -// InternalDNSObservedInterface is a live interface observation bounded by node liveness. -// The original deadline is preserved when publishing the contribution. +// InternalDNSObservedInterface reports confirmed interface eligibility. type InternalDNSObservedInterface struct { - Addresses []networkingv1alpha.NetworkInterfaceAddress - Eligible bool - ValidUntil time.Time + Addresses []networkingv1alpha.NetworkInterfaceAddress + Eligible bool } type InternalDNSObserveFunc func(context.Context, client.Reader, string, *computev1alpha.Instance, computev1alpha.InstanceNetworkInterface) (InternalDNSObservedInterface, error) -// InternalDNSObservationSource is an authenticated, uncached edge API reader. -// ClusterUID pins the kube-system namespace UID of the selected edge cluster. +// InternalDNSObservationSource supplies an authenticated edge API reader. type InternalDNSObservationSource struct { - Reader client.Reader - ClusterUID types.UID + Reader client.Reader } -// ObserveInternalDNSInterface validates source identity, interface programming, -// runtime placement, and a current node heartbeat. Projected positive status is -// never sufficient to renew a private address. -func ObserveInternalDNSInterface(ctx context.Context, project client.Reader, namespace string, instance *computev1alpha.Instance, nic computev1alpha.InstanceNetworkInterface, source InternalDNSObservationSource, now time.Time) (InternalDNSObservedInterface, error) { +// ObserveInternalDNSInterface checks interface programming and runtime health. +func ObserveInternalDNSInterface(ctx context.Context, project client.Reader, namespace string, instance *computev1alpha.Instance, nic computev1alpha.InstanceNetworkInterface, source InternalDNSObservationSource) (InternalDNSObservedInterface, error) { var result InternalDNSObservedInterface if nic.Network.Namespace != "" && nic.Network.Namespace != namespace { return result, fmt.Errorf("interface observations require a network in the Instance namespace") } - if source.Reader == nil || source.ClusterUID == "" { + if source.Reader == nil { return result, fmt.Errorf("no authenticated network observation source is configured for location %q", instance.Labels[computev1alpha.LocationLabel]) } - var cluster, projectNS corev1.Namespace - if err := source.Reader.Get(ctx, client.ObjectKey{Name: "kube-system"}, &cluster); err != nil { - return result, err - } - if cluster.UID != source.ClusterUID { - return result, fmt.Errorf("network observation source cluster lifetime does not match") - } + var projectNS corev1.Namespace if err := project.Get(ctx, client.ObjectKey{Name: namespace}, &projectNS); err != nil { return result, err } @@ -86,13 +69,12 @@ func ObserveInternalDNSInterface(ctx context.Context, project client.Reader, nam if err != nil || nodeName == "" { return result, err } - deadline, err := observeNodeDeadline(ctx, source.Reader, nodeName, now) - if err != nil || deadline.IsZero() { + healthy, err := observeNodeHealth(ctx, source.Reader, nodeName) + if err != nil || !healthy { return result, err } result.Addresses = append([]networkingv1alpha.NetworkInterfaceAddress(nil), iface.Spec.Addresses...) result.Eligible = true - result.ValidUntil = deadline return result, nil } @@ -172,33 +154,24 @@ func observeRuntimeNode(ctx context.Context, reader client.Reader, current *comp return nodeName, nil } -func observeNodeDeadline(ctx context.Context, reader client.Reader, nodeName string, now time.Time) (time.Time, error) { +func observeNodeHealth(ctx context.Context, reader client.Reader, nodeName string) (bool, error) { var node corev1.Node if err := reader.Get(ctx, client.ObjectKey{Name: nodeName}, &node); err != nil { - return time.Time{}, err - } - ready := false - for _, c := range node.Status.Conditions { - ready = ready || (c.Type == corev1.NodeReady && c.Status == corev1.ConditionTrue) + return false, err } - if !ready || node.UID == "" || !node.DeletionTimestamp.IsZero() { - return time.Time{}, nil + if node.UID == "" || !node.DeletionTimestamp.IsZero() { + return false, nil } - var lease coordinationv1.Lease - if err := reader.Get(ctx, client.ObjectKey{Namespace: "kube-node-lease", Name: node.Name}, &lease); err != nil { - return time.Time{}, err - } - owned := false - for _, owner := range lease.OwnerReferences { - owned = owned || (owner.Kind == "Node" && owner.UID == node.UID && owner.Name == node.Name) - } - if !owned || lease.Spec.RenewTime == nil || !lease.DeletionTimestamp.IsZero() { - return time.Time{}, nil - } - renewed := lease.Spec.RenewTime.Time - deadline := renewed.Add(dnsNodeObservationLifetime) - if renewed.After(now.Add(5*time.Second)) || !deadline.After(now) { - return time.Time{}, nil + for _, condition := range node.Status.Conditions { + if condition.Type != corev1.NodeReady { + continue + } + switch condition.Status { + case corev1.ConditionTrue: + return true, nil + case corev1.ConditionFalse: + return false, nil + } } - return deadline, nil + return false, fmt.Errorf("node %q health is unknown", nodeName) } diff --git a/internal/controller/internaldns_observation_test.go b/internal/controller/internaldns_observation_test.go index d8b7ca2d..6cc733a6 100644 --- a/internal/controller/internaldns_observation_test.go +++ b/internal/controller/internaldns_observation_test.go @@ -5,12 +5,10 @@ package controller import ( "context" "testing" - "time" "github.com/stretchr/testify/require" computev1alpha "go.datum.net/compute/api/v1alpha" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" - coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -19,50 +17,43 @@ import ( ) func TestObserveInternalDNSInterface(t *testing.T) { - now := time.Date(2026, 10, 9, 20, 0, 0, 0, time.UTC) for _, tc := range []struct { name string - mutate func(*computev1alpha.Instance, *networkingv1alpha.NetworkInterface, *corev1.Pod, *corev1.Node, *coordinationv1.Lease) + mutate func(*computev1alpha.Instance, *networkingv1alpha.NetworkInterface, *corev1.Pod, *corev1.Node) eligible bool wantErr bool networkUID types.UID }{ {name: "live network with unready application", eligible: true}, {name: "replaced network lifetime", networkUID: "replacement-network", wantErr: true}, - {name: "different instance lifetime", mutate: func(i *computev1alpha.Instance, _ *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node, _ *coordinationv1.Lease) { + {name: "different instance lifetime", mutate: func(i *computev1alpha.Instance, _ *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node) { i.UID = "replacement" }, wantErr: true}, - {name: "stale programming generation", mutate: func(_ *computev1alpha.Instance, i *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node, _ *coordinationv1.Lease) { + {name: "stale programming generation", mutate: func(_ *computev1alpha.Instance, i *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node) { i.Generation++ }}, - {name: "different network", mutate: func(_ *computev1alpha.Instance, i *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node, _ *coordinationv1.Lease) { + {name: "different network", mutate: func(_ *computev1alpha.Instance, i *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node) { i.Spec.Network.Name = internalDNSTestForeignValue }}, - {name: "different claim", mutate: func(_ *computev1alpha.Instance, i *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node, _ *coordinationv1.Lease) { + {name: "different claim", mutate: func(_ *computev1alpha.Instance, i *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node) { i.Spec.ClaimRef.Name = internalDNSTestForeignValue }, wantErr: true}, - {name: "runtime exited", mutate: func(_ *computev1alpha.Instance, _ *networkingv1alpha.NetworkInterface, p *corev1.Pod, _ *corev1.Node, _ *coordinationv1.Lease) { + {name: "runtime exited", mutate: func(_ *computev1alpha.Instance, _ *networkingv1alpha.NetworkInterface, p *corev1.Pod, _ *corev1.Node) { p.Status.Phase = corev1.PodFailed }}, - {name: "different runtime slot", mutate: func(_ *computev1alpha.Instance, _ *networkingv1alpha.NetworkInterface, p *corev1.Pod, _ *corev1.Node, _ *coordinationv1.Lease) { + {name: "different runtime slot", mutate: func(_ *computev1alpha.Instance, _ *networkingv1alpha.NetworkInterface, p *corev1.Pod, _ *corev1.Node) { p.Labels[computev1alpha.InstanceIndexLabel] = "different-slot" }}, - {name: "node unavailable", mutate: func(_ *computev1alpha.Instance, _ *networkingv1alpha.NetworkInterface, _ *corev1.Pod, n *corev1.Node, _ *coordinationv1.Lease) { + {name: "node unavailable", mutate: func(_ *computev1alpha.Instance, _ *networkingv1alpha.NetworkInterface, _ *corev1.Pod, n *corev1.Node) { n.Status.Conditions[0].Status = corev1.ConditionFalse }}, - {name: "expired heartbeat", mutate: func(_ *computev1alpha.Instance, _ *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node, l *coordinationv1.Lease) { - l.Spec.RenewTime = &metav1.MicroTime{Time: now.Add(-time.Minute)} - }}, - {name: "future heartbeat", mutate: func(_ *computev1alpha.Instance, _ *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node, l *coordinationv1.Lease) { - l.Spec.RenewTime = &metav1.MicroTime{Time: now.Add(time.Minute)} - }}, - {name: "different node lifetime", mutate: func(_ *computev1alpha.Instance, _ *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node, l *coordinationv1.Lease) { - l.OwnerReferences[0].UID = "other-node" - }}, + {name: "unknown node health", mutate: func(_ *computev1alpha.Instance, _ *networkingv1alpha.NetworkInterface, _ *corev1.Pod, n *corev1.Node) { + n.Status.Conditions[0].Status = corev1.ConditionUnknown + }, wantErr: true}, } { t.Run(tc.name, func(t *testing.T) { scheme := runtime.NewScheme() - for _, add := range []func(*runtime.Scheme) error{corev1.AddToScheme, coordinationv1.AddToScheme, computev1alpha.AddToScheme, networkingv1alpha.AddToScheme} { + for _, add := range []func(*runtime.Scheme) error{corev1.AddToScheme, computev1alpha.AddToScheme, networkingv1alpha.AddToScheme} { require.NoError(t, add(scheme)) } instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestInstanceName, Namespace: internalDNSTestObservationNamespace, UID: "project-instance", Labels: map[string]string{computev1alpha.InstanceSourceUIDLabel: internalDNSTestEdgeUID, computev1alpha.LocationLabel: "dfw"}}, Spec: computev1alpha.InstanceSpec{NetworkInterfaces: []computev1alpha.InstanceNetworkInterface{{Network: networkingv1alpha.NetworkRef{Name: internalDNSTestNetworkName}}}}} @@ -70,11 +61,10 @@ func TestObserveInternalDNSInterface(t *testing.T) { iface := &networkingv1alpha.NetworkInterface{ObjectMeta: metav1.ObjectMeta{Name: "interface", Namespace: edgeInstance.Namespace, UID: "interface-uid", Generation: 2}, Spec: networkingv1alpha.NetworkInterfaceSpec{Network: networkingv1alpha.LocalNetworkRef{Name: internalDNSTestNetworkName}, InterfaceName: defaultInterfaceName, ClaimRef: &networkingv1alpha.NetworkInterfaceClaimRef{Name: networkInterfaceClaimName(internalDNSTestInstanceName, defaultInterfaceName)}, Addresses: []networkingv1alpha.NetworkInterfaceAddress{{Address: "10.20.0.10/32"}}}, Status: networkingv1alpha.NetworkInterfaceStatus{Conditions: []metav1.Condition{{Type: "Allocated", Status: metav1.ConditionTrue, ObservedGeneration: 2}, {Type: "Programmed", Status: metav1.ConditionTrue, ObservedGeneration: 2}}}} pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "runtime", Namespace: edgeInstance.Namespace, OwnerReferences: []metav1.OwnerReference{{APIVersion: computev1alpha.GroupVersion.String(), Kind: internalDNSInstanceKind, Name: edgeInstance.Name, UID: edgeInstance.UID, Controller: ptrBool(true)}}}, Spec: corev1.PodSpec{NodeName: "worker"}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionFalse}}}} node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "worker", UID: "node-uid"}, Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}}} - lease := &coordinationv1.Lease{ObjectMeta: metav1.ObjectMeta{Name: node.Name, Namespace: "kube-node-lease", OwnerReferences: []metav1.OwnerReference{{Kind: "Node", Name: node.Name, UID: node.UID}}}, Spec: coordinationv1.LeaseSpec{RenewTime: &metav1.MicroTime{Time: now.Add(-10 * time.Second)}}} edgeInstance.Labels = map[string]string{computev1alpha.WorkloadDeploymentUIDLabel: "dns-deployment", computev1alpha.InstanceIndexLabel: "0"} pod.Labels = map[string]string{computev1alpha.WorkloadDeploymentUIDLabel: "dns-deployment", computev1alpha.InstanceIndexLabel: "0"} if tc.mutate != nil { - tc.mutate(edgeInstance, iface, pod, node, lease) + tc.mutate(edgeInstance, iface, pod, node) } iface.Status.NetworkContextRef = &networkingv1alpha.LocalNetworkContextRef{Name: "context"} networkUID := types.UID("network-uid") @@ -85,8 +75,8 @@ func TestObserveInternalDNSInterface(t *testing.T) { project := fake.NewClientBuilder().WithScheme(scheme).WithObjects(network, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestObservationNamespace, UID: "project-namespace"}}).Build() claim := &networkingv1alpha.NetworkInterfaceClaim{ObjectMeta: metav1.ObjectMeta{Name: networkInterfaceClaimName(internalDNSTestInstanceName, defaultInterfaceName), Namespace: edgeInstance.Namespace, UID: "claim-uid"}, Spec: networkingv1alpha.NetworkInterfaceClaimSpec{Network: networkingv1alpha.LocalNetworkRef{Name: internalDNSTestNetworkName}}, Status: networkingv1alpha.NetworkInterfaceClaimStatus{NetworkInterfaceRef: &networkingv1alpha.LocalNetworkInterfaceRef{Name: iface.Name}}} networkContext := &networkingv1alpha.NetworkContext{ObjectMeta: metav1.ObjectMeta{Name: "context", Namespace: edgeInstance.Namespace, UID: "network-context", Labels: map[string]string{networkingv1alpha.NetworkUIDLabel: "network-uid"}}, Spec: networkingv1alpha.NetworkContextSpec{Network: networkingv1alpha.LocalNetworkRef{Name: internalDNSTestNetworkName}, Location: networkingv1alpha.LocationReference{Name: "dfw"}}} - source := fake.NewClientBuilder().WithScheme(scheme).WithObjects(edgeInstance, iface, claim, networkContext, pod, node, lease, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "kube-system", UID: "edge-cluster"}}).Build() - got, err := ObserveInternalDNSInterface(context.Background(), project, internalDNSTestObservationNamespace, instance, instance.Spec.NetworkInterfaces[0], InternalDNSObservationSource{Reader: source, ClusterUID: "edge-cluster"}, now) + source := fake.NewClientBuilder().WithScheme(scheme).WithObjects(edgeInstance, iface, claim, networkContext, pod, node).Build() + got, err := ObserveInternalDNSInterface(context.Background(), project, internalDNSTestObservationNamespace, instance, instance.Spec.NetworkInterfaces[0], InternalDNSObservationSource{Reader: source}) if tc.wantErr { require.Error(t, err) } else { @@ -94,7 +84,6 @@ func TestObserveInternalDNSInterface(t *testing.T) { } require.Equal(t, tc.eligible, got.Eligible) if tc.eligible { - require.WithinDuration(t, now.Add(35*time.Second), got.ValidUntil, time.Nanosecond) require.Equal(t, iface.Spec.Addresses, got.Addresses) } }) diff --git a/internal/controller/internaldns_publisher.go b/internal/controller/internaldns_publisher.go index 9db20ce8..b1ef8f7a 100644 --- a/internal/controller/internaldns_publisher.go +++ b/internal/controller/internaldns_publisher.go @@ -47,8 +47,7 @@ const ( internalDNSVPCUID = "internal-dns.compute.datumapis.com/vpc-uid" internalDNSManagedBy = "internal-dns.compute.datumapis.com/managed-by" internalDNSManager = "compute-instance-publisher" - defaultDNSLease = 60 * time.Second - maxDNSLease = 90 * time.Second + internalDNSObservationInterval = 30 * time.Second internalDNSPendingWait = 10 * time.Second internalDNSAPITimeout = 10 * time.Second defaultDNSConcurrency = 4 @@ -58,7 +57,6 @@ const ( internalDNSRegistrationRefField = "registrationRef" internalDNSRecordTypesField = "recordTypes" internalDNSNameScopesField = "nameScopes" - internalDNSClusterUIDField = "clusterUID" internalDNSConditionActive = "Active" internalDNSConditionReady = "Ready" internalDNSConditionPublished = "Published" @@ -77,19 +75,13 @@ var ( dnsContributionGVK = schema.GroupVersionKind{Group: internalDNSGroup, Version: internalDNSVersion, Kind: "DNSRecordContribution"} ) -// InternalDNSProjectIdentity is the trusted identity shared with the DNS -// control plane for one Milo project. ProjectName is the discovery key, while -// the two UIDs are immutable authorization identities. +// InternalDNSProjectIdentity identifies the project that owns DNS resources. type InternalDNSProjectIdentity struct { - ProjectName string - ProjectUID types.UID - SourceClusterUID string + ProjectName string + ProjectUID types.UID } -// InternalDNSProjectAccess supplies an uncached reader and authenticated writer -// for the owning project API. The writer's Kubernetes subject must match the -// configured principal subject; DNS admission verifies both that subject and -// SourceClusterUID. +// InternalDNSProjectAccess supplies the owning project API reader and writer. type InternalDNSProjectAccess struct { Reader client.Reader Writer client.Client @@ -115,7 +107,6 @@ type InternalDNSPublisherReconciler struct { ProjectAccess InternalDNSProjectAccessFunc ProjectIdentities map[string]InternalDNSProjectIdentity PrincipalSubject string - LeaseDuration time.Duration MaxConcurrentReconciles int Now func() time.Time ObservationSources map[string]InternalDNSObservationSource @@ -132,9 +123,6 @@ func (r *InternalDNSPublisherReconciler) Reconcile(ctx context.Context, req mcre } var instance computev1alpha.Instance - // Lease renewal must come from a live API read. A cached projection can - // survive a disconnected source and would incorrectly keep old addresses - // eligible forever. if err := providerCluster.GetAPIReader().Get(ctx, req.NamespacedName, &instance); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } @@ -148,7 +136,7 @@ func (r *InternalDNSPublisherReconciler) Reconcile(ctx context.Context, req mcre return ctrl.Result{}, fmt.Errorf("resolve DNS project namespace: %w", err) } identity, ok := r.ProjectIdentities[projectID] - if !ok || identity.ProjectUID == "" || identity.SourceClusterUID == "" { + if !ok || identity.ProjectUID == "" { return ctrl.Result{}, fmt.Errorf("internal DNS identity is not configured for project %q", projectID) } access, err := r.projectAccess(ctx, projectID, providerCluster) @@ -212,7 +200,7 @@ func (r *InternalDNSPublisherReconciler) Reconcile(ctx context.Context, req mcre if pending { return ctrl.Result{RequeueAfter: internalDNSPendingWait}, publicationErr } - return ctrl.Result{RequeueAfter: r.leaseDuration() / 3}, publicationErr + return ctrl.Result{RequeueAfter: internalDNSObservationInterval}, publicationErr } type instanceDNSAttachment struct { @@ -220,7 +208,6 @@ type instanceDNSAttachment struct { Network networkingv1alpha.NetworkRef RecordSets []any Eligible bool - ValidUntil time.Time ObservationError error } @@ -257,9 +244,6 @@ func instanceDNSAttachments(ctx context.Context, reader client.Reader, projectNa continue } attachment.Eligible = true - if attachment.ValidUntil.IsZero() || observation.ValidUntil.Before(attachment.ValidUntil) { - attachment.ValidUntil = observation.ValidUntil - } for _, address := range observation.Addresses { addr, ok := hostAddress(address.Address) if !ok { @@ -307,6 +291,9 @@ func hostAddress(raw string) (netip.Addr, bool) { } func (r *InternalDNSPublisherReconciler) reconcileAttachment(ctx context.Context, access InternalDNSProjectAccess, identity InternalDNSProjectIdentity, namespace string, instance *computev1alpha.Instance, owner metav1.OwnerReference, attachment instanceDNSAttachment, resourceName string) (bool, error) { + if attachment.ObservationError != nil { + return false, attachment.ObservationError + } managed, err := findResolverContext(ctx, access.Reader, namespace, identity.ProjectUID, attachment.VPCUID) if err != nil { return false, err @@ -346,7 +333,7 @@ func (r *InternalDNSPublisherReconciler) reconcileAttachment(ctx context.Context } registrationRef := objectReference(registration) - grant, err := findContributionGrant(ctx, access.Reader, namespace, registration, identity.SourceClusterUID, r.PrincipalSubject, ownerName) + grant, err := findContributionGrant(ctx, access.Reader, namespace, registration, r.PrincipalSubject, ownerName) if err != nil { return false, err } @@ -374,30 +361,15 @@ func (r *InternalDNSPublisherReconciler) reconcileAttachment(ctx context.Context return false, nil } - return r.refreshObservation(ctx, access.Writer, contribution, writerEpoch, attachment.Eligible, attachment.ValidUntil) + return r.refreshObservation(ctx, access.Writer, contribution, writerEpoch, attachment.Eligible) } -func (r *InternalDNSPublisherReconciler) refreshObservation(ctx context.Context, writer client.Client, contribution *unstructured.Unstructured, writerEpoch int64, eligible bool, deadline time.Time) (bool, error) { - now := time.Now() - if r.Now != nil { - now = r.Now() - } - validThrough := now.Add(r.leaseDuration()) - if eligible { - if deadline.IsZero() || !deadline.After(now) { - eligible = false - } else if deadline.Before(validThrough) { - validThrough = deadline - } - } +func (r *InternalDNSPublisherReconciler) refreshObservation(ctx context.Context, writer client.Client, contribution *unstructured.Unstructured, writerEpoch int64, eligible bool) (bool, error) { oldEligible, _, _ := unstructured.NestedBool(contribution.Object, internalDNSStatusField, "eligible") observedGeneration, _, _ := unstructured.NestedInt64(contribution.Object, internalDNSStatusField, internalDNSObservedGenerationField) - validUntilRaw, _, _ := unstructured.NestedString(contribution.Object, internalDNSStatusField, "validUntil") - validUntil, _ := time.Parse(time.RFC3339Nano, validUntilRaw) - if oldEligible == eligible && observedGeneration == contribution.GetGeneration() && validUntil.After(now.Add(r.leaseDuration()/3)) && !validUntil.After(validThrough) { + if oldEligible == eligible && observedGeneration == contribution.GetGeneration() { return true, nil } - base := contribution.DeepCopy() sequence, _, _ := unstructured.NestedInt64(contribution.Object, internalDNSStatusField, "sequence") owned := []struct { @@ -409,7 +381,6 @@ func (r *InternalDNSPublisherReconciler) refreshObservation(ctx context.Context, {[]string{internalDNSStatusField, "sequence"}, sequence + 1}, {[]string{internalDNSStatusField, "eligible"}, eligible}, {[]string{internalDNSStatusField, internalDNSReasonField}, map[bool]string{true: "AddressAllocatedAndProgrammed", false: "AddressUnavailable"}[eligible]}, - {[]string{internalDNSStatusField, "validUntil"}, validThrough.UTC().Format(time.RFC3339Nano)}, } for _, field := range owned { if err := unstructured.SetNestedField(contribution.Object, field.value, field.path...); err != nil { @@ -470,7 +441,7 @@ func unstructuredConditionCurrent(obj *unstructured.Unstructured, conditionType return false } -func findContributionGrant(ctx context.Context, reader client.Reader, namespace string, registration *unstructured.Unstructured, clusterUID, subject, ownerName string) (*unstructured.Unstructured, error) { +func findContributionGrant(ctx context.Context, reader client.Reader, namespace string, registration *unstructured.Unstructured, subject, ownerName string) (*unstructured.Unstructured, error) { list := &unstructured.UnstructuredList{} list.SetGroupVersionKind(dnsGrantGVK.GroupVersion().WithKind(dnsGrantGVK.Kind + "List")) if err := reader.List(ctx, list, client.InNamespace(namespace)); err != nil { @@ -484,7 +455,7 @@ func findContributionGrant(ctx context.Context, reader client.Reader, namespace } ref, _, _ := unstructured.NestedMap(grant.Object, internalDNSSpecField, internalDNSRegistrationRefField) principal, _, _ := unstructured.NestedMap(grant.Object, internalDNSSpecField, internalDNSPrincipalField) - if !reflect.DeepEqual(ref, objectReference(registration)) || stringValue(principal, internalDNSClusterUIDField) != clusterUID || stringValue(principal, internalDNSSubjectField) != subject { + if !reflect.DeepEqual(ref, objectReference(registration)) || stringValue(principal, internalDNSSubjectField) != subject { continue } scopes, _, _ := unstructured.NestedStringSlice(grant.Object, internalDNSSpecField, internalDNSNameScopesField) @@ -622,10 +593,10 @@ func allocatedInstanceDNSName(instance *computev1alpha.Instance) string { if base == "" { base = "instance" } - if len(base) > 40 { - base = strings.TrimRight(base[:40], "-") + if len(base) > 63 { + return strings.TrimRight(base[:46], "-") + "-" + shortHash(instance.Name) } - return base + "-" + shortHash(string(instance.UID)) + return base } func internalDNSResourceName(instanceUID, vpcUID types.UID) string { @@ -658,16 +629,6 @@ func (r *InternalDNSPublisherReconciler) projectAccess(ctx context.Context, proj return InternalDNSProjectAccess{Reader: providerCluster.GetAPIReader(), Writer: providerCluster.GetClient()}, nil } -func (r *InternalDNSPublisherReconciler) leaseDuration() time.Duration { - if r.LeaseDuration <= 0 { - return defaultDNSLease - } - if r.LeaseDuration > maxDNSLease { - return maxDNSLease - } - return r.LeaseDuration -} - func (r *InternalDNSPublisherReconciler) SetupWithManager(mgr mcmanager.Manager) error { if strings.TrimSpace(r.PrincipalSubject) == "" { return errors.New("internal DNS principal subject is required") @@ -675,9 +636,6 @@ func (r *InternalDNSPublisherReconciler) SetupWithManager(mgr mcmanager.Manager) if len(r.ProjectIdentities) == 0 { return errors.New("at least one internal DNS project identity is required") } - if r.LeaseDuration > maxDNSLease { - return fmt.Errorf("internal DNS lease duration %s exceeds the platform maximum %s", r.LeaseDuration, maxDNSLease) - } r.mgr = mgr concurrency := r.MaxConcurrentReconciles if concurrency <= 0 { diff --git a/internal/controller/internaldns_publisher_test.go b/internal/controller/internaldns_publisher_test.go index 522ad9aa..7a300db3 100644 --- a/internal/controller/internaldns_publisher_test.go +++ b/internal/controller/internaldns_publisher_test.go @@ -62,7 +62,7 @@ func TestInstanceDNSAttachmentsMapsTenancyAndAddressFamilies(t *testing.T) { eligible := true observation := func(context.Context, client.Reader, string, *computev1alpha.Instance, computev1alpha.InstanceNetworkInterface) (InternalDNSObservedInterface, error) { - return InternalDNSObservedInterface{Addresses: []networkingv1alpha.NetworkInterfaceAddress{{Address: "10.0.0.8/32"}, {Address: "2001:db8::8/128"}, {Address: "2001:db8:1::/96"}}, Eligible: eligible, ValidUntil: time.Now().Add(time.Minute)}, nil + return InternalDNSObservedInterface{Addresses: []networkingv1alpha.NetworkInterfaceAddress{{Address: "10.0.0.8/32"}, {Address: "2001:db8::8/128"}, {Address: "2001:db8:1::/96"}}, Eligible: eligible}, nil } attachments, err := instanceDNSAttachments(context.Background(), cl, internalDNSTestNamespace, instance, observation) require.NoError(t, err) @@ -156,12 +156,12 @@ func TestRefreshObservationPreservesPlatformStatusAndIncrementsFence(t *testing. } cl := fake.NewClientBuilder().WithScheme(s).WithObjects(contribution).WithStatusSubresource(contribution).Build() now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) - r := &InternalDNSPublisherReconciler{LeaseDuration: 60 * time.Second, Now: func() time.Time { return now }} + r := &InternalDNSPublisherReconciler{Now: func() time.Time { return now }} var live unstructured.Unstructured live.SetGroupVersionKind(dnsContributionGVK) require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(contribution), &live)) - ready, err := r.refreshObservation(context.Background(), cl, &live, 7, true, now.Add(40*time.Second)) + ready, err := r.refreshObservation(context.Background(), cl, &live, 7, true) require.NoError(t, err) assert.True(t, ready) diff --git a/internal/controller/internaldns_status.go b/internal/controller/internaldns_status.go index 6b4762e3..7518fc21 100644 --- a/internal/controller/internaldns_status.go +++ b/internal/controller/internaldns_status.go @@ -28,7 +28,7 @@ func (r *InternalDNSPublisherReconciler) observeInterface(ctx context.Context, r if r.ObserveInterface != nil { return r.ObserveInterface(ctx, reader, namespace, instance, nic) } - return ObserveInternalDNSInterface(ctx, reader, namespace, instance, nic, r.ObservationSources[instance.Labels[computev1alpha.LocationLabel]], r.now()) + return ObserveInternalDNSInterface(ctx, reader, namespace, instance, nic, r.ObservationSources[instance.Labels[computev1alpha.LocationLabel]]) } func (r *InternalDNSPublisherReconciler) publicationStatus(ctx context.Context, reader client.Reader, identity InternalDNSProjectIdentity, namespace string, instance *computev1alpha.Instance, attachment instanceDNSAttachment, resourceName string, authorized bool, reconcileErr error) (computev1alpha.InstanceDNSStatus, error) { @@ -58,13 +58,12 @@ func (r *InternalDNSPublisherReconciler) publicationStatus(ctx context.Context, if err := reader.Get(ctx, client.ObjectKeyFromObject(contribution), contribution); err != nil && !apierrors.IsNotFound(err) { return result, err } - deadlineRaw, _, _ := unstructured.NestedString(contribution.Object, internalDNSStatusField, "validUntil") - deadline, _ := time.Parse(time.RFC3339Nano, deadlineRaw) - if authorized && len(result.Hostnames) > 0 && unstructuredConditionCurrent(registration, internalDNSConditionPublished) && unstructuredConditionCurrent(contribution, internalDNSConditionPublished) && deadline.After(r.now()) && attachment.Eligible && attachment.ObservationError == nil { + if authorized && len(result.Hostnames) > 0 && unstructuredConditionCurrent(registration, internalDNSConditionPublished) && unstructuredConditionCurrent(contribution, internalDNSConditionPublished) && attachment.Eligible && attachment.ObservationError == nil { condition.Status, condition.Reason, condition.Message = metav1.ConditionTrue, internalDNSConditionPublished, "Private name is published" } } if attachment.ObservationError != nil { + condition.Status = metav1.ConditionUnknown condition.Reason, condition.Message = "NetworkObservationUnavailable", "No current network observation is available" } else if !attachment.Eligible { condition.Reason, condition.Message = "AddressUnavailable", "No currently eligible private host address is available" diff --git a/internal/controller/internaldns_status_test.go b/internal/controller/internaldns_status_test.go index 2699e18d..eee7ca0b 100644 --- a/internal/controller/internaldns_status_test.go +++ b/internal/controller/internaldns_status_test.go @@ -23,15 +23,13 @@ func TestPublicationStatusRequiresCurrentUnexpiredPublication(t *testing.T) { name string domain string foreignZone bool - deadline time.Time current, authorized, wantReady bool }{ - {name: "published", deadline: now.Add(time.Minute), current: true, authorized: true, wantReady: true}, - {name: "custom domain", domain: "private.example.net", deadline: now.Add(time.Minute), current: true, authorized: true, wantReady: true}, - {name: "different zone lifetime", foreignZone: true, deadline: now.Add(time.Minute), current: true, authorized: true}, - {name: "expired", deadline: now.Add(-time.Second), current: true, authorized: true}, - {name: "stale generation", deadline: now.Add(time.Minute), authorized: true}, - {name: "revoked grant", deadline: now.Add(time.Minute), current: true}, + {name: "published", current: true, authorized: true, wantReady: true}, + {name: "custom domain", domain: "private.example.net", current: true, authorized: true, wantReady: true}, + {name: "different zone lifetime", foreignZone: true, current: true, authorized: true}, + {name: "stale generation", authorized: true}, + {name: "revoked grant", current: true}, } { t.Run(tc.name, func(t *testing.T) { if tc.domain == "" { @@ -59,7 +57,7 @@ func TestPublicationStatusRequiresCurrentUnexpiredPublication(t *testing.T) { zoneUID = internalDNSTestForeignValue } registration.Object[internalDNSSpecField] = map[string]any{internalDNSZoneRefField: map[string]any{internalDNSNameField: "zone", internalDNSUIDField: zoneUID}} - contribution.Object[internalDNSStatusField] = map[string]any{internalDNSConditionsField: conditions, "validUntil": tc.deadline.Format(time.RFC3339Nano)} + contribution.Object[internalDNSStatusField] = map[string]any{internalDNSConditionsField: conditions} cl := fake.NewClientBuilder().WithScheme(s).WithObjects(resolver, registration, contribution).Build() r := &InternalDNSPublisherReconciler{Now: func() time.Time { return now }} got, err := r.publicationStatus(context.Background(), cl, InternalDNSProjectIdentity{ProjectUID: "project-a"}, internalDNSTestNamespace, instance, instanceDNSAttachment{VPCUID: internalDNSTestVPCUID, Eligible: true}, internalDNSTestInstanceName, tc.authorized, nil) diff --git a/test/internaldns/README.md b/test/internaldns/README.md index 41a629b5..33582b2b 100644 --- a/test/internaldns/README.md +++ b/test/internaldns/README.md @@ -14,7 +14,7 @@ go test ./internal/controller ./internal/config ./cmd/... ``` The tests cover context identity, flat instance names, independently issued -grants, fenced contribution renewal, live edge observations, source lifetime +grants, fenced eligibility updates, live edge observations, source lifetime replacement, and project DNS status ownership. They use fixtures and do not qualify guest resolution. @@ -25,22 +25,21 @@ go build -o /tmp/compute-dns-publisher ./test/internaldns/publisher /tmp/compute-dns-publisher --feature-gates=InternalDNSPublishing=true \ --kubeconfig /tmp/project-publisher.kubeconfig \ --namespace production --project-uid PROJECT_UID \ - --source-uid PROJECT_API_UID \ --subject system:serviceaccount:compute-system:compute-manager \ --edge-kubeconfig /tmp/edge-observer.kubeconfig \ - --edge-uid EDGE_KUBE_SYSTEM_UID --location dfw + --location dfw ``` Provision a resolver context with a `datum.internal` managed zone, the scoped project publisher role, and a separate trusted grant issuer. Instances must carry their source lifetime and location labels. Edge namespaces use the project namespace UID. The observer requires allocated, programmed interfaces, live -claims, Instance-owned runtime Pods, and current node heartbeat leases. +claims, Instance-owned runtime Pods, and confirmed node health. The disabled runner starts without creating Kubernetes clients. Release qualification must exercise two VPCs using the same default domain, real address allocation, guest resolver application, and UDP/TCP queries through Galactic. Include address changes, withdrawal, publisher outages, grant -revocation, stale replay, source replacement, and controller restarts. Keep this -qualification local and reuse the shared Kubernetes foundation. The publisher +revocation, stale replay, source replacement, and controller restarts. Run this +qualification in CI and reuse the shared Kubernetes foundation. The publisher contract tests and runner do not establish that those release criteria pass. diff --git a/test/internaldns/publisher/main.go b/test/internaldns/publisher/main.go index 5afb8934..0ec80dd9 100644 --- a/test/internaldns/publisher/main.go +++ b/test/internaldns/publisher/main.go @@ -32,19 +32,15 @@ import ( ) func main() { - var kubeconfig, namespace, projectUID, sourceUID, subject, gates string - var edgeKubeconfig, edgeUID, location string - var lease time.Duration + var kubeconfig, namespace, projectUID, subject, gates string + var edgeKubeconfig, location string flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError) flags.StringVar(&kubeconfig, "kubeconfig", "", "Disposable project kubeconfig") flags.StringVar(&namespace, "namespace", "project-e2e", "Project namespace") flags.StringVar(&projectUID, "project-uid", "project-e2e-uid", "DNS project UID") - flags.StringVar(&sourceUID, "source-uid", "internal-dns-e2e-cluster", "DNS trusted source UID") flags.StringVar(&subject, "subject", "system:serviceaccount:compute-system:dns-publisher", "Authenticated writer") flags.StringVar(&gates, "feature-gates", "", "Compute feature gates") - flags.DurationVar(&lease, "lease", 60*time.Second, "Contribution lease") flags.StringVar(&edgeKubeconfig, "edge-kubeconfig", "", "Read-only edge observation kubeconfig") - flags.StringVar(&edgeUID, "edge-uid", "", "Edge kube-system namespace UID") flags.StringVar(&location, "location", "", "Instance location label") if err := flags.Parse(os.Args[1:]); err != nil { panic(err) @@ -57,8 +53,8 @@ func main() { fmt.Println("InternalDNSPublishing is disabled; no DNS client or controller started") return } - if edgeKubeconfig == "" || edgeUID == "" || location == "" { - panic("edge-kubeconfig, edge-uid, and location are required") + if edgeKubeconfig == "" || location == "" { + panic("edge-kubeconfig and location are required") } cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig) if err != nil { @@ -95,10 +91,10 @@ func main() { panic(err) } publisher := &controller.InternalDNSPublisherReconciler{ - ObservationSources: map[string]controller.InternalDNSObservationSource{location: {Reader: edge, ClusterUID: types.UID(edgeUID)}}, + ObservationSources: map[string]controller.InternalDNSObservationSource{location: {Reader: edge}}, ProjectIdentities: map[string]controller.InternalDNSProjectIdentity{"single": { - ProjectName: "single", ProjectUID: types.UID(projectUID), SourceClusterUID: sourceUID, - }}, PrincipalSubject: subject, LeaseDuration: lease, + ProjectName: "single", ProjectUID: types.UID(projectUID), + }}, PrincipalSubject: subject, } if err := publisher.SetupWithManager(mgr); err != nil { panic(err) From 164eef01cd6e66a3bb06735c10295844f2083d80 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 18:32:21 -0500 Subject: [PATCH 10/33] chore: Generate API schemas with pinned tools --- .../bases/compute.datumapis.com_instanceconsolesessions.yaml | 2 +- config/base/crd/bases/compute.datumapis.com_instances.yaml | 2 +- config/base/crd/bases/compute.datumapis.com_instancetypes.yaml | 2 +- config/base/crd/bases/compute.datumapis.com_runtimeclasses.yaml | 2 +- .../crd/bases/compute.datumapis.com_workloaddeployments.yaml | 2 +- config/base/crd/bases/compute.datumapis.com_workloads.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config/base/crd/bases/compute.datumapis.com_instanceconsolesessions.yaml b/config/base/crd/bases/compute.datumapis.com_instanceconsolesessions.yaml index 0cc7ace1..1f0347b8 100644 --- a/config/base/crd/bases/compute.datumapis.com_instanceconsolesessions.yaml +++ b/config/base/crd/bases/compute.datumapis.com_instanceconsolesessions.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.0 + controller-gen.kubebuilder.io/version: v0.16.4 discovery.miloapis.com/parent-contexts: Project name: instanceconsolesessions.compute.datumapis.com spec: diff --git a/config/base/crd/bases/compute.datumapis.com_instances.yaml b/config/base/crd/bases/compute.datumapis.com_instances.yaml index 7fe00402..4aaa89ed 100644 --- a/config/base/crd/bases/compute.datumapis.com_instances.yaml +++ b/config/base/crd/bases/compute.datumapis.com_instances.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.0 + controller-gen.kubebuilder.io/version: v0.16.4 discovery.miloapis.com/parent-contexts: Project name: instances.compute.datumapis.com spec: diff --git a/config/base/crd/bases/compute.datumapis.com_instancetypes.yaml b/config/base/crd/bases/compute.datumapis.com_instancetypes.yaml index 235c4dff..ccf11e3b 100644 --- a/config/base/crd/bases/compute.datumapis.com_instancetypes.yaml +++ b/config/base/crd/bases/compute.datumapis.com_instancetypes.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.0 + controller-gen.kubebuilder.io/version: v0.16.4 discovery.miloapis.com/parent-contexts: Platform,Project name: instancetypes.compute.datumapis.com spec: diff --git a/config/base/crd/bases/compute.datumapis.com_runtimeclasses.yaml b/config/base/crd/bases/compute.datumapis.com_runtimeclasses.yaml index a1e8d12e..0002298a 100644 --- a/config/base/crd/bases/compute.datumapis.com_runtimeclasses.yaml +++ b/config/base/crd/bases/compute.datumapis.com_runtimeclasses.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.0 + controller-gen.kubebuilder.io/version: v0.16.4 discovery.miloapis.com/parent-contexts: Platform,Project name: runtimeclasses.compute.datumapis.com spec: diff --git a/config/base/crd/bases/compute.datumapis.com_workloaddeployments.yaml b/config/base/crd/bases/compute.datumapis.com_workloaddeployments.yaml index 488bdfcb..6b74d7fb 100644 --- a/config/base/crd/bases/compute.datumapis.com_workloaddeployments.yaml +++ b/config/base/crd/bases/compute.datumapis.com_workloaddeployments.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.0 + controller-gen.kubebuilder.io/version: v0.16.4 discovery.miloapis.com/parent-contexts: Project name: workloaddeployments.compute.datumapis.com spec: diff --git a/config/base/crd/bases/compute.datumapis.com_workloads.yaml b/config/base/crd/bases/compute.datumapis.com_workloads.yaml index a169e646..325a8306 100644 --- a/config/base/crd/bases/compute.datumapis.com_workloads.yaml +++ b/config/base/crd/bases/compute.datumapis.com_workloads.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.0 + controller-gen.kubebuilder.io/version: v0.16.4 discovery.miloapis.com/parent-contexts: Project name: workloads.compute.datumapis.com spec: From 46784c6721c72fd76cae6f119a9a96b57532e30b Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 18:36:09 -0500 Subject: [PATCH 11/33] test: Align DNS observation contract checks --- cmd/main_test.go | 1 - internal/controller/internaldns_observation_test.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/main_test.go b/cmd/main_test.go index 75f64ad3..16e8edf0 100644 --- a/cmd/main_test.go +++ b/cmd/main_test.go @@ -7,7 +7,6 @@ import ( "path/filepath" "slices" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/internal/controller/internaldns_observation_test.go b/internal/controller/internaldns_observation_test.go index 6cc733a6..caba5a28 100644 --- a/internal/controller/internaldns_observation_test.go +++ b/internal/controller/internaldns_observation_test.go @@ -91,6 +91,6 @@ func TestObserveInternalDNSInterface(t *testing.T) { } func TestObserveInternalDNSInterfaceRejectsMissingSource(t *testing.T) { - _, err := ObserveInternalDNSInterface(context.Background(), nil, internalDNSTestObservationNamespace, &computev1alpha.Instance{}, computev1alpha.InstanceNetworkInterface{}, InternalDNSObservationSource{}, time.Now()) + _, err := ObserveInternalDNSInterface(context.Background(), nil, internalDNSTestObservationNamespace, &computev1alpha.Instance{}, computev1alpha.InstanceNetworkInterface{}, InternalDNSObservationSource{}) require.Error(t, err) } From b4b56592ec55bcaf2cbd6a91ce011579c9dfe887 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 18:43:41 -0500 Subject: [PATCH 12/33] test: Remove unused DNS observation import --- cmd/main_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/main_test.go b/cmd/main_test.go index 16e8edf0..af63e43e 100644 --- a/cmd/main_test.go +++ b/cmd/main_test.go @@ -11,7 +11,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" rbacv1 "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/yaml" From 071d9b9179527b9fbb8af3f853e0052216786e60 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 18:49:55 -0500 Subject: [PATCH 13/33] refactor: Remove unused DNS expiry clock --- internal/controller/internaldns_status.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/internal/controller/internaldns_status.go b/internal/controller/internaldns_status.go index 7518fc21..8dcfc51f 100644 --- a/internal/controller/internaldns_status.go +++ b/internal/controller/internaldns_status.go @@ -7,7 +7,6 @@ import ( "fmt" "reflect" "strings" - "time" computev1alpha "go.datum.net/compute/api/v1alpha" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -17,13 +16,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -func (r *InternalDNSPublisherReconciler) now() time.Time { - if r.Now != nil { - return r.Now() - } - return time.Now() -} - func (r *InternalDNSPublisherReconciler) observeInterface(ctx context.Context, reader client.Reader, namespace string, instance *computev1alpha.Instance, nic computev1alpha.InstanceNetworkInterface) (InternalDNSObservedInterface, error) { if r.ObserveInterface != nil { return r.ObserveInterface(ctx, reader, namespace, instance, nic) From 8e66037b745a6437d62b38cd96c0e3c303d360ab Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 18:57:04 -0500 Subject: [PATCH 14/33] test: Preserve published endpoints during observation failures --- .../controller/internaldns_authorization_test.go | 13 +++++++++++-- internal/controller/internaldns_status.go | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/internal/controller/internaldns_authorization_test.go b/internal/controller/internaldns_authorization_test.go index a2449852..36bcf129 100644 --- a/internal/controller/internaldns_authorization_test.go +++ b/internal/controller/internaldns_authorization_test.go @@ -79,10 +79,19 @@ func TestObservationPersistsUntilExplicitWithdrawal(t *testing.T) { func TestUnavailableObservationLeavesPublishedStateUntouched(t *testing.T) { s := runtime.NewScheme() - cl := fake.NewClientBuilder().WithScheme(s).Build() + s.AddKnownTypeWithName(dnsContributionGVK, &unstructured.Unstructured{}) + published := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, "record") + published.SetUID("contribution") + published.Object[internalDNSSpecField] = map[string]any{"recordSets": []any{map[string]any{"recordType": "A", "records": []any{map[string]any{"name": "api", "a": map[string]any{"content": "10.0.0.8"}}}}}} + published.Object[internalDNSStatusField] = map[string]any{"writerEpoch": int64(3), "sequence": int64(9), "eligible": true} + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(published).WithStatusSubresource(published).Build() r := &InternalDNSPublisherReconciler{} - _, err := r.reconcileAttachment(context.Background(), InternalDNSProjectAccess{Reader: cl, Writer: cl}, InternalDNSProjectIdentity{}, internalDNSTestNamespace, nil, metav1.OwnerReference{}, instanceDNSAttachment{ObservationError: context.DeadlineExceeded}, "record") + instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: "api", UID: internalDNSTestInstanceUID}} + _, err := r.reconcileAttachment(context.Background(), InternalDNSProjectAccess{Reader: cl, Writer: cl}, InternalDNSProjectIdentity{}, internalDNSTestNamespace, instance, metav1.OwnerReference{}, instanceDNSAttachment{ObservationError: context.DeadlineExceeded}, "record") require.ErrorIs(t, err, context.DeadlineExceeded) + current := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, "record") + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(published), current)) + require.Equal(t, published.Object, current.Object) } func TestAllocatedInstanceDNSName(t *testing.T) { diff --git a/internal/controller/internaldns_status.go b/internal/controller/internaldns_status.go index 8dcfc51f..b04253d8 100644 --- a/internal/controller/internaldns_status.go +++ b/internal/controller/internaldns_status.go @@ -60,7 +60,7 @@ func (r *InternalDNSPublisherReconciler) publicationStatus(ctx context.Context, } else if !attachment.Eligible { condition.Reason, condition.Message = "AddressUnavailable", "No currently eligible private host address is available" } - if reconcileErr != nil { + if reconcileErr != nil && attachment.ObservationError == nil { condition.Reason, condition.Message = "PublicationFailed", "DNS publication could not be reconciled" } apimeta.SetStatusCondition(&result.Conditions, condition) From 4a5697c44c8ba4e5701874b55fdf8f5b9a18645f Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 19:02:00 -0500 Subject: [PATCH 15/33] refactor: Share DNS observation field names --- .../controller/internaldns_authorization_test.go | 10 +++++----- internal/controller/internaldns_publisher.go | 16 ++++++++++------ .../controller/internaldns_publisher_test.go | 6 +++--- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/internal/controller/internaldns_authorization_test.go b/internal/controller/internaldns_authorization_test.go index 36bcf129..9b3ce01a 100644 --- a/internal/controller/internaldns_authorization_test.go +++ b/internal/controller/internaldns_authorization_test.go @@ -70,7 +70,7 @@ func TestObservationPersistsUntilExplicitWithdrawal(t *testing.T) { require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(c), c)) _, err := r.refreshObservation(context.Background(), cl, c, 3, eligible) require.NoError(t, err) - got, _, _ := unstructured.NestedBool(c.Object, internalDNSStatusField, "eligible") + got, _, _ := unstructured.NestedBool(c.Object, internalDNSStatusField, internalDNSEligibleField) require.Equal(t, eligible, got) _, found, _ := unstructured.NestedString(c.Object, internalDNSStatusField, "validUntil") require.False(t, found) @@ -82,8 +82,8 @@ func TestUnavailableObservationLeavesPublishedStateUntouched(t *testing.T) { s.AddKnownTypeWithName(dnsContributionGVK, &unstructured.Unstructured{}) published := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, "record") published.SetUID("contribution") - published.Object[internalDNSSpecField] = map[string]any{"recordSets": []any{map[string]any{"recordType": "A", "records": []any{map[string]any{"name": "api", "a": map[string]any{"content": "10.0.0.8"}}}}}} - published.Object[internalDNSStatusField] = map[string]any{"writerEpoch": int64(3), "sequence": int64(9), "eligible": true} + published.Object[internalDNSSpecField] = map[string]any{"recordSets": []any{map[string]any{internalDNSRecordTypeField: "A", "records": []any{map[string]any{internalDNSNameField: "api", "a": map[string]any{internalDNSContentField: "10.0.0.8"}}}}}} + published.Object[internalDNSStatusField] = map[string]any{"writerEpoch": int64(3), internalDNSSequenceField: int64(9), internalDNSEligibleField: true} cl := fake.NewClientBuilder().WithScheme(s).WithObjects(published).WithStatusSubresource(published).Build() r := &InternalDNSPublisherReconciler{} instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: "api", UID: internalDNSTestInstanceUID}} @@ -136,7 +136,7 @@ func TestPublisherWaitsForIssuerAndNeverMutatesGrants(t *testing.T) { owner := metav1.OwnerReference{APIVersion: computev1alpha.GroupVersion.String(), Kind: internalDNSInstanceKind, Name: instance.Name, UID: instance.UID, Controller: ptrBool(true)} access := InternalDNSProjectAccess{Reader: guard, Writer: guard} identity := InternalDNSProjectIdentity{ProjectUID: "project-a"} - attachment := instanceDNSAttachment{VPCUID: internalDNSTestVPCUID, Eligible: true, RecordSets: []any{map[string]any{"recordType": "A", "records": []any{map[string]any{"name": "", "ttl": int64(30), "a": map[string]any{"content": "10.0.0.8"}}}}}} + attachment := instanceDNSAttachment{VPCUID: internalDNSTestVPCUID, Eligible: true, RecordSets: []any{map[string]any{internalDNSRecordTypeField: "A", "records": []any{map[string]any{internalDNSNameField: "", "ttl": int64(30), "a": map[string]any{internalDNSContentField: "10.0.0.8"}}}}}} ctx := context.Background() for range 2 { ready, err := r.reconcileAttachment(ctx, access, identity, internalDNSTestNamespace, instance, owner, attachment, "record") @@ -145,7 +145,7 @@ func TestPublisherWaitsForIssuerAndNeverMutatesGrants(t *testing.T) { } registration := newDNSObject(dnsRegistrationGVK, internalDNSTestNamespace, "record") require.NoError(t, cl.Get(ctx, client.ObjectKeyFromObject(registration), registration)) - name, _, _ := unstructured.NestedString(registration.Object, internalDNSSpecField, "name") + name, _, _ := unstructured.NestedString(registration.Object, internalDNSSpecField, internalDNSNameField) require.Equal(t, allocatedInstanceDNSName(instance), name) grant := newDNSObject(dnsGrantGVK, internalDNSTestNamespace, "issuer-grant") grant.Object[internalDNSSpecField] = map[string]any{internalDNSRegistrationRefField: objectReference(registration), internalDNSPrincipalField: map[string]any{internalDNSSubjectField: internalDNSTestSubject}, internalDNSNameScopesField: []any{name}, internalDNSRecordTypesField: []any{"A", "AAAA"}} diff --git a/internal/controller/internaldns_publisher.go b/internal/controller/internaldns_publisher.go index b1ef8f7a..76d3f60e 100644 --- a/internal/controller/internaldns_publisher.go +++ b/internal/controller/internaldns_publisher.go @@ -34,6 +34,10 @@ import ( ) const ( + internalDNSSequenceField = "sequence" + internalDNSEligibleField = "eligible" + internalDNSContentField = "content" + internalDNSRecordTypeField = "recordType" internalDNSInstanceKind = "Instance" internalDNSRecordTypeAAAA = "AAAA" internalDNSNameField = "name" @@ -254,8 +258,8 @@ func instanceDNSAttachments(ctx context.Context, reader client.Reader, projectNa rrtype, field = internalDNSRecordTypeAAAA, "aaaa" } attachment.RecordSets = append(attachment.RecordSets, map[string]any{ - "recordType": rrtype, - "records": []any{map[string]any{internalDNSNameField: "", "ttl": int64(30), field: map[string]any{"content": addr.String()}}}, + internalDNSRecordTypeField: rrtype, + "records": []any{map[string]any{internalDNSNameField: "", "ttl": int64(30), field: map[string]any{internalDNSContentField: addr.String()}}}, }) } } @@ -365,21 +369,21 @@ func (r *InternalDNSPublisherReconciler) reconcileAttachment(ctx context.Context } func (r *InternalDNSPublisherReconciler) refreshObservation(ctx context.Context, writer client.Client, contribution *unstructured.Unstructured, writerEpoch int64, eligible bool) (bool, error) { - oldEligible, _, _ := unstructured.NestedBool(contribution.Object, internalDNSStatusField, "eligible") + oldEligible, _, _ := unstructured.NestedBool(contribution.Object, internalDNSStatusField, internalDNSEligibleField) observedGeneration, _, _ := unstructured.NestedInt64(contribution.Object, internalDNSStatusField, internalDNSObservedGenerationField) if oldEligible == eligible && observedGeneration == contribution.GetGeneration() { return true, nil } base := contribution.DeepCopy() - sequence, _, _ := unstructured.NestedInt64(contribution.Object, internalDNSStatusField, "sequence") + sequence, _, _ := unstructured.NestedInt64(contribution.Object, internalDNSStatusField, internalDNSSequenceField) owned := []struct { path []string value any }{ {[]string{internalDNSStatusField, internalDNSObservedGenerationField}, contribution.GetGeneration()}, {[]string{internalDNSStatusField, internalDNSWriterEpochField}, writerEpoch}, - {[]string{internalDNSStatusField, "sequence"}, sequence + 1}, - {[]string{internalDNSStatusField, "eligible"}, eligible}, + {[]string{internalDNSStatusField, internalDNSSequenceField}, sequence + 1}, + {[]string{internalDNSStatusField, internalDNSEligibleField}, eligible}, {[]string{internalDNSStatusField, internalDNSReasonField}, map[bool]string{true: "AddressAllocatedAndProgrammed", false: "AddressUnavailable"}[eligible]}, } for _, field := range owned { diff --git a/internal/controller/internaldns_publisher_test.go b/internal/controller/internaldns_publisher_test.go index 7a300db3..3d89a9b6 100644 --- a/internal/controller/internaldns_publisher_test.go +++ b/internal/controller/internaldns_publisher_test.go @@ -149,8 +149,8 @@ func TestRefreshObservationPreservesPlatformStatusAndIncrementsFence(t *testing. contribution.Object[internalDNSSpecField] = map[string]any{internalDNSRecordSetsField: []any{}} contribution.Object[internalDNSStatusField] = map[string]any{ internalDNSWriterEpochField: int64(7), - "sequence": int64(11), - "eligible": false, + internalDNSSequenceField: int64(11), + internalDNSEligibleField: false, "publishedRevision": int64(44), internalDNSConditionsField: []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionPublished, internalDNSStatusField: string(metav1.ConditionTrue)}}, } @@ -168,7 +168,7 @@ func TestRefreshObservationPreservesPlatformStatusAndIncrementsFence(t *testing. var got unstructured.Unstructured got.SetGroupVersionKind(dnsContributionGVK) require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(contribution), &got)) - sequence, _, _ := unstructured.NestedInt64(got.Object, internalDNSStatusField, "sequence") + sequence, _, _ := unstructured.NestedInt64(got.Object, internalDNSStatusField, internalDNSSequenceField) published, _, _ := unstructured.NestedInt64(got.Object, internalDNSStatusField, "publishedRevision") conditions, _, _ := unstructured.NestedSlice(got.Object, internalDNSStatusField, internalDNSConditionsField) assert.Equal(t, int64(12), sequence) From 8797cb53b25afa029d8594fec97874ca6940413f Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 19:03:56 -0500 Subject: [PATCH 16/33] fix: Preserve DNS records when interface programming health is unknown --- internal/controller/internaldns_observation.go | 10 +++++++++- internal/controller/internaldns_observation_test.go | 5 ++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/internal/controller/internaldns_observation.go b/internal/controller/internaldns_observation.go index 2365cd2e..a21bcfb1 100644 --- a/internal/controller/internaldns_observation.go +++ b/internal/controller/internaldns_observation.go @@ -109,12 +109,20 @@ func observeProgrammedInterface(ctx context.Context, reader client.Reader, curre if iface.UID == "" || !iface.DeletionTimestamp.IsZero() || iface.Spec.Network.Name != nic.Network.Name || iface.Spec.InterfaceName != internalDNSInterfaceName(nic) || iface.Spec.ClaimRef == nil { return nil, nil } + unknown := false for _, kind := range []string{"Allocated", "Programmed"} { condition := apimeta.FindStatusCondition(iface.Status.Conditions, kind) - if condition == nil || condition.Status != metav1.ConditionTrue || condition.ObservedGeneration != iface.Generation { + if condition == nil || condition.ObservedGeneration != iface.Generation || condition.Status == metav1.ConditionUnknown { + unknown = true + continue + } + if condition.Status == metav1.ConditionFalse { return nil, nil } } + if unknown { + return nil, fmt.Errorf("interface %q programming health is unknown", iface.Name) + } if iface.Spec.ClaimRef.Name != networkInterfaceClaimName(current.Name, internalDNSInterfaceName(nic)) { return nil, fmt.Errorf("interface is held by a different instance slot") } diff --git a/internal/controller/internaldns_observation_test.go b/internal/controller/internaldns_observation_test.go index caba5a28..e8708602 100644 --- a/internal/controller/internaldns_observation_test.go +++ b/internal/controller/internaldns_observation_test.go @@ -31,7 +31,10 @@ func TestObserveInternalDNSInterface(t *testing.T) { }, wantErr: true}, {name: "stale programming generation", mutate: func(_ *computev1alpha.Instance, i *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node) { i.Generation++ - }}, + }, wantErr: true}, + {name: "unknown interface programming", mutate: func(_ *computev1alpha.Instance, i *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node) { + i.Status.Conditions[1].Status = metav1.ConditionUnknown + }, wantErr: true}, {name: "different network", mutate: func(_ *computev1alpha.Instance, i *networkingv1alpha.NetworkInterface, _ *corev1.Pod, _ *corev1.Node) { i.Spec.Network.Name = internalDNSTestForeignValue }}, From 8cfe258f19247ca3cce190b3d400a92281bdd6af Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 19:06:52 -0500 Subject: [PATCH 17/33] refactor: Share DNS record field name --- internal/controller/internaldns_authorization_test.go | 4 ++-- internal/controller/internaldns_publisher.go | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/controller/internaldns_authorization_test.go b/internal/controller/internaldns_authorization_test.go index 9b3ce01a..d795b65a 100644 --- a/internal/controller/internaldns_authorization_test.go +++ b/internal/controller/internaldns_authorization_test.go @@ -82,7 +82,7 @@ func TestUnavailableObservationLeavesPublishedStateUntouched(t *testing.T) { s.AddKnownTypeWithName(dnsContributionGVK, &unstructured.Unstructured{}) published := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, "record") published.SetUID("contribution") - published.Object[internalDNSSpecField] = map[string]any{"recordSets": []any{map[string]any{internalDNSRecordTypeField: "A", "records": []any{map[string]any{internalDNSNameField: "api", "a": map[string]any{internalDNSContentField: "10.0.0.8"}}}}}} + published.Object[internalDNSSpecField] = map[string]any{"recordSets": []any{map[string]any{internalDNSRecordTypeField: "A", internalDNSRecordsField: []any{map[string]any{internalDNSNameField: "api", "a": map[string]any{internalDNSContentField: "10.0.0.8"}}}}}} published.Object[internalDNSStatusField] = map[string]any{"writerEpoch": int64(3), internalDNSSequenceField: int64(9), internalDNSEligibleField: true} cl := fake.NewClientBuilder().WithScheme(s).WithObjects(published).WithStatusSubresource(published).Build() r := &InternalDNSPublisherReconciler{} @@ -136,7 +136,7 @@ func TestPublisherWaitsForIssuerAndNeverMutatesGrants(t *testing.T) { owner := metav1.OwnerReference{APIVersion: computev1alpha.GroupVersion.String(), Kind: internalDNSInstanceKind, Name: instance.Name, UID: instance.UID, Controller: ptrBool(true)} access := InternalDNSProjectAccess{Reader: guard, Writer: guard} identity := InternalDNSProjectIdentity{ProjectUID: "project-a"} - attachment := instanceDNSAttachment{VPCUID: internalDNSTestVPCUID, Eligible: true, RecordSets: []any{map[string]any{internalDNSRecordTypeField: "A", "records": []any{map[string]any{internalDNSNameField: "", "ttl": int64(30), "a": map[string]any{internalDNSContentField: "10.0.0.8"}}}}}} + attachment := instanceDNSAttachment{VPCUID: internalDNSTestVPCUID, Eligible: true, RecordSets: []any{map[string]any{internalDNSRecordTypeField: "A", internalDNSRecordsField: []any{map[string]any{internalDNSNameField: "", "ttl": int64(30), "a": map[string]any{internalDNSContentField: "10.0.0.8"}}}}}} ctx := context.Background() for range 2 { ready, err := r.reconcileAttachment(ctx, access, identity, internalDNSTestNamespace, instance, owner, attachment, "record") diff --git a/internal/controller/internaldns_publisher.go b/internal/controller/internaldns_publisher.go index 76d3f60e..c4abd44b 100644 --- a/internal/controller/internaldns_publisher.go +++ b/internal/controller/internaldns_publisher.go @@ -38,6 +38,7 @@ const ( internalDNSEligibleField = "eligible" internalDNSContentField = "content" internalDNSRecordTypeField = "recordType" + internalDNSRecordsField = "records" internalDNSInstanceKind = "Instance" internalDNSRecordTypeAAAA = "AAAA" internalDNSNameField = "name" @@ -259,7 +260,7 @@ func instanceDNSAttachments(ctx context.Context, reader client.Reader, projectNa } attachment.RecordSets = append(attachment.RecordSets, map[string]any{ internalDNSRecordTypeField: rrtype, - "records": []any{map[string]any{internalDNSNameField: "", "ttl": int64(30), field: map[string]any{internalDNSContentField: addr.String()}}}, + internalDNSRecordsField: []any{map[string]any{internalDNSNameField: "", "ttl": int64(30), field: map[string]any{internalDNSContentField: addr.String()}}}, }) } } @@ -318,7 +319,7 @@ func (r *InternalDNSPublisherReconciler) reconcileAttachment(ctx context.Context ownerName := allocatedInstanceDNSName(instance) for i := range attachment.RecordSets { rrset := attachment.RecordSets[i].(map[string]any) - records := rrset["records"].([]any) + records := rrset[internalDNSRecordsField].([]any) for _, record := range records { record.(map[string]any)[internalDNSNameField] = ownerName } From 01165bd78c44fbf81bf49d6a5098127300377af1 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 06:00:52 -0500 Subject: [PATCH 18/33] api: identify instance DNS status by network reference --- api/v1alpha/instance_types.go | 5 +-- .../compute.datumapis.com_instances.yaml | 8 +--- .../controller/instance_projector_test.go | 8 ++-- internal/controller/internaldns_status.go | 4 +- .../controller/internaldns_status_test.go | 37 ++++++++++++++++++- 5 files changed, 44 insertions(+), 18 deletions(-) diff --git a/api/v1alpha/instance_types.go b/api/v1alpha/instance_types.go index 1c8bf82c..c6452cd3 100644 --- a/api/v1alpha/instance_types.go +++ b/api/v1alpha/instance_types.go @@ -816,8 +816,7 @@ type InstanceStatus struct { // DNS reports assigned private names and publication readiness per network. // Resolver access and record eligibility are managed by platform services. // +kubebuilder:validation:Optional - // +listType=map - // +listMapKey=networkUID + // +listType=atomic DNS []InstanceDNSStatus `json:"dns,omitempty"` // Controller contains status information about the controller managing the instance. @@ -837,8 +836,6 @@ type InstanceStatus struct { type InstanceDNSStatus struct { // Network identifies the attached network in the owning project. Network networkingv1alpha.NetworkRef `json:"network"` - // NetworkUID pins the lifetime that owns these names. - NetworkUID string `json:"networkUID"` // Hostnames contains the fully qualified names allocated for this Instance. Hostnames []string `json:"hostnames,omitempty"` // Conditions reports DNS readiness independently of runtime readiness. diff --git a/config/base/crd/bases/compute.datumapis.com_instances.yaml b/config/base/crd/bases/compute.datumapis.com_instances.yaml index 4aaa89ed..c01dc749 100644 --- a/config/base/crd/bases/compute.datumapis.com_instances.yaml +++ b/config/base/crd/bases/compute.datumapis.com_instances.yaml @@ -1352,17 +1352,11 @@ spec: required: - name type: object - networkUID: - description: NetworkUID pins the lifetime that owns these names. - type: string required: - network - - networkUID type: object type: array - x-kubernetes-list-map-keys: - - networkUID - x-kubernetes-list-type: map + x-kubernetes-list-type: atomic networkInterfaces: description: Network interface information items: diff --git a/internal/controller/instance_projector_test.go b/internal/controller/instance_projector_test.go index e9377c44..2d3a9eda 100644 --- a/internal/controller/instance_projector_test.go +++ b/internal/controller/instance_projector_test.go @@ -33,6 +33,7 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" computev1alpha "go.datum.net/compute/api/v1alpha" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" "go.miloapis.com/milo/pkg/downstreamclient" ) @@ -440,14 +441,15 @@ func TestInstanceProjector_SpecCopied(t *testing.T) { func TestInstanceProjectorPreservesProjectDNSStatus(t *testing.T) { upstream := projTestKarmadaInstance(map[string]string{computev1alpha.InstanceSourceUIDLabel: internalDNSTestEdgeUID}) - upstream.Status.DNS = []computev1alpha.InstanceDNSStatus{{NetworkUID: "untrusted-edge-status"}} - projection := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: projTestInstanceName, Namespace: projTestProjNS, Labels: map[string]string{computev1alpha.InstanceSourceUIDLabel: internalDNSTestEdgeUID}}, Status: computev1alpha.InstanceStatus{DNS: []computev1alpha.InstanceDNSStatus{{NetworkUID: "project-network", Hostnames: []string{"web.datum.internal"}}}}} + upstream.Status.DNS = []computev1alpha.InstanceDNSStatus{{Network: networkingv1alpha.NetworkRef{Name: "untrusted-edge-network", Namespace: "edge"}}} + network := networkingv1alpha.NetworkRef{Name: "project-network", Namespace: projTestProjNS} + projection := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: projTestInstanceName, Namespace: projTestProjNS, Labels: map[string]string{computev1alpha.InstanceSourceUIDLabel: internalDNSTestEdgeUID}}, Status: computev1alpha.InstanceStatus{DNS: []computev1alpha.InstanceDNSStatus{{Network: network, Hostnames: []string{"web.datum.internal"}}}}} cl := fake.NewClientBuilder().WithScheme(newProjectScheme()).WithObjects(projTestWorkloadDeployment(), projection).WithStatusSubresource(&computev1alpha.Instance{}).Build() r := newTestProjector(newKarmadaFakeClient(upstream), cl) _, err := r.Reconcile(context.Background(), projectorRequest()) require.NoError(t, err) require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(projection), projection)) - require.Equal(t, "project-network", projection.Status.DNS[0].NetworkUID) + require.Equal(t, network, projection.Status.DNS[0].Network) } func TestInstanceProjectorReplacesDifferentSourceLifetime(t *testing.T) { diff --git a/internal/controller/internaldns_status.go b/internal/controller/internaldns_status.go index b04253d8..1cb4ab20 100644 --- a/internal/controller/internaldns_status.go +++ b/internal/controller/internaldns_status.go @@ -24,9 +24,9 @@ func (r *InternalDNSPublisherReconciler) observeInterface(ctx context.Context, r } func (r *InternalDNSPublisherReconciler) publicationStatus(ctx context.Context, reader client.Reader, identity InternalDNSProjectIdentity, namespace string, instance *computev1alpha.Instance, attachment instanceDNSAttachment, resourceName string, authorized bool, reconcileErr error) (computev1alpha.InstanceDNSStatus, error) { - result := computev1alpha.InstanceDNSStatus{Network: attachment.Network, NetworkUID: string(attachment.VPCUID)} + result := computev1alpha.InstanceDNSStatus{Network: attachment.Network} for _, previous := range instance.Status.DNS { - if previous.NetworkUID == result.NetworkUID { + if previous.Network.Name == result.Network.Name && previous.Network.Namespace == result.Network.Namespace { result.Conditions = append([]metav1.Condition(nil), previous.Conditions...) } } diff --git a/internal/controller/internaldns_status_test.go b/internal/controller/internaldns_status_test.go index eee7ca0b..6af857b4 100644 --- a/internal/controller/internaldns_status_test.go +++ b/internal/controller/internaldns_status_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/require" computev1alpha "go.datum.net/compute/api/v1alpha" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -16,7 +17,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" ) -func TestPublicationStatusRequiresCurrentUnexpiredPublication(t *testing.T) { +func TestPublicationStatusRequiresCurrentAuthorizedPublication(t *testing.T) { now := time.Now().UTC().Truncate(time.Second) instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestInstanceName, UID: internalDNSTestInstanceUID, Generation: 2}} for _, tc := range []struct { @@ -79,8 +80,40 @@ func TestReportPublicationPreservesRuntimeStatus(t *testing.T) { cl := fake.NewClientBuilder().WithScheme(s).WithObjects(instance).WithStatusSubresource(instance).Build() require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(instance), instance)) r := &InternalDNSPublisherReconciler{} - require.NoError(t, r.reportPublication(context.Background(), cl, instance, []computev1alpha.InstanceDNSStatus{{NetworkUID: internalDNSTestVPCUID}})) + network := networkingv1alpha.NetworkRef{Name: "prod", Namespace: internalDNSTestNamespace} + require.NoError(t, r.reportPublication(context.Background(), cl, instance, []computev1alpha.InstanceDNSStatus{{Network: network}})) require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(instance), instance)) require.Equal(t, metav1.ConditionTrue, instance.Status.Conditions[0].Status) require.Len(t, instance.Status.DNS, 1) + require.Equal(t, network, instance.Status.DNS[0].Network) +} + +func TestPublicationStatusMatchesNamespacedNetworkReference(t *testing.T) { + network := networkingv1alpha.NetworkRef{Name: "prod", Namespace: internalDNSTestNamespace} + transition := metav1.NewTime(time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC)) + for _, tc := range []struct { + name string + network networkingv1alpha.NetworkRef + matches bool + }{ + {name: "same network", network: network, matches: true}, + {name: "different name", network: networkingv1alpha.NetworkRef{Name: "other", Namespace: network.Namespace}}, + {name: "same name in another namespace", network: networkingv1alpha.NetworkRef{Name: network.Name, Namespace: "other"}}, + } { + t.Run(tc.name, func(t *testing.T) { + s := runtime.NewScheme() + s.AddKnownTypeWithName(dnsRegistrationGVK, &unstructured.Unstructured{}) + cl := fake.NewClientBuilder().WithScheme(s).Build() + instance := &computev1alpha.Instance{Status: computev1alpha.InstanceStatus{DNS: []computev1alpha.InstanceDNSStatus{{ + Network: tc.network, + Conditions: []metav1.Condition{{Type: internalDNSConditionReady, Status: metav1.ConditionFalse, Reason: "PendingPublication", LastTransitionTime: transition}}, + }}}} + r := &InternalDNSPublisherReconciler{} + got, err := r.publicationStatus(context.Background(), cl, InternalDNSProjectIdentity{}, internalDNSTestNamespace, instance, instanceDNSAttachment{Network: network, Eligible: true}, internalDNSTestInstanceName, false, nil) + require.NoError(t, err) + require.Equal(t, network, got.Network) + require.Len(t, got.Conditions, 1) + require.Equal(t, tc.matches, got.Conditions[0].LastTransitionTime.Equal(&transition)) + }) + } } From 34800841590c8931c7d2a67a28422772cca52c44 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 06:03:16 -0500 Subject: [PATCH 19/33] test: use shared network status fixture name --- internal/controller/internaldns_status_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/controller/internaldns_status_test.go b/internal/controller/internaldns_status_test.go index 6af857b4..c79cdb88 100644 --- a/internal/controller/internaldns_status_test.go +++ b/internal/controller/internaldns_status_test.go @@ -80,7 +80,7 @@ func TestReportPublicationPreservesRuntimeStatus(t *testing.T) { cl := fake.NewClientBuilder().WithScheme(s).WithObjects(instance).WithStatusSubresource(instance).Build() require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(instance), instance)) r := &InternalDNSPublisherReconciler{} - network := networkingv1alpha.NetworkRef{Name: "prod", Namespace: internalDNSTestNamespace} + network := networkingv1alpha.NetworkRef{Name: internalDNSTestNetworkName, Namespace: internalDNSTestNamespace} require.NoError(t, r.reportPublication(context.Background(), cl, instance, []computev1alpha.InstanceDNSStatus{{Network: network}})) require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(instance), instance)) require.Equal(t, metav1.ConditionTrue, instance.Status.Conditions[0].Status) @@ -89,7 +89,7 @@ func TestReportPublicationPreservesRuntimeStatus(t *testing.T) { } func TestPublicationStatusMatchesNamespacedNetworkReference(t *testing.T) { - network := networkingv1alpha.NetworkRef{Name: "prod", Namespace: internalDNSTestNamespace} + network := networkingv1alpha.NetworkRef{Name: internalDNSTestNetworkName, Namespace: internalDNSTestNamespace} transition := metav1.NewTime(time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC)) for _, tc := range []struct { name string @@ -97,8 +97,8 @@ func TestPublicationStatusMatchesNamespacedNetworkReference(t *testing.T) { matches bool }{ {name: "same network", network: network, matches: true}, - {name: "different name", network: networkingv1alpha.NetworkRef{Name: "other", Namespace: network.Namespace}}, - {name: "same name in another namespace", network: networkingv1alpha.NetworkRef{Name: network.Name, Namespace: "other"}}, + {name: "different name", network: networkingv1alpha.NetworkRef{Name: "another-network", Namespace: network.Namespace}}, + {name: "same name in another namespace", network: networkingv1alpha.NetworkRef{Name: network.Name, Namespace: "another-namespace"}}, } { t.Run(tc.name, func(t *testing.T) { s := runtime.NewScheme() From dcb0f1dedf912ee613836272a182e4756fe4a2f6 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 08:18:35 -0500 Subject: [PATCH 20/33] fix: Publish DNS registrations using zone names --- internal/controller/internaldns_publisher.go | 4 ++-- internal/controller/internaldns_publisher_test.go | 2 +- internal/controller/internaldns_status.go | 2 +- internal/controller/internaldns_status_test.go | 8 ++++---- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/controller/internaldns_publisher.go b/internal/controller/internaldns_publisher.go index c4abd44b..6a3d87ed 100644 --- a/internal/controller/internaldns_publisher.go +++ b/internal/controller/internaldns_publisher.go @@ -429,10 +429,10 @@ func allocatedDNSNamespace(resolver *unstructured.Unstructured) (string, map[str suffix, _, _ := unstructured.NestedString(resolver.Object, internalDNSStatusField, "managedNamespace", "suffix") suffix = strings.TrimSuffix(strings.ToLower(suffix), ".") zoneRef, _, _ := unstructured.NestedMap(resolver.Object, internalDNSStatusField, "managedNamespace", internalDNSZoneRefField) - if len(validation.IsDNS1123Subdomain(suffix)) > 0 || stringValue(zoneRef, internalDNSNameField) == "" || stringValue(zoneRef, internalDNSUIDField) == "" { + if len(validation.IsDNS1123Subdomain(suffix)) > 0 || stringValue(zoneRef, internalDNSNameField) == "" { return "", nil, fmt.Errorf("DNS context %q has no valid allocated namespace", resolver.GetName()) } - return suffix, zoneRef, nil + return suffix, map[string]any{internalDNSNameField: stringValue(zoneRef, internalDNSNameField)}, nil } func unstructuredConditionCurrent(obj *unstructured.Unstructured, conditionType string) bool { diff --git a/internal/controller/internaldns_publisher_test.go b/internal/controller/internaldns_publisher_test.go index 3d89a9b6..0d029dd6 100644 --- a/internal/controller/internaldns_publisher_test.go +++ b/internal/controller/internaldns_publisher_test.go @@ -133,7 +133,7 @@ func resolverContextObject(name, projectUID, vpcUID string, accepted bool) *unst obj.Object[internalDNSSpecField] = map[string]any{"consumerID": projectUID + "/" + vpcUID} obj.Object[internalDNSStatusField] = map[string]any{ internalDNSConditionsField: []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionReady, internalDNSStatusField: status, internalDNSObservedGenerationField: int64(1)}}, - "managedNamespace": map[string]any{"suffix": "datum.internal", internalDNSZoneRefField: map[string]any{internalDNSNameField: "zone", internalDNSUIDField: "zone-uid"}}, + "managedNamespace": map[string]any{"suffix": "datum.internal", internalDNSZoneRefField: map[string]any{internalDNSNameField: "zone"}}, } return obj } diff --git a/internal/controller/internaldns_status.go b/internal/controller/internaldns_status.go index 1cb4ab20..e096d97e 100644 --- a/internal/controller/internaldns_status.go +++ b/internal/controller/internaldns_status.go @@ -77,7 +77,7 @@ func publicationMatchesContext(ctx context.Context, reader client.Reader, identi return false, err } registeredZone, _, _ := unstructured.NestedMap(registration.Object, internalDNSSpecField, internalDNSZoneRefField) - return canonical == allocatedInstanceDNSName(instance)+"."+suffix && stringValue(registeredZone, internalDNSNameField) == stringValue(zone, internalDNSNameField) && stringValue(registeredZone, internalDNSUIDField) == stringValue(zone, internalDNSUIDField), nil + return canonical == allocatedInstanceDNSName(instance)+"."+suffix && stringValue(registeredZone, internalDNSNameField) == stringValue(zone, internalDNSNameField), nil } func (r *InternalDNSPublisherReconciler) reportPublication(ctx context.Context, writer client.Client, instance *computev1alpha.Instance, status []computev1alpha.InstanceDNSStatus) error { diff --git a/internal/controller/internaldns_status_test.go b/internal/controller/internaldns_status_test.go index c79cdb88..fb071caf 100644 --- a/internal/controller/internaldns_status_test.go +++ b/internal/controller/internaldns_status_test.go @@ -28,7 +28,7 @@ func TestPublicationStatusRequiresCurrentAuthorizedPublication(t *testing.T) { }{ {name: "published", current: true, authorized: true, wantReady: true}, {name: "custom domain", domain: "private.example.net", current: true, authorized: true, wantReady: true}, - {name: "different zone lifetime", foreignZone: true, current: true, authorized: true}, + {name: "different zone", foreignZone: true, current: true, authorized: true}, {name: "stale generation", authorized: true}, {name: "revoked grant", current: true}, } { @@ -53,11 +53,11 @@ func TestPublicationStatusRequiresCurrentAuthorizedPublication(t *testing.T) { } conditions := []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionPublished, internalDNSStatusField: string(metav1.ConditionTrue), internalDNSObservedGenerationField: observed}} registration.Object[internalDNSStatusField] = map[string]any{"canonicalFQDN": allocatedInstanceDNSName(instance) + "." + tc.domain + ".", internalDNSConditionsField: conditions} - zoneUID := "zone-uid" + zoneName := "zone" if tc.foreignZone { - zoneUID = internalDNSTestForeignValue + zoneName = internalDNSTestForeignValue } - registration.Object[internalDNSSpecField] = map[string]any{internalDNSZoneRefField: map[string]any{internalDNSNameField: "zone", internalDNSUIDField: zoneUID}} + registration.Object[internalDNSSpecField] = map[string]any{internalDNSZoneRefField: map[string]any{internalDNSNameField: zoneName}} contribution.Object[internalDNSStatusField] = map[string]any{internalDNSConditionsField: conditions} cl := fake.NewClientBuilder().WithScheme(s).WithObjects(resolver, registration, contribution).Build() r := &InternalDNSPublisherReconciler{Now: func() time.Time { return now }} From 6ebf0aea5be60d096c8137951ef98df49268c5e5 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 08:30:02 -0500 Subject: [PATCH 21/33] fix: Preserve read-only interface DNS status --- go.mod | 14 +++++++------- go.sum | 14 ++++++++++++++ internal/controller/internaldns_publisher.go | 1 + internal/controller/internaldns_status.go | 2 +- internal/controller/internaldns_status_test.go | 9 ++++++++- 5 files changed, 31 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index bb2672fd..3892a7ea 100644 --- a/go.mod +++ b/go.mod @@ -10,17 +10,17 @@ require ( github.com/onsi/gomega v1.42.1 github.com/prometheus/client_golang v1.23.2 github.com/stretchr/testify v1.12.1 - go.datum.net/network-services-operator v0.27.0 + go.datum.net/network-services-operator v0.32.9-0.20261013130742-99e3b4b399a9 go.miloapis.com/locations v0.0.1 go.miloapis.com/milo v0.32.1 golang.org/x/crypto v0.56.0 golang.org/x/sync v0.22.0 google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af - k8s.io/api v0.36.1 - k8s.io/apimachinery v0.36.1 - k8s.io/client-go v0.36.1 + k8s.io/api v0.36.3 + k8s.io/apimachinery v0.36.3 + k8s.io/client-go v0.36.3 k8s.io/component-base v0.36.1 - k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 + k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 sigs.k8s.io/controller-runtime v0.24.1 sigs.k8s.io/gateway-api v1.5.1 sigs.k8s.io/multicluster-runtime v0.23.3 @@ -108,7 +108,7 @@ require ( go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.69.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect - k8s.io/streaming v0.36.1 // indirect + k8s.io/streaming v0.36.3 // indirect lukechampine.com/blake3 v1.4.1 // indirect ) @@ -186,7 +186,7 @@ require ( sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.3 // indirect sigs.k8s.io/yaml v1.6.0 ) diff --git a/go.sum b/go.sum index 95fd6935..be9825bf 100644 --- a/go.sum +++ b/go.sum @@ -375,6 +375,8 @@ go.datum.net/datumctl v0.19.0 h1:e47nA0rfCVayfC4Yuta1Frf5t/c39JN/QOeQTgaMQyY= go.datum.net/datumctl v0.19.0/go.mod h1:KN/puWjphNMBxrHXxB2BfZHU/C09UppRZE+6grqcO0Y= go.datum.net/network-services-operator v0.27.0 h1:LYCUjc6i0/f3c5YXSgisnbV8gB3R8emDwNfiYapdCAo= go.datum.net/network-services-operator v0.27.0/go.mod h1:9nuuBWdrkdnIBMaWJsWM3j4CcbKJIF1GDuCmAjdpIHo= +go.datum.net/network-services-operator v0.32.9-0.20261013130742-99e3b4b399a9 h1:KEGfxu3Wi2ip1t7xwQTctOSvpJ++XVRVWAEXUgYU7eU= +go.datum.net/network-services-operator v0.32.9-0.20261013130742-99e3b4b399a9/go.mod h1:yiko/vL/LsXCR5dlBCxPde6wvxNyPZZ1SqF0p2rk85w= go.miloapis.com/locations v0.0.1 h1:voJKqBzyLX5x96M3+5y/ga7fgq9/+vypTizd+bBvqUY= go.miloapis.com/locations v0.0.1/go.mod h1:gzfAfHhSMwl/N68k/uSNYXOKK3IOBJCdXYaBgCE3gdE= go.miloapis.com/milo v0.32.1 h1:xjtJ7bGJpF3KHkpJ2UXxyHmKDa7hBmTO8G8xjELNQXY= @@ -471,14 +473,20 @@ gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= k8s.io/api v0.36.1 h1:XbL/EMj8K2aJpJtePmqUyQMsM0D4QI2pvl7YKJ20FTY= k8s.io/api v0.36.1/go.mod h1:KOWo4ey3TINlXjeHVuwB3i+tXXnu+UcwFBHlI/9dvEo= +k8s.io/api v0.36.3 h1:NxB+05W2UGqXWFXcLO0RB5cnqnUPP5v5sVlaOH0Iz4w= +k8s.io/api v0.36.3/go.mod h1:JzLQKqRHC5+I8RVj/lS3lCg0mg6nWI9Fo/Sk3ElxHzg= k8s.io/apiextensions-apiserver v0.36.1 h1:6JfYmPUsuUIHuN+3QxutXYWj492RqF5fBSx67GYK5Ks= k8s.io/apiextensions-apiserver v0.36.1/go.mod h1:pLzZin90riwisdzKwv/GoTwENooytoIx5zWJb4Hkby8= k8s.io/apimachinery v0.36.1 h1:G63Gjx2W+q0YD+72Vo8oY0nDnePVwnuzTmmy5ENrVSA= k8s.io/apimachinery v0.36.1/go.mod h1:ibYOR00vW/I1kzvi5SF0dRuJ52BvKtfvRdOn35GPQ+8= +k8s.io/apimachinery v0.36.3 h1:PkzMRBRG8joFD8EhCuQAtNPvJlxb82FwplP26HIzvAM= +k8s.io/apimachinery v0.36.3/go.mod h1:cTSjBWgPe/6CQyBKzY/hDIRWCQQQeK0mfLbml0UYFHE= k8s.io/apiserver v0.36.1 h1:iMS5V+rPUertv5P9RaqJgmHHTuh4quWpoxchvMUY+JY= k8s.io/apiserver v0.36.1/go.mod h1:Cby1PbLWztu0GDOxoO6iFOyyqIsziHNEW+w9zVQ22Kw= k8s.io/client-go v0.36.1 h1:FN/K8QIT2CEDt+2WB2HnWrUANZ50AP5GII43/SP2JR0= k8s.io/client-go v0.36.1/go.mod h1:s6rAnCtTGYDQnpNjEhSaISV+2O8jwruZ6m3QOYBFbtU= +k8s.io/client-go v0.36.3 h1:M4JdVzXxYcZk4fGpfDdYnxSwhLKWCFoQsHW6t+z8Hfg= +k8s.io/client-go v0.36.3/go.mod h1:gcPwr0c87vjjG6HB6pWEqOeuYVoXSsREjzux2j6GF30= k8s.io/component-base v0.36.1 h1:iG6GsELftXqTNG9HG6kiVjatSgAw1sf5pJ6R5a6N0kA= k8s.io/component-base v0.36.1/go.mod h1:nf9XPlntRdqO6WMeEWAA5F93Y4ICZQdeT9GeqLDB3JI= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= @@ -487,8 +495,12 @@ k8s.io/kube-openapi v0.0.0-20260330154417-16be699c7b31 h1:V+sn9a/1fEYDGwnllCmqXB k8s.io/kube-openapi v0.0.0-20260330154417-16be699c7b31/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= k8s.io/streaming v0.36.1 h1:L+K68n4Gg940BGNNYtUBvL1WTLL0YnKT3s+P1MNAmR4= k8s.io/streaming v0.36.1/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= +k8s.io/streaming v0.36.3 h1:9rAaqBk0C0Pc7+/fqGekj07NV+/Xrew58p647A0JT8w= +k8s.io/streaming v0.36.3/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 h1:kBawHLSnx/mYHmRnNUf9d4CpjREbeZuxoSGOX/J+aYM= k8s.io/utils v0.0.0-20260319190234-28399d86e0b5/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= +k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 h1:jVkFFVfXdXP74B/zbO3hM3hpSFD0xvhQ5U686DPurkE= +k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3/go.mod h1:M2s5JB1lIYP3jzZdorPLHXIPJzt9vv2muW5a6L9DtNM= lukechampine.com/blake3 v1.4.1 h1:I3Smz7gso8w4/TunLKec6K2fn+kyKtDxr/xcQEN84Wg= lukechampine.com/blake3 v1.4.1/go.mod h1:QFosUxmjB8mnrWFSNwKmvxHpfY72bmD2tQ0kBMM3kwo= mvdan.cc/sh/v3 v3.12.0 h1:ejKUR7ONP5bb+UGHGEG/k9V5+pRVIyD+LsZz7o8KHrI= @@ -509,5 +521,7 @@ sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8= sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/structured-merge-diff/v6 v6.3.3 h1:u08YRbVUi59ri4YD6cg0UqNM4Dimn0sIl+wldcx5PYw= +sigs.k8s.io/structured-merge-diff/v6 v6.3.3/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/internal/controller/internaldns_publisher.go b/internal/controller/internaldns_publisher.go index 6a3d87ed..09808651 100644 --- a/internal/controller/internaldns_publisher.go +++ b/internal/controller/internaldns_publisher.go @@ -46,6 +46,7 @@ const ( internalDNSReasonField = "reason" internalDNSUIDField = "uid" internalDNSZoneRefField = "dnsZoneRef" + internalDNSConditionAccepted = "Accepted" internalDNSGroup = "dns.networking.miloapis.com" internalDNSVersion = "v1alpha1" internalDNSInstanceUID = "internal-dns.compute.datumapis.com/instance-uid" diff --git a/internal/controller/internaldns_status.go b/internal/controller/internaldns_status.go index e096d97e..43798594 100644 --- a/internal/controller/internaldns_status.go +++ b/internal/controller/internaldns_status.go @@ -50,7 +50,7 @@ func (r *InternalDNSPublisherReconciler) publicationStatus(ctx context.Context, if err := reader.Get(ctx, client.ObjectKeyFromObject(contribution), contribution); err != nil && !apierrors.IsNotFound(err) { return result, err } - if authorized && len(result.Hostnames) > 0 && unstructuredConditionCurrent(registration, internalDNSConditionPublished) && unstructuredConditionCurrent(contribution, internalDNSConditionPublished) && attachment.Eligible && attachment.ObservationError == nil { + if authorized && len(result.Hostnames) > 0 && unstructuredConditionCurrent(registration, internalDNSConditionAccepted) && unstructuredConditionCurrent(registration, internalDNSConditionPublished) && unstructuredConditionCurrent(contribution, internalDNSConditionPublished) && attachment.Eligible && attachment.ObservationError == nil { condition.Status, condition.Reason, condition.Message = metav1.ConditionTrue, internalDNSConditionPublished, "Private name is published" } } diff --git a/internal/controller/internaldns_status_test.go b/internal/controller/internaldns_status_test.go index fb071caf..f63cf941 100644 --- a/internal/controller/internaldns_status_test.go +++ b/internal/controller/internaldns_status_test.go @@ -24,6 +24,7 @@ func TestPublicationStatusRequiresCurrentAuthorizedPublication(t *testing.T) { name string domain string foreignZone bool + rejectedRegistration bool current, authorized, wantReady bool }{ {name: "published", current: true, authorized: true, wantReady: true}, @@ -31,6 +32,7 @@ func TestPublicationStatusRequiresCurrentAuthorizedPublication(t *testing.T) { {name: "different zone", foreignZone: true, current: true, authorized: true}, {name: "stale generation", authorized: true}, {name: "revoked grant", current: true}, + {name: "rejected registration", current: true, authorized: true, rejectedRegistration: true}, } { t.Run(tc.name, func(t *testing.T) { if tc.domain == "" { @@ -52,7 +54,12 @@ func TestPublicationStatusRequiresCurrentAuthorizedPublication(t *testing.T) { observed = 2 } conditions := []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionPublished, internalDNSStatusField: string(metav1.ConditionTrue), internalDNSObservedGenerationField: observed}} - registration.Object[internalDNSStatusField] = map[string]any{"canonicalFQDN": allocatedInstanceDNSName(instance) + "." + tc.domain + ".", internalDNSConditionsField: conditions} + accepted := metav1.ConditionTrue + if tc.rejectedRegistration { + accepted = metav1.ConditionFalse + } + registrationConditions := append(append([]any(nil), conditions...), map[string]any{internalDNSConditionTypeField: internalDNSConditionAccepted, internalDNSStatusField: string(accepted), internalDNSObservedGenerationField: observed}) + registration.Object[internalDNSStatusField] = map[string]any{"canonicalFQDN": allocatedInstanceDNSName(instance) + "." + tc.domain + ".", internalDNSConditionsField: registrationConditions} zoneName := "zone" if tc.foreignZone { zoneName = internalDNSTestForeignValue From a7df4778719f7e4fb348eeb086297d68ef79da25 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 08:33:57 -0500 Subject: [PATCH 22/33] test: Preserve interface DNS availability status --- internal/controller/networkinterface_holder_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/controller/networkinterface_holder_test.go b/internal/controller/networkinterface_holder_test.go index 84437cb9..617d9c80 100644 --- a/internal/controller/networkinterface_holder_test.go +++ b/internal/controller/networkinterface_holder_test.go @@ -162,7 +162,9 @@ func TestReconcileHolderAvailability_BecomesAvailable(t *testing.T) { Status: metav1.ConditionTrue, Reason: computev1alpha.InstanceAvailableReasonAvailable, }) - cl := holderTestClient(instance, newHolderTestInterface()) + iface := newHolderTestInterface() + iface.Status.DNS = &networkingv1alpha.NetworkDNSResolverSettings{Nameservers: []string{"fd53::53"}, Searches: []string{"datum.internal"}} + cl := holderTestClient(instance, iface) r := &InstanceReconciler{NetworkingEnabled: true} require.NoError(t, r.reconcileHolderAvailability(context.Background(), cl, instance)) @@ -173,6 +175,9 @@ func TestReconcileHolderAvailability_BecomesAvailable(t *testing.T) { assert.Equal(t, computev1alpha.InstanceAvailableReasonAvailable, condition.Reason) assert.Equal(t, int64(4), condition.ObservedGeneration, "the interface's generation, not the instance's") + var current networkingv1alpha.NetworkInterface + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(iface), ¤t)) + require.Equal(t, iface.Status.DNS, current.Status.DNS) } // TestReconcileHolderAvailability_LeavesAvailable covers the drain path: an From 05e3e27e35ff49db6a234c4db2c1b501d5cb2865 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 08:46:22 -0500 Subject: [PATCH 23/33] fix: Patch only interface availability status --- go.mod | 16 ++++++------ go.sum | 14 ---------- .../controller/networkinterface_holder.go | 3 ++- .../networkinterface_holder_test.go | 26 +++++++++++++++---- 4 files changed, 31 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index 3892a7ea..166760fb 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module go.datum.net/compute go 1.26.4 require ( + github.com/evanphx/json-patch/v5 v5.9.11 github.com/KimMachineGun/automemlimit v0.7.5 github.com/google/go-cmp v0.7.0 github.com/karmada-io/api v1.15.0 @@ -10,17 +11,17 @@ require ( github.com/onsi/gomega v1.42.1 github.com/prometheus/client_golang v1.23.2 github.com/stretchr/testify v1.12.1 - go.datum.net/network-services-operator v0.32.9-0.20261013130742-99e3b4b399a9 + go.datum.net/network-services-operator v0.27.0 go.miloapis.com/locations v0.0.1 go.miloapis.com/milo v0.32.1 golang.org/x/crypto v0.56.0 golang.org/x/sync v0.22.0 google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af - k8s.io/api v0.36.3 - k8s.io/apimachinery v0.36.3 - k8s.io/client-go v0.36.3 + k8s.io/api v0.36.1 + k8s.io/apimachinery v0.36.1 + k8s.io/client-go v0.36.1 k8s.io/component-base v0.36.1 - k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 + k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 sigs.k8s.io/controller-runtime v0.24.1 sigs.k8s.io/gateway-api v1.5.1 sigs.k8s.io/multicluster-runtime v0.23.3 @@ -108,7 +109,7 @@ require ( go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.69.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect - k8s.io/streaming v0.36.3 // indirect + k8s.io/streaming v0.36.1 // indirect lukechampine.com/blake3 v1.4.1 // indirect ) @@ -121,7 +122,6 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect - github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/felixge/httpsnoop v1.1.0 // indirect github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect @@ -186,7 +186,7 @@ require ( sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.3.3 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect sigs.k8s.io/yaml v1.6.0 ) diff --git a/go.sum b/go.sum index be9825bf..95fd6935 100644 --- a/go.sum +++ b/go.sum @@ -375,8 +375,6 @@ go.datum.net/datumctl v0.19.0 h1:e47nA0rfCVayfC4Yuta1Frf5t/c39JN/QOeQTgaMQyY= go.datum.net/datumctl v0.19.0/go.mod h1:KN/puWjphNMBxrHXxB2BfZHU/C09UppRZE+6grqcO0Y= go.datum.net/network-services-operator v0.27.0 h1:LYCUjc6i0/f3c5YXSgisnbV8gB3R8emDwNfiYapdCAo= go.datum.net/network-services-operator v0.27.0/go.mod h1:9nuuBWdrkdnIBMaWJsWM3j4CcbKJIF1GDuCmAjdpIHo= -go.datum.net/network-services-operator v0.32.9-0.20261013130742-99e3b4b399a9 h1:KEGfxu3Wi2ip1t7xwQTctOSvpJ++XVRVWAEXUgYU7eU= -go.datum.net/network-services-operator v0.32.9-0.20261013130742-99e3b4b399a9/go.mod h1:yiko/vL/LsXCR5dlBCxPde6wvxNyPZZ1SqF0p2rk85w= go.miloapis.com/locations v0.0.1 h1:voJKqBzyLX5x96M3+5y/ga7fgq9/+vypTizd+bBvqUY= go.miloapis.com/locations v0.0.1/go.mod h1:gzfAfHhSMwl/N68k/uSNYXOKK3IOBJCdXYaBgCE3gdE= go.miloapis.com/milo v0.32.1 h1:xjtJ7bGJpF3KHkpJ2UXxyHmKDa7hBmTO8G8xjELNQXY= @@ -473,20 +471,14 @@ gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= k8s.io/api v0.36.1 h1:XbL/EMj8K2aJpJtePmqUyQMsM0D4QI2pvl7YKJ20FTY= k8s.io/api v0.36.1/go.mod h1:KOWo4ey3TINlXjeHVuwB3i+tXXnu+UcwFBHlI/9dvEo= -k8s.io/api v0.36.3 h1:NxB+05W2UGqXWFXcLO0RB5cnqnUPP5v5sVlaOH0Iz4w= -k8s.io/api v0.36.3/go.mod h1:JzLQKqRHC5+I8RVj/lS3lCg0mg6nWI9Fo/Sk3ElxHzg= k8s.io/apiextensions-apiserver v0.36.1 h1:6JfYmPUsuUIHuN+3QxutXYWj492RqF5fBSx67GYK5Ks= k8s.io/apiextensions-apiserver v0.36.1/go.mod h1:pLzZin90riwisdzKwv/GoTwENooytoIx5zWJb4Hkby8= k8s.io/apimachinery v0.36.1 h1:G63Gjx2W+q0YD+72Vo8oY0nDnePVwnuzTmmy5ENrVSA= k8s.io/apimachinery v0.36.1/go.mod h1:ibYOR00vW/I1kzvi5SF0dRuJ52BvKtfvRdOn35GPQ+8= -k8s.io/apimachinery v0.36.3 h1:PkzMRBRG8joFD8EhCuQAtNPvJlxb82FwplP26HIzvAM= -k8s.io/apimachinery v0.36.3/go.mod h1:cTSjBWgPe/6CQyBKzY/hDIRWCQQQeK0mfLbml0UYFHE= k8s.io/apiserver v0.36.1 h1:iMS5V+rPUertv5P9RaqJgmHHTuh4quWpoxchvMUY+JY= k8s.io/apiserver v0.36.1/go.mod h1:Cby1PbLWztu0GDOxoO6iFOyyqIsziHNEW+w9zVQ22Kw= k8s.io/client-go v0.36.1 h1:FN/K8QIT2CEDt+2WB2HnWrUANZ50AP5GII43/SP2JR0= k8s.io/client-go v0.36.1/go.mod h1:s6rAnCtTGYDQnpNjEhSaISV+2O8jwruZ6m3QOYBFbtU= -k8s.io/client-go v0.36.3 h1:M4JdVzXxYcZk4fGpfDdYnxSwhLKWCFoQsHW6t+z8Hfg= -k8s.io/client-go v0.36.3/go.mod h1:gcPwr0c87vjjG6HB6pWEqOeuYVoXSsREjzux2j6GF30= k8s.io/component-base v0.36.1 h1:iG6GsELftXqTNG9HG6kiVjatSgAw1sf5pJ6R5a6N0kA= k8s.io/component-base v0.36.1/go.mod h1:nf9XPlntRdqO6WMeEWAA5F93Y4ICZQdeT9GeqLDB3JI= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= @@ -495,12 +487,8 @@ k8s.io/kube-openapi v0.0.0-20260330154417-16be699c7b31 h1:V+sn9a/1fEYDGwnllCmqXB k8s.io/kube-openapi v0.0.0-20260330154417-16be699c7b31/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= k8s.io/streaming v0.36.1 h1:L+K68n4Gg940BGNNYtUBvL1WTLL0YnKT3s+P1MNAmR4= k8s.io/streaming v0.36.1/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= -k8s.io/streaming v0.36.3 h1:9rAaqBk0C0Pc7+/fqGekj07NV+/Xrew58p647A0JT8w= -k8s.io/streaming v0.36.3/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 h1:kBawHLSnx/mYHmRnNUf9d4CpjREbeZuxoSGOX/J+aYM= k8s.io/utils v0.0.0-20260319190234-28399d86e0b5/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= -k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 h1:jVkFFVfXdXP74B/zbO3hM3hpSFD0xvhQ5U686DPurkE= -k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3/go.mod h1:M2s5JB1lIYP3jzZdorPLHXIPJzt9vv2muW5a6L9DtNM= lukechampine.com/blake3 v1.4.1 h1:I3Smz7gso8w4/TunLKec6K2fn+kyKtDxr/xcQEN84Wg= lukechampine.com/blake3 v1.4.1/go.mod h1:QFosUxmjB8mnrWFSNwKmvxHpfY72bmD2tQ0kBMM3kwo= mvdan.cc/sh/v3 v3.12.0 h1:ejKUR7ONP5bb+UGHGEG/k9V5+pRVIyD+LsZz7o8KHrI= @@ -521,7 +509,5 @@ sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8= sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= -sigs.k8s.io/structured-merge-diff/v6 v6.3.3 h1:u08YRbVUi59ri4YD6cg0UqNM4Dimn0sIl+wldcx5PYw= -sigs.k8s.io/structured-merge-diff/v6 v6.3.3/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/internal/controller/networkinterface_holder.go b/internal/controller/networkinterface_holder.go index d295c382..b5d18ba0 100644 --- a/internal/controller/networkinterface_holder.go +++ b/internal/controller/networkinterface_holder.go @@ -139,11 +139,12 @@ func (r *InstanceReconciler) reconcileHolderAvailability( projected := condition projected.ObservedGeneration = networkInterface.Generation + base := networkInterface.DeepCopy() if !apimeta.SetStatusCondition(&networkInterface.Status.Conditions, projected) { continue } - if err := clusterClient.Status().Update(ctx, &networkInterface); err != nil { + if err := clusterClient.Status().Patch(ctx, &networkInterface, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { errs = append(errs, fmt.Errorf("update network interface %s status: %w", key, err)) } } diff --git a/internal/controller/networkinterface_holder_test.go b/internal/controller/networkinterface_holder_test.go index 617d9c80..d5b410df 100644 --- a/internal/controller/networkinterface_holder_test.go +++ b/internal/controller/networkinterface_holder_test.go @@ -4,15 +4,18 @@ package controller import ( "context" + "encoding/json" "errors" "testing" + jsonpatch "github.com/evanphx/json-patch/v5" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" computev1alpha "go.datum.net/compute/api/v1alpha" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" @@ -163,8 +166,23 @@ func TestReconcileHolderAvailability_BecomesAvailable(t *testing.T) { Reason: computev1alpha.InstanceAvailableReasonAvailable, }) iface := newHolderTestInterface() - iface.Status.DNS = &networkingv1alpha.NetworkDNSResolverSettings{Nameservers: []string{"fd53::53"}, Searches: []string{"datum.internal"}} - cl := holderTestClient(instance, iface) + patched := false + cl := fake.NewClientBuilder().WithScheme(newClaimTestScheme()).WithObjects(instance, iface). + WithStatusSubresource(iface, instance).WithInterceptorFuncs(interceptor.Funcs{ + SubResourcePatch: func(ctx context.Context, cl client.Client, subresource string, object client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + data, err := patch.Data(object) + require.NoError(t, err) + merged, err := jsonpatch.MergePatch([]byte(`{"status":{"dns":{"nameservers":["fd53::53"],"searches":["datum.internal"]}}}`), data) + require.NoError(t, err) + var result map[string]any + require.NoError(t, json.Unmarshal(merged, &result)) + dns := result["status"].(map[string]any)["dns"].(map[string]any) + require.Equal(t, []any{"fd53::53"}, dns["nameservers"]) + require.Equal(t, []any{"datum.internal"}, dns["searches"]) + patched = true + return cl.SubResource(subresource).Patch(ctx, object, patch, opts...) + }, + }).Build() r := &InstanceReconciler{NetworkingEnabled: true} require.NoError(t, r.reconcileHolderAvailability(context.Background(), cl, instance)) @@ -175,9 +193,7 @@ func TestReconcileHolderAvailability_BecomesAvailable(t *testing.T) { assert.Equal(t, computev1alpha.InstanceAvailableReasonAvailable, condition.Reason) assert.Equal(t, int64(4), condition.ObservedGeneration, "the interface's generation, not the instance's") - var current networkingv1alpha.NetworkInterface - require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(iface), ¤t)) - require.Equal(t, iface.Status.DNS, current.Status.DNS) + require.True(t, patched) } // TestReconcileHolderAvailability_LeavesAvailable covers the drain path: an From 85900ff2536aa986bafc5e625f70063c683498a2 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 08:51:07 -0500 Subject: [PATCH 24/33] test: Reuse the managed DNS domain fixture --- internal/controller/internaldns_publisher_test.go | 3 ++- internal/controller/internaldns_status_test.go | 2 +- internal/controller/networkinterface_holder_test.go | 7 +++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/internal/controller/internaldns_publisher_test.go b/internal/controller/internaldns_publisher_test.go index 0d029dd6..b1131027 100644 --- a/internal/controller/internaldns_publisher_test.go +++ b/internal/controller/internaldns_publisher_test.go @@ -22,6 +22,7 @@ import ( const ( internalDNSTestNamespace = "dns-test" + internalDNSTestDefaultDomain = "datum.internal" internalDNSTestInstanceName = "api" internalDNSTestInstanceUID = "dns-instance-uid" internalDNSTestEdgeUID = "dns-edge-instance" @@ -133,7 +134,7 @@ func resolverContextObject(name, projectUID, vpcUID string, accepted bool) *unst obj.Object[internalDNSSpecField] = map[string]any{"consumerID": projectUID + "/" + vpcUID} obj.Object[internalDNSStatusField] = map[string]any{ internalDNSConditionsField: []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionReady, internalDNSStatusField: status, internalDNSObservedGenerationField: int64(1)}}, - "managedNamespace": map[string]any{"suffix": "datum.internal", internalDNSZoneRefField: map[string]any{internalDNSNameField: "zone"}}, + "managedNamespace": map[string]any{"suffix": internalDNSTestDefaultDomain, internalDNSZoneRefField: map[string]any{internalDNSNameField: "zone"}}, } return obj } diff --git a/internal/controller/internaldns_status_test.go b/internal/controller/internaldns_status_test.go index f63cf941..d74186ec 100644 --- a/internal/controller/internaldns_status_test.go +++ b/internal/controller/internaldns_status_test.go @@ -36,7 +36,7 @@ func TestPublicationStatusRequiresCurrentAuthorizedPublication(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { if tc.domain == "" { - tc.domain = "datum.internal" + tc.domain = internalDNSTestDefaultDomain } s := runtime.NewScheme() s.AddKnownTypeWithName(dnsResolverContextGVK, &unstructured.Unstructured{}) diff --git a/internal/controller/networkinterface_holder_test.go b/internal/controller/networkinterface_holder_test.go index d5b410df..284d4376 100644 --- a/internal/controller/networkinterface_holder_test.go +++ b/internal/controller/networkinterface_holder_test.go @@ -170,15 +170,18 @@ func TestReconcileHolderAvailability_BecomesAvailable(t *testing.T) { cl := fake.NewClientBuilder().WithScheme(newClaimTestScheme()).WithObjects(instance, iface). WithStatusSubresource(iface, instance).WithInterceptorFuncs(interceptor.Funcs{ SubResourcePatch: func(ctx context.Context, cl client.Client, subresource string, object client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + settings := map[string]any{"nameservers": []any{"fd53::53"}, "searches": []any{internalDNSTestDefaultDomain}} + original, err := json.Marshal(map[string]any{"status": map[string]any{"dns": settings}}) + require.NoError(t, err) data, err := patch.Data(object) require.NoError(t, err) - merged, err := jsonpatch.MergePatch([]byte(`{"status":{"dns":{"nameservers":["fd53::53"],"searches":["datum.internal"]}}}`), data) + merged, err := jsonpatch.MergePatch(original, data) require.NoError(t, err) var result map[string]any require.NoError(t, json.Unmarshal(merged, &result)) dns := result["status"].(map[string]any)["dns"].(map[string]any) require.Equal(t, []any{"fd53::53"}, dns["nameservers"]) - require.Equal(t, []any{"datum.internal"}, dns["searches"]) + require.Equal(t, []any{internalDNSTestDefaultDomain}, dns["searches"]) patched = true return cl.SubResource(subresource).Patch(ctx, object, patch, opts...) }, From b7df134b163065b29313655db2732a3ae147b3ab Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 12:29:15 -0500 Subject: [PATCH 25/33] refactor: Publish instance DNS through project RBAC --- cmd/main.go | 4 - cmd/main_test.go | 5 +- config/components/internal-dns-rbac/role.yaml | 2 +- .../internal-dns-instance-publishing.md | 10 +- internal/config/config.go | 7 +- internal/config/config_test.go | 1 - .../internaldns_authorization_test.go | 166 ------------------ .../internaldns_publication_test.go | 100 +++++++++++ internal/controller/internaldns_publisher.go | 77 +------- .../controller/internaldns_publisher_test.go | 11 +- internal/controller/internaldns_status.go | 4 +- .../controller/internaldns_status_test.go | 24 +-- test/internaldns/README.md | 9 +- test/internaldns/publisher/main.go | 5 +- 14 files changed, 135 insertions(+), 290 deletions(-) delete mode 100644 internal/controller/internaldns_authorization_test.go create mode 100644 internal/controller/internaldns_publication_test.go diff --git a/cmd/main.go b/cmd/main.go index e2b19c6d..cdb12253 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -466,7 +466,6 @@ func main() { } dnsPublisher := &controller.InternalDNSPublisherReconciler{ ProjectIdentities: identities, - PrincipalSubject: serverConfig.InternalDNS.PrincipalSubject, ObservationSources: observers, } if err = dnsPublisher.SetupWithManager(mgr); err != nil { @@ -1004,9 +1003,6 @@ func internalDNSObservationSources( func internalDNSProjectIdentities( cfg config.InternalDNSConfig, ) (map[string]controller.InternalDNSProjectIdentity, error) { - if cfg.PrincipalSubject == "" { - return nil, errors.New("internalDNS.principalSubject is required") - } identities := make(map[string]controller.InternalDNSProjectIdentity, len(cfg.Projects)) for i, project := range cfg.Projects { if project.Name == "" || project.ProjectUID == "" { diff --git a/cmd/main_test.go b/cmd/main_test.go index af63e43e..1892d2fb 100644 --- a/cmd/main_test.go +++ b/cmd/main_test.go @@ -25,12 +25,11 @@ import ( func TestInternalDNSProjectIdentities(t *testing.T) { identities, err := internalDNSProjectIdentities(config.InternalDNSConfig{ - PrincipalSubject: "system:serviceaccount:compute-system:compute-manager", - Projects: []config.InternalDNSProjectIdentity{{Name: "project-a", ProjectUID: "project-uid"}}, + Projects: []config.InternalDNSProjectIdentity{{Name: "project-a", ProjectUID: "project-uid"}}, }) require.NoError(t, err) assert.EqualValues(t, "project-uid", identities["project-a"].ProjectUID) - _, err = internalDNSProjectIdentities(config.InternalDNSConfig{PrincipalSubject: "subject"}) + _, err = internalDNSProjectIdentities(config.InternalDNSConfig{}) assert.Error(t, err) } diff --git a/config/components/internal-dns-rbac/role.yaml b/config/components/internal-dns-rbac/role.yaml index afe766b9..38fc1ec2 100644 --- a/config/components/internal-dns-rbac/role.yaml +++ b/config/components/internal-dns-rbac/role.yaml @@ -4,7 +4,7 @@ metadata: name: compute-internal-dns-publisher rules: - apiGroups: ["dns.networking.miloapis.com"] - resources: ["dnsresolvercontexts", "dnscontributiongrants"] + resources: ["dnsresolvercontexts"] verbs: ["get", "list"] - apiGroups: ["dns.networking.miloapis.com"] resources: ["dnsregistrations", "dnsrecordcontributions"] diff --git a/docs/enhancements/internal-dns-instance-publishing.md b/docs/enhancements/internal-dns-instance-publishing.md index bf320f2e..2c1b7ee4 100644 --- a/docs/enhancements/internal-dns-instance-publishing.md +++ b/docs/enhancements/internal-dns-instance-publishing.md @@ -31,8 +31,6 @@ DNS, project permissions, and read-only edge observation credentials. apiVersion: apiserver.config.datumapis.com/v1alpha1 kind: WorkloadOperator internalDNS: - # Match the authenticated subject used for project DNS writes. - principalSubject: system:serviceaccount:compute-system:compute-manager projects: - name: example-project projectUID: 00000000-0000-0000-0000-000000000001 @@ -43,9 +41,9 @@ internalDNS: ``` Compute selects the network's resolver context and reads its managed zone. It -reserves a name through `DNSRegistration`, reads an independently issued scoped -`DNSContributionGrant`, and publishes `DNSRecordContribution` resources. Project -RBAC authorizes writes. Object lifetime references and writer versions prevent +reserves a name through `DNSRegistration` and publishes endpoint observations +through `DNSRecordContribution`. Project RBAC authorizes writes. Object lifetime +references and observation sequences prevent stale publishers from changing replacement resources. Only allocated, programmed private host addresses are eligible. Confirmed @@ -56,7 +54,7 @@ requirement. Instance deletion reclaims records through project owner references Observation credentials read Instances, NetworkInterfaces, NetworkInterfaceClaims, NetworkContexts, Pods, Nodes, and project namespaces. Scope namespaced reads to -supported edge namespaces. These credentials cannot publish records or issue grants. +supported edge namespaces. These credentials cannot publish DNS records. Workload providers apply the network's resolver address and allocated search domain to guests. Resolver delivery belongs to network services. The publisher diff --git a/internal/config/config.go b/internal/config/config.go index 68dfeafd..d993fcf2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -44,7 +44,7 @@ type WorkloadOperator struct { // ReferencedData configures the ReferencedDataController. ReferencedData ReferencedDataConfig `json:"referencedData,omitempty"` - // InternalDNS configures producer identity for the optional instance DNS + // InternalDNS configures project identities for the optional instance DNS // publisher. It is ignored unless the InternalDNSPublishing feature gate is // enabled. InternalDNS InternalDNSConfig `json:"internalDNS,omitempty"` @@ -59,11 +59,6 @@ type WorkloadOperator struct { // +k8s:deepcopy-gen=true type InternalDNSConfig struct { - // PrincipalSubject is the authenticated Kubernetes username used by this - // process when writing project DNS resources. DNS admission verifies it; it - // is not an authentication substitute. - PrincipalSubject string `json:"principalSubject,omitempty"` - // Projects identifies the consumer project APIs. Projects []InternalDNSProjectIdentity `json:"projects,omitempty"` diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 981e7c1e..cd5981d8 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -35,7 +35,6 @@ kind: WorkloadOperator metricsServer: bindAddress: "0" internalDNS: - principalSubject: system:serviceaccount:compute-system:compute-manager observationSources: - location: dfw kubeconfigPath: /etc/compute/edge.kubeconfig diff --git a/internal/controller/internaldns_authorization_test.go b/internal/controller/internaldns_authorization_test.go deleted file mode 100644 index d795b65a..00000000 --- a/internal/controller/internaldns_authorization_test.go +++ /dev/null @@ -1,166 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only - -package controller - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/require" - computev1alpha "go.datum.net/compute/api/v1alpha" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/client/interceptor" -) - -func TestFindContributionGrantRequiresCurrentScopedAuthority(t *testing.T) { - registration := newDNSObject(dnsRegistrationGVK, internalDNSTestNamespace, internalDNSTestInstanceName) - registration.SetUID("registration-uid") - registration.SetGeneration(2) - for _, tc := range []struct { - name string - path []string - value any - found bool - }{ - {name: "active matching grant", found: true}, - {name: "different principal", path: []string{internalDNSSpecField, internalDNSPrincipalField, internalDNSSubjectField}, value: internalDNSTestForeignValue}, - {name: "different registration lifetime", path: []string{internalDNSSpecField, internalDNSRegistrationRefField, "uid"}, value: internalDNSTestForeignValue}, - {name: "stale registration generation", path: []string{internalDNSSpecField, internalDNSRegistrationRefField, "generation"}, value: int64(1)}, - {name: "different name scope", path: []string{internalDNSSpecField, internalDNSNameScopesField}, value: []any{internalDNSTestForeignValue}}, - {name: "missing record type", path: []string{internalDNSSpecField, internalDNSRecordTypesField}, value: []any{"A"}}, - {name: "inactive grant", path: []string{internalDNSStatusField, internalDNSConditionsField}, value: []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionActive, internalDNSStatusField: string(metav1.ConditionFalse), internalDNSObservedGenerationField: int64(1)}}}, - {name: "stale active condition", path: []string{internalDNSStatusField, internalDNSConditionsField}, value: []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionActive, internalDNSStatusField: string(metav1.ConditionTrue), internalDNSObservedGenerationField: int64(0)}}}, - } { - t.Run(tc.name, func(t *testing.T) { - s := runtime.NewScheme() - s.AddKnownTypeWithName(dnsGrantGVK, &unstructured.Unstructured{}) - s.AddKnownTypeWithName(dnsGrantGVK.GroupVersion().WithKind("DNSContributionGrantList"), &unstructured.UnstructuredList{}) - grant := newDNSObject(dnsGrantGVK, internalDNSTestNamespace, "trusted-grant") - grant.SetUID("grant-uid") - grant.SetGeneration(1) - grant.Object[internalDNSSpecField] = map[string]any{internalDNSRegistrationRefField: objectReference(registration), internalDNSPrincipalField: map[string]any{internalDNSSubjectField: internalDNSTestSubject}, internalDNSNameScopesField: []any{internalDNSTestInstanceName}, internalDNSRecordTypesField: []any{"A", "AAAA"}} - grant.Object[internalDNSStatusField] = map[string]any{internalDNSConditionsField: []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionActive, internalDNSStatusField: string(metav1.ConditionTrue), internalDNSObservedGenerationField: int64(1)}}} - if len(tc.path) > 0 { - require.NoError(t, unstructured.SetNestedField(grant.Object, tc.value, tc.path...)) - } - cl := fake.NewClientBuilder().WithScheme(s).WithObjects(grant).Build() - got, err := findContributionGrant(context.Background(), cl, internalDNSTestNamespace, registration, internalDNSTestSubject, internalDNSTestInstanceName) - require.NoError(t, err) - require.Equal(t, tc.found, got != nil) - }) - } -} - -func TestObservationPersistsUntilExplicitWithdrawal(t *testing.T) { - s := runtime.NewScheme() - s.AddKnownTypeWithName(dnsContributionGVK, &unstructured.Unstructured{}) - c := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, internalDNSTestInstanceName) - c.SetUID("contribution") - c.SetGeneration(1) - cl := fake.NewClientBuilder().WithScheme(s).WithObjects(c).WithStatusSubresource(c).Build() - r := &InternalDNSPublisherReconciler{} - for _, eligible := range []bool{true, true, false} { - require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(c), c)) - _, err := r.refreshObservation(context.Background(), cl, c, 3, eligible) - require.NoError(t, err) - got, _, _ := unstructured.NestedBool(c.Object, internalDNSStatusField, internalDNSEligibleField) - require.Equal(t, eligible, got) - _, found, _ := unstructured.NestedString(c.Object, internalDNSStatusField, "validUntil") - require.False(t, found) - } -} - -func TestUnavailableObservationLeavesPublishedStateUntouched(t *testing.T) { - s := runtime.NewScheme() - s.AddKnownTypeWithName(dnsContributionGVK, &unstructured.Unstructured{}) - published := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, "record") - published.SetUID("contribution") - published.Object[internalDNSSpecField] = map[string]any{"recordSets": []any{map[string]any{internalDNSRecordTypeField: "A", internalDNSRecordsField: []any{map[string]any{internalDNSNameField: "api", "a": map[string]any{internalDNSContentField: "10.0.0.8"}}}}}} - published.Object[internalDNSStatusField] = map[string]any{"writerEpoch": int64(3), internalDNSSequenceField: int64(9), internalDNSEligibleField: true} - cl := fake.NewClientBuilder().WithScheme(s).WithObjects(published).WithStatusSubresource(published).Build() - r := &InternalDNSPublisherReconciler{} - instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: "api", UID: internalDNSTestInstanceUID}} - _, err := r.reconcileAttachment(context.Background(), InternalDNSProjectAccess{Reader: cl, Writer: cl}, InternalDNSProjectIdentity{}, internalDNSTestNamespace, instance, metav1.OwnerReference{}, instanceDNSAttachment{ObservationError: context.DeadlineExceeded}, "record") - require.ErrorIs(t, err, context.DeadlineExceeded) - current := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, "record") - require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(published), current)) - require.Equal(t, published.Object, current.Object) -} - -func TestAllocatedInstanceDNSName(t *testing.T) { - first := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: "Web_01", UID: "first"}} - second := first.DeepCopy() - second.UID = "second" - require.Equal(t, "web-01", allocatedInstanceDNSName(first)) - require.Equal(t, allocatedInstanceDNSName(first), allocatedInstanceDNSName(second)) - require.Equal(t, allocatedInstanceDNSName(first), allocatedInstanceDNSName(first.DeepCopy())) -} - -func TestPublisherWaitsForIssuerAndNeverMutatesGrants(t *testing.T) { - s := runtime.NewScheme() - for _, gvk := range []schema.GroupVersionKind{dnsResolverContextGVK, dnsRegistrationGVK, dnsGrantGVK, dnsContributionGVK} { - s.AddKnownTypeWithName(gvk, &unstructured.Unstructured{}) - s.AddKnownTypeWithName(gvk.GroupVersion().WithKind(gvk.Kind+"List"), &unstructured.UnstructuredList{}) - } - contextObject := resolverContextObject("context", "project-a", internalDNSTestVPCUID, true) - require.NoError(t, unstructured.SetNestedField(contextObject.Object, "private.example.net", internalDNSStatusField, "managedNamespace", "suffix")) - cl := fake.NewClientBuilder().WithScheme(s).WithObjects(contextObject).WithStatusSubresource(newDNSObject(dnsRegistrationGVK, "", ""), newDNSObject(dnsContributionGVK, "", "")).WithInterceptorFuncs(interceptor.Funcs{Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { - obj.SetUID(types.UID(obj.GetName() + "-uid")) - obj.SetGeneration(1) - return c.Create(ctx, obj, opts...) - }}).Build() - guard := interceptor.NewClient(cl, interceptor.Funcs{ - Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { - require.NotEqual(t, dnsGrantGVK, obj.GetObjectKind().GroupVersionKind()) - return c.Create(ctx, obj, opts...) - }, - Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { - require.NotEqual(t, dnsGrantGVK, obj.GetObjectKind().GroupVersionKind()) - return c.Update(ctx, obj, opts...) - }, - Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { - require.NotEqual(t, dnsGrantGVK, obj.GetObjectKind().GroupVersionKind()) - return c.Delete(ctx, obj, opts...) - }, - }) - now := time.Now().UTC().Truncate(time.Second) - r := &InternalDNSPublisherReconciler{PrincipalSubject: internalDNSTestSubject, Now: func() time.Time { return now }} - instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestInstanceName, Namespace: internalDNSTestNamespace, UID: internalDNSTestInstanceUID}} - owner := metav1.OwnerReference{APIVersion: computev1alpha.GroupVersion.String(), Kind: internalDNSInstanceKind, Name: instance.Name, UID: instance.UID, Controller: ptrBool(true)} - access := InternalDNSProjectAccess{Reader: guard, Writer: guard} - identity := InternalDNSProjectIdentity{ProjectUID: "project-a"} - attachment := instanceDNSAttachment{VPCUID: internalDNSTestVPCUID, Eligible: true, RecordSets: []any{map[string]any{internalDNSRecordTypeField: "A", internalDNSRecordsField: []any{map[string]any{internalDNSNameField: "", "ttl": int64(30), "a": map[string]any{internalDNSContentField: "10.0.0.8"}}}}}} - ctx := context.Background() - for range 2 { - ready, err := r.reconcileAttachment(ctx, access, identity, internalDNSTestNamespace, instance, owner, attachment, "record") - require.NoError(t, err) - require.False(t, ready) - } - registration := newDNSObject(dnsRegistrationGVK, internalDNSTestNamespace, "record") - require.NoError(t, cl.Get(ctx, client.ObjectKeyFromObject(registration), registration)) - name, _, _ := unstructured.NestedString(registration.Object, internalDNSSpecField, internalDNSNameField) - require.Equal(t, allocatedInstanceDNSName(instance), name) - grant := newDNSObject(dnsGrantGVK, internalDNSTestNamespace, "issuer-grant") - grant.Object[internalDNSSpecField] = map[string]any{internalDNSRegistrationRefField: objectReference(registration), internalDNSPrincipalField: map[string]any{internalDNSSubjectField: internalDNSTestSubject}, internalDNSNameScopesField: []any{name}, internalDNSRecordTypesField: []any{"A", "AAAA"}} - grant.Object[internalDNSStatusField] = map[string]any{internalDNSConditionsField: []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionActive, internalDNSStatusField: string(metav1.ConditionTrue), internalDNSObservedGenerationField: int64(1)}}, "activeWriterEpoch": int64(3), "observedGrantGeneration": int64(1), "observedRegistrationGeneration": int64(1)} - require.NoError(t, cl.Create(ctx, grant)) - ready, err := r.reconcileAttachment(ctx, access, identity, internalDNSTestNamespace, instance, owner, attachment, "record") - require.NoError(t, err) - require.False(t, ready) - contribution := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, "record") - require.NoError(t, cl.Get(ctx, client.ObjectKeyFromObject(contribution), contribution)) - contribution.Object[internalDNSStatusField] = map[string]any{internalDNSWriterEpochField: int64(3)} - require.NoError(t, cl.Status().Update(ctx, contribution)) - ready, err = r.reconcileAttachment(ctx, access, identity, internalDNSTestNamespace, instance, owner, attachment, "record") - require.NoError(t, err) - require.True(t, ready) - require.NoError(t, r.deletePublishedResources(ctx, access, internalDNSTestNamespace, instance.UID, nil)) - require.NoError(t, cl.Get(ctx, client.ObjectKeyFromObject(grant), grant)) -} diff --git a/internal/controller/internaldns_publication_test.go b/internal/controller/internaldns_publication_test.go new file mode 100644 index 00000000..a01e8f44 --- /dev/null +++ b/internal/controller/internaldns_publication_test.go @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + computev1alpha "go.datum.net/compute/api/v1alpha" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +func TestObservationPersistsUntilExplicitWithdrawal(t *testing.T) { + s := runtime.NewScheme() + s.AddKnownTypeWithName(dnsContributionGVK, &unstructured.Unstructured{}) + c := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, internalDNSTestInstanceName) + c.SetUID("contribution") + c.SetGeneration(1) + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(c).WithStatusSubresource(c).Build() + r := &InternalDNSPublisherReconciler{} + for _, eligible := range []bool{true, true, false} { + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(c), c)) + _, err := r.refreshObservation(context.Background(), cl, c, eligible) + require.NoError(t, err) + got, _, _ := unstructured.NestedBool(c.Object, internalDNSStatusField, internalDNSEligibleField) + require.Equal(t, eligible, got) + _, found, _ := unstructured.NestedString(c.Object, internalDNSStatusField, "validUntil") + require.False(t, found) + } +} + +func TestUnavailableObservationLeavesPublishedStateUntouched(t *testing.T) { + s := runtime.NewScheme() + s.AddKnownTypeWithName(dnsContributionGVK, &unstructured.Unstructured{}) + published := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, "record") + published.SetUID("contribution") + published.Object[internalDNSSpecField] = map[string]any{"recordSets": []any{map[string]any{internalDNSRecordTypeField: "A", internalDNSRecordsField: []any{map[string]any{internalDNSNameField: "api", "a": map[string]any{internalDNSContentField: "10.0.0.8"}}}}}} + published.Object[internalDNSStatusField] = map[string]any{internalDNSSequenceField: int64(9), internalDNSEligibleField: true} + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(published).WithStatusSubresource(published).Build() + r := &InternalDNSPublisherReconciler{} + instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: "api", UID: internalDNSTestInstanceUID}} + _, err := r.reconcileAttachment(context.Background(), InternalDNSProjectAccess{Reader: cl, Writer: cl}, InternalDNSProjectIdentity{}, internalDNSTestNamespace, instance, metav1.OwnerReference{}, instanceDNSAttachment{ObservationError: context.DeadlineExceeded}, "record") + require.ErrorIs(t, err, context.DeadlineExceeded) + current := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, "record") + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(published), current)) + require.Equal(t, published.Object, current.Object) +} + +func TestAllocatedInstanceDNSName(t *testing.T) { + first := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: "Web_01", UID: "first"}} + second := first.DeepCopy() + second.UID = "second" + require.Equal(t, "web-01", allocatedInstanceDNSName(first)) + require.Equal(t, allocatedInstanceDNSName(first), allocatedInstanceDNSName(second)) + require.Equal(t, allocatedInstanceDNSName(first), allocatedInstanceDNSName(first.DeepCopy())) +} + +func TestPublisherCreatesRegistrationAndContribution(t *testing.T) { + scheme := runtime.NewScheme() + for _, gvk := range []schema.GroupVersionKind{dnsResolverContextGVK, dnsRegistrationGVK, dnsContributionGVK} { + scheme.AddKnownTypeWithName(gvk, &unstructured.Unstructured{}) + scheme.AddKnownTypeWithName(gvk.GroupVersion().WithKind(gvk.Kind+"List"), &unstructured.UnstructuredList{}) + } + resolver := resolverContextObject("context", "project-a", internalDNSTestVPCUID, true) + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(resolver).WithStatusSubresource(newDNSObject(dnsContributionGVK, internalDNSTestNamespace, "record")).WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + obj.SetUID(types.UID(obj.GetName() + "-uid")) + obj.SetGeneration(1) + return c.Create(ctx, obj, opts...) + }, + }).Build() + reconciler := &InternalDNSPublisherReconciler{} + instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestInstanceName, Namespace: internalDNSTestNamespace, UID: internalDNSTestInstanceUID}} + owner := metav1.OwnerReference{APIVersion: computev1alpha.GroupVersion.String(), Kind: internalDNSInstanceKind, Name: instance.Name, UID: instance.UID, Controller: ptrBool(true)} + access := InternalDNSProjectAccess{Reader: cl, Writer: cl} + attachment := instanceDNSAttachment{VPCUID: internalDNSTestVPCUID, Eligible: true, RecordSets: []any{map[string]any{internalDNSRecordTypeField: "A", internalDNSRecordsField: []any{map[string]any{internalDNSNameField: "", "a": map[string]any{internalDNSContentField: "10.0.0.8"}}}}}} + ready := false + for range 3 { + var err error + ready, err = reconciler.reconcileAttachment(context.Background(), access, InternalDNSProjectIdentity{ProjectUID: "project-a"}, internalDNSTestNamespace, instance, owner, attachment, "record") + require.NoError(t, err) + } + require.True(t, ready) + contribution := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, "record") + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(contribution), contribution)) + eligible, _, _ := unstructured.NestedBool(contribution.Object, internalDNSStatusField, internalDNSEligibleField) + sequence, _, _ := unstructured.NestedInt64(contribution.Object, internalDNSStatusField, internalDNSSequenceField) + require.True(t, eligible) + require.Equal(t, int64(1), sequence) + _, found, _ := unstructured.NestedFieldNoCopy(contribution.Object, internalDNSSpecField, "grantRef") + require.False(t, found) +} diff --git a/internal/controller/internaldns_publisher.go b/internal/controller/internaldns_publisher.go index 09808651..6dbc7d94 100644 --- a/internal/controller/internaldns_publisher.go +++ b/internal/controller/internaldns_publisher.go @@ -59,25 +59,18 @@ const ( defaultDNSConcurrency = 4 internalDNSConditionsField = "conditions" internalDNSObservedGenerationField = "observedGeneration" - internalDNSWriterEpochField = "writerEpoch" internalDNSRegistrationRefField = "registrationRef" internalDNSRecordTypesField = "recordTypes" - internalDNSNameScopesField = "nameScopes" - internalDNSConditionActive = "Active" internalDNSConditionReady = "Ready" internalDNSConditionPublished = "Published" - internalDNSPrincipalField = "principal" internalDNSConditionTypeField = "type" internalDNSSpecField = "spec" - internalDNSSubjectField = "subject" - internalDNSGrantRefField = "grantRef" internalDNSRecordSetsField = "recordSets" ) var ( dnsResolverContextGVK = schema.GroupVersionKind{Group: internalDNSGroup, Version: internalDNSVersion, Kind: "DNSResolverContext"} dnsRegistrationGVK = schema.GroupVersionKind{Group: internalDNSGroup, Version: internalDNSVersion, Kind: "DNSRegistration"} - dnsGrantGVK = schema.GroupVersionKind{Group: internalDNSGroup, Version: internalDNSVersion, Kind: "DNSContributionGrant"} dnsContributionGVK = schema.GroupVersionKind{Group: internalDNSGroup, Version: internalDNSVersion, Kind: "DNSRecordContribution"} ) @@ -112,7 +105,6 @@ type InternalDNSPublisherReconciler struct { ProjectNamespaceForInstance func(context.Context, multicluster.ClusterName, *computev1alpha.Instance) (string, error) ProjectAccess InternalDNSProjectAccessFunc ProjectIdentities map[string]InternalDNSProjectIdentity - PrincipalSubject string MaxConcurrentReconciles int Now func() time.Time ObservationSources map[string]InternalDNSObservationSource @@ -339,38 +331,18 @@ func (r *InternalDNSPublisherReconciler) reconcileAttachment(ctx context.Context } registrationRef := objectReference(registration) - grant, err := findContributionGrant(ctx, access.Reader, namespace, registration, r.PrincipalSubject, ownerName) - if err != nil { - return false, err - } - if grant == nil { - return false, nil - } - writerEpoch, _, _ := unstructured.NestedInt64(grant.Object, internalDNSStatusField, "activeWriterEpoch") - observedGrant, _, _ := unstructured.NestedInt64(grant.Object, internalDNSStatusField, "observedGrantGeneration") - observedRegistration, _, _ := unstructured.NestedInt64(grant.Object, internalDNSStatusField, "observedRegistrationGeneration") - if writerEpoch < 1 || observedGrant != grant.GetGeneration() || observedRegistration != registration.GetGeneration() { - return false, nil - } - contributionSpec := map[string]any{ internalDNSRegistrationRefField: registrationRef, - internalDNSGrantRefField: objectReference(grant), internalDNSRecordSetsField: attachment.RecordSets, } contribution, changed, err := ensureDNSObject(ctx, access, dnsContributionGVK, namespace, resourceName, labels, owner, contributionSpec) if err != nil || changed { return false, err } - boundEpoch, _, _ := unstructured.NestedInt64(contribution.Object, internalDNSStatusField, internalDNSWriterEpochField) - if boundEpoch != writerEpoch { - return false, nil - } - - return r.refreshObservation(ctx, access.Writer, contribution, writerEpoch, attachment.Eligible) + return r.refreshObservation(ctx, access.Writer, contribution, attachment.Eligible) } -func (r *InternalDNSPublisherReconciler) refreshObservation(ctx context.Context, writer client.Client, contribution *unstructured.Unstructured, writerEpoch int64, eligible bool) (bool, error) { +func (r *InternalDNSPublisherReconciler) refreshObservation(ctx context.Context, writer client.Client, contribution *unstructured.Unstructured, eligible bool) (bool, error) { oldEligible, _, _ := unstructured.NestedBool(contribution.Object, internalDNSStatusField, internalDNSEligibleField) observedGeneration, _, _ := unstructured.NestedInt64(contribution.Object, internalDNSStatusField, internalDNSObservedGenerationField) if oldEligible == eligible && observedGeneration == contribution.GetGeneration() { @@ -383,7 +355,6 @@ func (r *InternalDNSPublisherReconciler) refreshObservation(ctx context.Context, value any }{ {[]string{internalDNSStatusField, internalDNSObservedGenerationField}, contribution.GetGeneration()}, - {[]string{internalDNSStatusField, internalDNSWriterEpochField}, writerEpoch}, {[]string{internalDNSStatusField, internalDNSSequenceField}, sequence + 1}, {[]string{internalDNSStatusField, internalDNSEligibleField}, eligible}, {[]string{internalDNSStatusField, internalDNSReasonField}, map[bool]string{true: "AddressAllocatedAndProgrammed", false: "AddressUnavailable"}[eligible]}, @@ -447,45 +418,6 @@ func unstructuredConditionCurrent(obj *unstructured.Unstructured, conditionType return false } -func findContributionGrant(ctx context.Context, reader client.Reader, namespace string, registration *unstructured.Unstructured, subject, ownerName string) (*unstructured.Unstructured, error) { - list := &unstructured.UnstructuredList{} - list.SetGroupVersionKind(dnsGrantGVK.GroupVersion().WithKind(dnsGrantGVK.Kind + "List")) - if err := reader.List(ctx, list, client.InNamespace(namespace)); err != nil { - return nil, fmt.Errorf("list DNS contribution grants: %w", err) - } - var match *unstructured.Unstructured - for i := range list.Items { - grant := &list.Items[i] - if !grant.GetDeletionTimestamp().IsZero() || grant.GetUID() == "" || grant.GetGeneration() < 1 || !unstructuredConditionCurrent(grant, internalDNSConditionActive) { - continue - } - ref, _, _ := unstructured.NestedMap(grant.Object, internalDNSSpecField, internalDNSRegistrationRefField) - principal, _, _ := unstructured.NestedMap(grant.Object, internalDNSSpecField, internalDNSPrincipalField) - if !reflect.DeepEqual(ref, objectReference(registration)) || stringValue(principal, internalDNSSubjectField) != subject { - continue - } - scopes, _, _ := unstructured.NestedStringSlice(grant.Object, internalDNSSpecField, internalDNSNameScopesField) - types, _, _ := unstructured.NestedStringSlice(grant.Object, internalDNSSpecField, internalDNSRecordTypesField) - if !containsDNSString(scopes, ownerName) || !containsDNSString(types, "A") || !containsDNSString(types, "AAAA") { - continue - } - if match != nil { - return nil, fmt.Errorf("multiple contribution grants authorize registration %q", registration.GetName()) - } - match = grant.DeepCopy() - } - return match, nil -} - -func containsDNSString(values []string, value string) bool { - for _, candidate := range values { - if candidate == value { - return true - } - } - return false -} - func ensureDNSObject(ctx context.Context, access InternalDNSProjectAccess, gvk schema.GroupVersionKind, namespace, name string, labels map[string]string, owner metav1.OwnerReference, spec map[string]any) (*unstructured.Unstructured, bool, error) { desired := newDNSObject(gvk, namespace, name) desired.SetLabels(labels) @@ -542,7 +474,7 @@ func authorityReferencesChanged(gvk schema.GroupVersionKind, current, desired ma if gvk != dnsContributionGVK { return false } - return !reflect.DeepEqual(current[internalDNSRegistrationRefField], desired[internalDNSRegistrationRefField]) || !reflect.DeepEqual(current[internalDNSGrantRefField], desired[internalDNSGrantRefField]) + return !reflect.DeepEqual(current[internalDNSRegistrationRefField], desired[internalDNSRegistrationRefField]) } func (r *InternalDNSPublisherReconciler) deletePublishedResources(ctx context.Context, access InternalDNSProjectAccess, namespace string, instanceUID types.UID, keep map[string]struct{}) error { @@ -636,9 +568,6 @@ func (r *InternalDNSPublisherReconciler) projectAccess(ctx context.Context, proj } func (r *InternalDNSPublisherReconciler) SetupWithManager(mgr mcmanager.Manager) error { - if strings.TrimSpace(r.PrincipalSubject) == "" { - return errors.New("internal DNS principal subject is required") - } if len(r.ProjectIdentities) == 0 { return errors.New("at least one internal DNS project identity is required") } diff --git a/internal/controller/internaldns_publisher_test.go b/internal/controller/internaldns_publisher_test.go index b1131027..a2763761 100644 --- a/internal/controller/internaldns_publisher_test.go +++ b/internal/controller/internaldns_publisher_test.go @@ -149,11 +149,10 @@ func TestRefreshObservationPreservesPlatformStatusAndIncrementsFence(t *testing. contribution.SetResourceVersion("1") contribution.Object[internalDNSSpecField] = map[string]any{internalDNSRecordSetsField: []any{}} contribution.Object[internalDNSStatusField] = map[string]any{ - internalDNSWriterEpochField: int64(7), - internalDNSSequenceField: int64(11), - internalDNSEligibleField: false, - "publishedRevision": int64(44), - internalDNSConditionsField: []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionPublished, internalDNSStatusField: string(metav1.ConditionTrue)}}, + internalDNSSequenceField: int64(11), + internalDNSEligibleField: false, + "publishedRevision": int64(44), + internalDNSConditionsField: []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionPublished, internalDNSStatusField: string(metav1.ConditionTrue)}}, } cl := fake.NewClientBuilder().WithScheme(s).WithObjects(contribution).WithStatusSubresource(contribution).Build() now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) @@ -162,7 +161,7 @@ func TestRefreshObservationPreservesPlatformStatusAndIncrementsFence(t *testing. var live unstructured.Unstructured live.SetGroupVersionKind(dnsContributionGVK) require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(contribution), &live)) - ready, err := r.refreshObservation(context.Background(), cl, &live, 7, true) + ready, err := r.refreshObservation(context.Background(), cl, &live, true) require.NoError(t, err) assert.True(t, ready) diff --git a/internal/controller/internaldns_status.go b/internal/controller/internaldns_status.go index 43798594..8c15c48d 100644 --- a/internal/controller/internaldns_status.go +++ b/internal/controller/internaldns_status.go @@ -23,7 +23,7 @@ func (r *InternalDNSPublisherReconciler) observeInterface(ctx context.Context, r return ObserveInternalDNSInterface(ctx, reader, namespace, instance, nic, r.ObservationSources[instance.Labels[computev1alpha.LocationLabel]]) } -func (r *InternalDNSPublisherReconciler) publicationStatus(ctx context.Context, reader client.Reader, identity InternalDNSProjectIdentity, namespace string, instance *computev1alpha.Instance, attachment instanceDNSAttachment, resourceName string, authorized bool, reconcileErr error) (computev1alpha.InstanceDNSStatus, error) { +func (r *InternalDNSPublisherReconciler) publicationStatus(ctx context.Context, reader client.Reader, identity InternalDNSProjectIdentity, namespace string, instance *computev1alpha.Instance, attachment instanceDNSAttachment, resourceName string, observed bool, reconcileErr error) (computev1alpha.InstanceDNSStatus, error) { result := computev1alpha.InstanceDNSStatus{Network: attachment.Network} for _, previous := range instance.Status.DNS { if previous.Network.Name == result.Network.Name && previous.Network.Namespace == result.Network.Namespace { @@ -50,7 +50,7 @@ func (r *InternalDNSPublisherReconciler) publicationStatus(ctx context.Context, if err := reader.Get(ctx, client.ObjectKeyFromObject(contribution), contribution); err != nil && !apierrors.IsNotFound(err) { return result, err } - if authorized && len(result.Hostnames) > 0 && unstructuredConditionCurrent(registration, internalDNSConditionAccepted) && unstructuredConditionCurrent(registration, internalDNSConditionPublished) && unstructuredConditionCurrent(contribution, internalDNSConditionPublished) && attachment.Eligible && attachment.ObservationError == nil { + if observed && len(result.Hostnames) > 0 && unstructuredConditionCurrent(registration, internalDNSConditionAccepted) && unstructuredConditionCurrent(registration, internalDNSConditionPublished) && unstructuredConditionCurrent(contribution, internalDNSConditionPublished) && attachment.Eligible && attachment.ObservationError == nil { condition.Status, condition.Reason, condition.Message = metav1.ConditionTrue, internalDNSConditionPublished, "Private name is published" } } diff --git a/internal/controller/internaldns_status_test.go b/internal/controller/internaldns_status_test.go index d74186ec..8a2dfa9c 100644 --- a/internal/controller/internaldns_status_test.go +++ b/internal/controller/internaldns_status_test.go @@ -21,18 +21,18 @@ func TestPublicationStatusRequiresCurrentAuthorizedPublication(t *testing.T) { now := time.Now().UTC().Truncate(time.Second) instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestInstanceName, UID: internalDNSTestInstanceUID, Generation: 2}} for _, tc := range []struct { - name string - domain string - foreignZone bool - rejectedRegistration bool - current, authorized, wantReady bool + name string + domain string + foreignZone bool + rejectedRegistration bool + current, observed, wantReady bool }{ - {name: "published", current: true, authorized: true, wantReady: true}, - {name: "custom domain", domain: "private.example.net", current: true, authorized: true, wantReady: true}, - {name: "different zone", foreignZone: true, current: true, authorized: true}, - {name: "stale generation", authorized: true}, - {name: "revoked grant", current: true}, - {name: "rejected registration", current: true, authorized: true, rejectedRegistration: true}, + {name: "published", current: true, observed: true, wantReady: true}, + {name: "custom domain", domain: "private.example.net", current: true, observed: true, wantReady: true}, + {name: "different zone", foreignZone: true, current: true, observed: true}, + {name: "stale generation", observed: true}, + {name: "rejected contribution", current: true}, + {name: "rejected registration", current: true, observed: true, rejectedRegistration: true}, } { t.Run(tc.name, func(t *testing.T) { if tc.domain == "" { @@ -68,7 +68,7 @@ func TestPublicationStatusRequiresCurrentAuthorizedPublication(t *testing.T) { contribution.Object[internalDNSStatusField] = map[string]any{internalDNSConditionsField: conditions} cl := fake.NewClientBuilder().WithScheme(s).WithObjects(resolver, registration, contribution).Build() r := &InternalDNSPublisherReconciler{Now: func() time.Time { return now }} - got, err := r.publicationStatus(context.Background(), cl, InternalDNSProjectIdentity{ProjectUID: "project-a"}, internalDNSTestNamespace, instance, instanceDNSAttachment{VPCUID: internalDNSTestVPCUID, Eligible: true}, internalDNSTestInstanceName, tc.authorized, nil) + got, err := r.publicationStatus(context.Background(), cl, InternalDNSProjectIdentity{ProjectUID: "project-a"}, internalDNSTestNamespace, instance, instanceDNSAttachment{VPCUID: internalDNSTestVPCUID, Eligible: true}, internalDNSTestInstanceName, tc.observed, nil) require.NoError(t, err) if tc.foreignZone { require.Empty(t, got.Hostnames) diff --git a/test/internaldns/README.md b/test/internaldns/README.md index 33582b2b..34189e13 100644 --- a/test/internaldns/README.md +++ b/test/internaldns/README.md @@ -13,8 +13,7 @@ bin/setup-envtest use 1.31.0 --bin-dir bin -p path go test ./internal/controller ./internal/config ./cmd/... ``` -The tests cover context identity, flat instance names, independently issued -grants, fenced eligibility updates, live edge observations, source lifetime +The tests cover context identity, flat instance names, versioned endpoint observations, live edge observations, source lifetime replacement, and project DNS status ownership. They use fixtures and do not qualify guest resolution. @@ -25,13 +24,12 @@ go build -o /tmp/compute-dns-publisher ./test/internaldns/publisher /tmp/compute-dns-publisher --feature-gates=InternalDNSPublishing=true \ --kubeconfig /tmp/project-publisher.kubeconfig \ --namespace production --project-uid PROJECT_UID \ - --subject system:serviceaccount:compute-system:compute-manager \ --edge-kubeconfig /tmp/edge-observer.kubeconfig \ --location dfw ``` Provision a resolver context with a `datum.internal` managed zone, the scoped -project publisher role, and a separate trusted grant issuer. Instances must +project publisher role. Instances must carry their source lifetime and location labels. Edge namespaces use the project namespace UID. The observer requires allocated, programmed interfaces, live claims, Instance-owned runtime Pods, and confirmed node health. @@ -39,7 +37,6 @@ The disabled runner starts without creating Kubernetes clients. Release qualification must exercise two VPCs using the same default domain, real address allocation, guest resolver application, and UDP/TCP queries through -Galactic. Include address changes, withdrawal, publisher outages, grant -revocation, stale replay, source replacement, and controller restarts. Run this +Galactic. Include address changes, withdrawal, publisher outages, stale replay, source replacement, and controller restarts. Run this qualification in CI and reuse the shared Kubernetes foundation. The publisher contract tests and runner do not establish that those release criteria pass. diff --git a/test/internaldns/publisher/main.go b/test/internaldns/publisher/main.go index 0ec80dd9..8377f850 100644 --- a/test/internaldns/publisher/main.go +++ b/test/internaldns/publisher/main.go @@ -32,13 +32,12 @@ import ( ) func main() { - var kubeconfig, namespace, projectUID, subject, gates string + var kubeconfig, namespace, projectUID, gates string var edgeKubeconfig, location string flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError) flags.StringVar(&kubeconfig, "kubeconfig", "", "Disposable project kubeconfig") flags.StringVar(&namespace, "namespace", "project-e2e", "Project namespace") flags.StringVar(&projectUID, "project-uid", "project-e2e-uid", "DNS project UID") - flags.StringVar(&subject, "subject", "system:serviceaccount:compute-system:dns-publisher", "Authenticated writer") flags.StringVar(&gates, "feature-gates", "", "Compute feature gates") flags.StringVar(&edgeKubeconfig, "edge-kubeconfig", "", "Read-only edge observation kubeconfig") flags.StringVar(&location, "location", "", "Instance location label") @@ -94,7 +93,7 @@ func main() { ObservationSources: map[string]controller.InternalDNSObservationSource{location: {Reader: edge}}, ProjectIdentities: map[string]controller.InternalDNSProjectIdentity{"single": { ProjectName: "single", ProjectUID: types.UID(projectUID), - }}, PrincipalSubject: subject, + }}, } if err := publisher.SetupWithManager(mgr); err != nil { panic(err) From 9586fdcb4749f9a397151bb8817d5a3c1987db8c Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 12:35:53 -0500 Subject: [PATCH 26/33] refactor: Read DNS reference names directly --- internal/controller/internaldns_publisher.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/controller/internaldns_publisher.go b/internal/controller/internaldns_publisher.go index 6dbc7d94..726a66fd 100644 --- a/internal/controller/internaldns_publisher.go +++ b/internal/controller/internaldns_publisher.go @@ -401,10 +401,10 @@ func allocatedDNSNamespace(resolver *unstructured.Unstructured) (string, map[str suffix, _, _ := unstructured.NestedString(resolver.Object, internalDNSStatusField, "managedNamespace", "suffix") suffix = strings.TrimSuffix(strings.ToLower(suffix), ".") zoneRef, _, _ := unstructured.NestedMap(resolver.Object, internalDNSStatusField, "managedNamespace", internalDNSZoneRefField) - if len(validation.IsDNS1123Subdomain(suffix)) > 0 || stringValue(zoneRef, internalDNSNameField) == "" { + if len(validation.IsDNS1123Subdomain(suffix)) > 0 || referenceName(zoneRef) == "" { return "", nil, fmt.Errorf("DNS context %q has no valid allocated namespace", resolver.GetName()) } - return suffix, map[string]any{internalDNSNameField: stringValue(zoneRef, internalDNSNameField)}, nil + return suffix, map[string]any{internalDNSNameField: referenceName(zoneRef)}, nil } func unstructuredConditionCurrent(obj *unstructured.Unstructured, conditionType string) bool { @@ -512,8 +512,8 @@ func objectReference(obj *unstructured.Unstructured) map[string]any { return ref } -func stringValue(values map[string]any, key string) string { - value, _ := values[key].(string) +func referenceName(values map[string]any) string { + value, _ := values[internalDNSNameField].(string) return value } From fcf3a4efbed87dd53f4f15d3be80bba9b58ac1d1 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 12:49:33 -0500 Subject: [PATCH 27/33] fix: Read publication zone references consistently --- internal/controller/internaldns_status.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/controller/internaldns_status.go b/internal/controller/internaldns_status.go index 8c15c48d..c44e3eae 100644 --- a/internal/controller/internaldns_status.go +++ b/internal/controller/internaldns_status.go @@ -77,7 +77,7 @@ func publicationMatchesContext(ctx context.Context, reader client.Reader, identi return false, err } registeredZone, _, _ := unstructured.NestedMap(registration.Object, internalDNSSpecField, internalDNSZoneRefField) - return canonical == allocatedInstanceDNSName(instance)+"."+suffix && stringValue(registeredZone, internalDNSNameField) == stringValue(zone, internalDNSNameField), nil + return canonical == allocatedInstanceDNSName(instance)+"."+suffix && referenceName(registeredZone) == referenceName(zone), nil } func (r *InternalDNSPublisherReconciler) reportPublication(ctx context.Context, writer client.Client, instance *computev1alpha.Instance, status []computev1alpha.InstanceDNSStatus) error { From 330826eb15a01558ad5019e06bcb0562c6ef55a8 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 13:32:12 -0500 Subject: [PATCH 28/33] feat: Publish DNS addresses and health together Keep private names aligned with endpoint eligibility through one atomic publication. Preserve DNS-owned readiness and retain records when edge observations are unavailable. --- .../internal-dns-instance-publishing.md | 8 ++-- .../internaldns_publication_test.go | 45 +++++++++---------- internal/controller/internaldns_publisher.go | 42 ++--------------- .../controller/internaldns_publisher_test.go | 38 ---------------- test/internaldns/README.md | 8 ++-- 5 files changed, 32 insertions(+), 109 deletions(-) diff --git a/docs/enhancements/internal-dns-instance-publishing.md b/docs/enhancements/internal-dns-instance-publishing.md index 2c1b7ee4..51747358 100644 --- a/docs/enhancements/internal-dns-instance-publishing.md +++ b/docs/enhancements/internal-dns-instance-publishing.md @@ -41,10 +41,10 @@ internalDNS: ``` Compute selects the network's resolver context and reads its managed zone. It -reserves a name through `DNSRegistration` and publishes endpoint observations -through `DNSRecordContribution`. Project RBAC authorizes writes. Object lifetime -references and observation sequences prevent -stale publishers from changing replacement resources. +reserves a name through `DNSRegistration` and publishes addresses and eligibility +atomically through `DNSRecordContribution`. Project RBAC authorizes writes. +Object lifetime references and contribution generations prevent stale publishers +from changing replacement resources. Only allocated, programmed private host addresses are eligible. Confirmed interface withdrawal, stopped runtime, or unhealthy node withdraws addresses. diff --git a/internal/controller/internaldns_publication_test.go b/internal/controller/internaldns_publication_test.go index a01e8f44..25f77e7c 100644 --- a/internal/controller/internaldns_publication_test.go +++ b/internal/controller/internaldns_publication_test.go @@ -18,32 +18,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/interceptor" ) -func TestObservationPersistsUntilExplicitWithdrawal(t *testing.T) { - s := runtime.NewScheme() - s.AddKnownTypeWithName(dnsContributionGVK, &unstructured.Unstructured{}) - c := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, internalDNSTestInstanceName) - c.SetUID("contribution") - c.SetGeneration(1) - cl := fake.NewClientBuilder().WithScheme(s).WithObjects(c).WithStatusSubresource(c).Build() - r := &InternalDNSPublisherReconciler{} - for _, eligible := range []bool{true, true, false} { - require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(c), c)) - _, err := r.refreshObservation(context.Background(), cl, c, eligible) - require.NoError(t, err) - got, _, _ := unstructured.NestedBool(c.Object, internalDNSStatusField, internalDNSEligibleField) - require.Equal(t, eligible, got) - _, found, _ := unstructured.NestedString(c.Object, internalDNSStatusField, "validUntil") - require.False(t, found) - } -} - func TestUnavailableObservationLeavesPublishedStateUntouched(t *testing.T) { s := runtime.NewScheme() s.AddKnownTypeWithName(dnsContributionGVK, &unstructured.Unstructured{}) published := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, "record") published.SetUID("contribution") published.Object[internalDNSSpecField] = map[string]any{"recordSets": []any{map[string]any{internalDNSRecordTypeField: "A", internalDNSRecordsField: []any{map[string]any{internalDNSNameField: "api", "a": map[string]any{internalDNSContentField: "10.0.0.8"}}}}}} - published.Object[internalDNSStatusField] = map[string]any{internalDNSSequenceField: int64(9), internalDNSEligibleField: true} + published.Object[internalDNSSpecField].(map[string]any)[internalDNSEligibleField] = true + published.Object[internalDNSStatusField] = map[string]any{internalDNSConditionsField: []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionPublished, internalDNSStatusField: string(metav1.ConditionTrue)}}} cl := fake.NewClientBuilder().WithScheme(s).WithObjects(published).WithStatusSubresource(published).Build() r := &InternalDNSPublisherReconciler{} instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{Name: "api", UID: internalDNSTestInstanceUID}} @@ -91,10 +73,23 @@ func TestPublisherCreatesRegistrationAndContribution(t *testing.T) { require.True(t, ready) contribution := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, "record") require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(contribution), contribution)) - eligible, _, _ := unstructured.NestedBool(contribution.Object, internalDNSStatusField, internalDNSEligibleField) - sequence, _, _ := unstructured.NestedInt64(contribution.Object, internalDNSStatusField, internalDNSSequenceField) + eligible, _, _ := unstructured.NestedBool(contribution.Object, internalDNSSpecField, internalDNSEligibleField) require.True(t, eligible) - require.Equal(t, int64(1), sequence) - _, found, _ := unstructured.NestedFieldNoCopy(contribution.Object, internalDNSSpecField, "grantRef") - require.False(t, found) + reason, _, _ := unstructured.NestedString(contribution.Object, internalDNSSpecField, internalDNSReasonField) + require.Equal(t, "AddressAllocatedAndProgrammed", reason) + + contribution.Object[internalDNSStatusField] = map[string]any{internalDNSConditionsField: []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionPublished, internalDNSStatusField: string(metav1.ConditionTrue)}}} + require.NoError(t, cl.Status().Update(context.Background(), contribution)) + publishedStatus := contribution.Object[internalDNSStatusField] + attachment.Eligible = false + attachment.RecordSets = []any{map[string]any{internalDNSRecordTypeField: "A", internalDNSRecordsField: []any{map[string]any{internalDNSNameField: "", "a": map[string]any{internalDNSContentField: "10.0.0.9"}}}}} + _, err := reconciler.reconcileAttachment(context.Background(), access, InternalDNSProjectIdentity{ProjectUID: "project-a"}, internalDNSTestNamespace, instance, owner, attachment, "record") + require.NoError(t, err) + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(contribution), contribution)) + eligible, _, _ = unstructured.NestedBool(contribution.Object, internalDNSSpecField, internalDNSEligibleField) + require.False(t, eligible) + recordSets, _, _ := unstructured.NestedSlice(contribution.Object, internalDNSSpecField, internalDNSRecordSetsField) + record := recordSets[0].(map[string]any)[internalDNSRecordsField].([]any)[0].(map[string]any) + require.Equal(t, "10.0.0.9", record["a"].(map[string]any)[internalDNSContentField]) + require.Equal(t, publishedStatus, contribution.Object[internalDNSStatusField]) } diff --git a/internal/controller/internaldns_publisher.go b/internal/controller/internaldns_publisher.go index 726a66fd..5a966d14 100644 --- a/internal/controller/internaldns_publisher.go +++ b/internal/controller/internaldns_publisher.go @@ -34,7 +34,6 @@ import ( ) const ( - internalDNSSequenceField = "sequence" internalDNSEligibleField = "eligible" internalDNSContentField = "content" internalDNSRecordTypeField = "recordType" @@ -322,7 +321,6 @@ func (r *InternalDNSPublisherReconciler) reconcileAttachment(ctx context.Context internalDNSZoneRefField: zoneRef, internalDNSNameField: ownerName, internalDNSRecordTypesField: []any{"A", internalDNSRecordTypeAAAA}, - "publicationPolicy": "EligibleContributions", "ttlSeconds": int64(30), } registration, changed, err := ensureDNSObject(ctx, access, dnsRegistrationGVK, namespace, resourceName, labels, owner, registrationSpec) @@ -334,43 +332,11 @@ func (r *InternalDNSPublisherReconciler) reconcileAttachment(ctx context.Context contributionSpec := map[string]any{ internalDNSRegistrationRefField: registrationRef, internalDNSRecordSetsField: attachment.RecordSets, + internalDNSEligibleField: attachment.Eligible, + internalDNSReasonField: map[bool]string{true: "AddressAllocatedAndProgrammed", false: "AddressUnavailable"}[attachment.Eligible], } - contribution, changed, err := ensureDNSObject(ctx, access, dnsContributionGVK, namespace, resourceName, labels, owner, contributionSpec) - if err != nil || changed { - return false, err - } - return r.refreshObservation(ctx, access.Writer, contribution, attachment.Eligible) -} - -func (r *InternalDNSPublisherReconciler) refreshObservation(ctx context.Context, writer client.Client, contribution *unstructured.Unstructured, eligible bool) (bool, error) { - oldEligible, _, _ := unstructured.NestedBool(contribution.Object, internalDNSStatusField, internalDNSEligibleField) - observedGeneration, _, _ := unstructured.NestedInt64(contribution.Object, internalDNSStatusField, internalDNSObservedGenerationField) - if oldEligible == eligible && observedGeneration == contribution.GetGeneration() { - return true, nil - } - base := contribution.DeepCopy() - sequence, _, _ := unstructured.NestedInt64(contribution.Object, internalDNSStatusField, internalDNSSequenceField) - owned := []struct { - path []string - value any - }{ - {[]string{internalDNSStatusField, internalDNSObservedGenerationField}, contribution.GetGeneration()}, - {[]string{internalDNSStatusField, internalDNSSequenceField}, sequence + 1}, - {[]string{internalDNSStatusField, internalDNSEligibleField}, eligible}, - {[]string{internalDNSStatusField, internalDNSReasonField}, map[bool]string{true: "AddressAllocatedAndProgrammed", false: "AddressUnavailable"}[eligible]}, - } - for _, field := range owned { - if err := unstructured.SetNestedField(contribution.Object, field.value, field.path...); err != nil { - return false, err - } - } - // Optimistic locking turns a concurrent DNS status write into a conflict and - // retry. The retry reads the live sequence and platform-owned status fields; - // it cannot overwrite a publication condition from a stale snapshot. - if err := writer.Status().Patch(ctx, contribution, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { - return false, fmt.Errorf("refresh DNS contribution observation: %w", err) - } - return true, nil + _, changed, err = ensureDNSObject(ctx, access, dnsContributionGVK, namespace, resourceName, labels, owner, contributionSpec) + return err == nil && !changed, err } func findResolverContext(ctx context.Context, reader client.Reader, namespace string, projectUID, vpcUID types.UID) (*unstructured.Unstructured, error) { diff --git a/internal/controller/internaldns_publisher_test.go b/internal/controller/internaldns_publisher_test.go index a2763761..bc821475 100644 --- a/internal/controller/internaldns_publisher_test.go +++ b/internal/controller/internaldns_publisher_test.go @@ -5,7 +5,6 @@ package controller import ( "context" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -139,43 +138,6 @@ func resolverContextObject(name, projectUID, vpcUID string, accepted bool) *unst return obj } -func TestRefreshObservationPreservesPlatformStatusAndIncrementsFence(t *testing.T) { - s := runtime.NewScheme() - s.AddKnownTypeWithName(dnsContributionGVK, &unstructured.Unstructured{}) - s.AddKnownTypeWithName(dnsContributionGVK.GroupVersion().WithKind("DNSRecordContributionList"), &unstructured.UnstructuredList{}) - contribution := newDNSObject(dnsContributionGVK, internalDNSTestNamespace, "endpoint") - contribution.SetUID("contribution-uid") - contribution.SetGeneration(3) - contribution.SetResourceVersion("1") - contribution.Object[internalDNSSpecField] = map[string]any{internalDNSRecordSetsField: []any{}} - contribution.Object[internalDNSStatusField] = map[string]any{ - internalDNSSequenceField: int64(11), - internalDNSEligibleField: false, - "publishedRevision": int64(44), - internalDNSConditionsField: []any{map[string]any{internalDNSConditionTypeField: internalDNSConditionPublished, internalDNSStatusField: string(metav1.ConditionTrue)}}, - } - cl := fake.NewClientBuilder().WithScheme(s).WithObjects(contribution).WithStatusSubresource(contribution).Build() - now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) - r := &InternalDNSPublisherReconciler{Now: func() time.Time { return now }} - - var live unstructured.Unstructured - live.SetGroupVersionKind(dnsContributionGVK) - require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(contribution), &live)) - ready, err := r.refreshObservation(context.Background(), cl, &live, true) - require.NoError(t, err) - assert.True(t, ready) - - var got unstructured.Unstructured - got.SetGroupVersionKind(dnsContributionGVK) - require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(contribution), &got)) - sequence, _, _ := unstructured.NestedInt64(got.Object, internalDNSStatusField, internalDNSSequenceField) - published, _, _ := unstructured.NestedInt64(got.Object, internalDNSStatusField, "publishedRevision") - conditions, _, _ := unstructured.NestedSlice(got.Object, internalDNSStatusField, internalDNSConditionsField) - assert.Equal(t, int64(12), sequence) - assert.Equal(t, int64(44), published) - assert.Len(t, conditions, 1) -} - func TestEnsureDNSObjectRefusesForeignCollision(t *testing.T) { s := runtime.NewScheme() s.AddKnownTypeWithName(dnsRegistrationGVK, &unstructured.Unstructured{}) diff --git a/test/internaldns/README.md b/test/internaldns/README.md index 34189e13..e684f46c 100644 --- a/test/internaldns/README.md +++ b/test/internaldns/README.md @@ -13,8 +13,8 @@ bin/setup-envtest use 1.31.0 --bin-dir bin -p path go test ./internal/controller ./internal/config ./cmd/... ``` -The tests cover context identity, flat instance names, versioned endpoint observations, live edge observations, source lifetime -replacement, and project DNS status ownership. They use fixtures and do not +The tests cover context identity, flat instance names, atomic endpoint publications, live edge observations, +source lifetime replacement, and DNS-owned publication status. They use fixtures and do not qualify guest resolution. The standalone runner starts the production publisher on an existing project API: @@ -28,8 +28,8 @@ go build -o /tmp/compute-dns-publisher ./test/internaldns/publisher --location dfw ``` -Provision a resolver context with a `datum.internal` managed zone, the scoped -project publisher role. Instances must +Provision a resolver context with a `datum.internal` managed zone and scoped +project publisher permissions. Instances must carry their source lifetime and location labels. Edge namespaces use the project namespace UID. The observer requires allocated, programmed interfaces, live claims, Instance-owned runtime Pods, and confirmed node health. From f6b65066ae7ebcbc7f51141c37d2c401c1086a4d Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 13:35:49 -0500 Subject: [PATCH 29/33] test: Share the DNS project identity --- internal/controller/internaldns_publication_test.go | 6 +++--- internal/controller/internaldns_publisher_test.go | 1 + internal/controller/internaldns_status_test.go | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/internal/controller/internaldns_publication_test.go b/internal/controller/internaldns_publication_test.go index 25f77e7c..3384faa2 100644 --- a/internal/controller/internaldns_publication_test.go +++ b/internal/controller/internaldns_publication_test.go @@ -51,7 +51,7 @@ func TestPublisherCreatesRegistrationAndContribution(t *testing.T) { scheme.AddKnownTypeWithName(gvk, &unstructured.Unstructured{}) scheme.AddKnownTypeWithName(gvk.GroupVersion().WithKind(gvk.Kind+"List"), &unstructured.UnstructuredList{}) } - resolver := resolverContextObject("context", "project-a", internalDNSTestVPCUID, true) + resolver := resolverContextObject("context", internalDNSTestProjectUID, internalDNSTestVPCUID, true) cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(resolver).WithStatusSubresource(newDNSObject(dnsContributionGVK, internalDNSTestNamespace, "record")).WithInterceptorFuncs(interceptor.Funcs{ Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { obj.SetUID(types.UID(obj.GetName() + "-uid")) @@ -67,7 +67,7 @@ func TestPublisherCreatesRegistrationAndContribution(t *testing.T) { ready := false for range 3 { var err error - ready, err = reconciler.reconcileAttachment(context.Background(), access, InternalDNSProjectIdentity{ProjectUID: "project-a"}, internalDNSTestNamespace, instance, owner, attachment, "record") + ready, err = reconciler.reconcileAttachment(context.Background(), access, InternalDNSProjectIdentity{ProjectUID: internalDNSTestProjectUID}, internalDNSTestNamespace, instance, owner, attachment, "record") require.NoError(t, err) } require.True(t, ready) @@ -83,7 +83,7 @@ func TestPublisherCreatesRegistrationAndContribution(t *testing.T) { publishedStatus := contribution.Object[internalDNSStatusField] attachment.Eligible = false attachment.RecordSets = []any{map[string]any{internalDNSRecordTypeField: "A", internalDNSRecordsField: []any{map[string]any{internalDNSNameField: "", "a": map[string]any{internalDNSContentField: "10.0.0.9"}}}}} - _, err := reconciler.reconcileAttachment(context.Background(), access, InternalDNSProjectIdentity{ProjectUID: "project-a"}, internalDNSTestNamespace, instance, owner, attachment, "record") + _, err := reconciler.reconcileAttachment(context.Background(), access, InternalDNSProjectIdentity{ProjectUID: internalDNSTestProjectUID}, internalDNSTestNamespace, instance, owner, attachment, "record") require.NoError(t, err) require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(contribution), contribution)) eligible, _, _ = unstructured.NestedBool(contribution.Object, internalDNSSpecField, internalDNSEligibleField) diff --git a/internal/controller/internaldns_publisher_test.go b/internal/controller/internaldns_publisher_test.go index bc821475..f631f82a 100644 --- a/internal/controller/internaldns_publisher_test.go +++ b/internal/controller/internaldns_publisher_test.go @@ -20,6 +20,7 @@ import ( ) const ( + internalDNSTestProjectUID = "project-a" internalDNSTestNamespace = "dns-test" internalDNSTestDefaultDomain = "datum.internal" internalDNSTestInstanceName = "api" diff --git a/internal/controller/internaldns_status_test.go b/internal/controller/internaldns_status_test.go index 8a2dfa9c..8feeacf4 100644 --- a/internal/controller/internaldns_status_test.go +++ b/internal/controller/internaldns_status_test.go @@ -41,7 +41,7 @@ func TestPublicationStatusRequiresCurrentAuthorizedPublication(t *testing.T) { s := runtime.NewScheme() s.AddKnownTypeWithName(dnsResolverContextGVK, &unstructured.Unstructured{}) s.AddKnownTypeWithName(dnsResolverContextGVK.GroupVersion().WithKind("DNSResolverContextList"), &unstructured.UnstructuredList{}) - resolver := resolverContextObject("context", "project-a", internalDNSTestVPCUID, true) + resolver := resolverContextObject("context", internalDNSTestProjectUID, internalDNSTestVPCUID, true) require.NoError(t, unstructured.SetNestedField(resolver.Object, tc.domain, internalDNSStatusField, "managedNamespace", "suffix")) s.AddKnownTypeWithName(dnsRegistrationGVK, &unstructured.Unstructured{}) s.AddKnownTypeWithName(dnsContributionGVK, &unstructured.Unstructured{}) @@ -68,7 +68,7 @@ func TestPublicationStatusRequiresCurrentAuthorizedPublication(t *testing.T) { contribution.Object[internalDNSStatusField] = map[string]any{internalDNSConditionsField: conditions} cl := fake.NewClientBuilder().WithScheme(s).WithObjects(resolver, registration, contribution).Build() r := &InternalDNSPublisherReconciler{Now: func() time.Time { return now }} - got, err := r.publicationStatus(context.Background(), cl, InternalDNSProjectIdentity{ProjectUID: "project-a"}, internalDNSTestNamespace, instance, instanceDNSAttachment{VPCUID: internalDNSTestVPCUID, Eligible: true}, internalDNSTestInstanceName, tc.observed, nil) + got, err := r.publicationStatus(context.Background(), cl, InternalDNSProjectIdentity{ProjectUID: internalDNSTestProjectUID}, internalDNSTestNamespace, instance, instanceDNSAttachment{VPCUID: internalDNSTestVPCUID, Eligible: true}, internalDNSTestInstanceName, tc.observed, nil) require.NoError(t, err) if tc.foreignZone { require.Empty(t, got.Hostnames) From de544965e6f708dded6a050b776f30427f91b5a4 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 15:47:36 -0500 Subject: [PATCH 30/33] Discover internal DNS project identities through Milo --- cmd/main.go | 36 ++++--------- cmd/main_test.go | 10 ---- .../internal-dns-instance-publishing.md | 6 +-- internal/config/config.go | 9 ---- internal/config/config_test.go | 7 --- internal/config/zz_generated.deepcopy.go | 20 ------- internal/controller/internaldns_publisher.go | 38 ++++++++++++-- .../controller/internaldns_publisher_test.go | 52 +++++++++++++++++++ 8 files changed, 96 insertions(+), 82 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index cdb12253..df96733a 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -454,9 +454,14 @@ func main() { "InternalDNSPublishing requires Milo discovery mode and management controllers") os.Exit(1) } - identities, identityErr := internalDNSProjectIdentities(serverConfig.InternalDNS) + identityConfig, identityErr := serverConfig.Discovery.DiscoveryRestConfig() if identityErr != nil { - setupLog.Error(identityErr, "invalid internal DNS publisher configuration") + setupLog.Error(identityErr, "unable to load internal DNS project discovery credentials") + os.Exit(1) + } + identityClient, identityErr := client.New(identityConfig, client.Options{Scheme: scheme}) + if identityErr != nil { + setupLog.Error(identityErr, "unable to create internal DNS project discovery client") os.Exit(1) } observers, observationErr := internalDNSObservationSources(serverConfig.InternalDNS) @@ -465,8 +470,8 @@ func main() { os.Exit(1) } dnsPublisher := &controller.InternalDNSPublisherReconciler{ - ProjectIdentities: identities, - ObservationSources: observers, + ResolveProjectIdentity: controller.InternalDNSProjectIdentityResolver(identityClient), + ObservationSources: observers, } if err = dnsPublisher.SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "InternalDNSPublisher") @@ -999,26 +1004,3 @@ func internalDNSObservationSources( } return sources, nil } - -func internalDNSProjectIdentities( - cfg config.InternalDNSConfig, -) (map[string]controller.InternalDNSProjectIdentity, error) { - identities := make(map[string]controller.InternalDNSProjectIdentity, len(cfg.Projects)) - for i, project := range cfg.Projects { - if project.Name == "" || project.ProjectUID == "" { - return nil, fmt.Errorf("internalDNS.projects[%d] requires name and projectUID", i) - } - if _, duplicate := identities[project.Name]; duplicate { - return nil, fmt.Errorf("internalDNS.projects contains duplicate project %q", project.Name) - } - identities[project.Name] = controller.InternalDNSProjectIdentity{ - ProjectName: project.Name, - ProjectUID: project.ProjectUID, - } - } - if len(identities) == 0 { - return nil, errors.New("internalDNS.projects must contain at least one project") - } - - return identities, nil -} diff --git a/cmd/main_test.go b/cmd/main_test.go index 1892d2fb..aff5b53d 100644 --- a/cmd/main_test.go +++ b/cmd/main_test.go @@ -23,16 +23,6 @@ import ( "go.datum.net/compute/internal/locations" ) -func TestInternalDNSProjectIdentities(t *testing.T) { - identities, err := internalDNSProjectIdentities(config.InternalDNSConfig{ - Projects: []config.InternalDNSProjectIdentity{{Name: "project-a", ProjectUID: "project-uid"}}, - }) - require.NoError(t, err) - assert.EqualValues(t, "project-uid", identities["project-a"].ProjectUID) - _, err = internalDNSProjectIdentities(config.InternalDNSConfig{}) - assert.Error(t, err) -} - func TestInternalDNSObservationSourcesRequireExplicitCredentials(t *testing.T) { _, err := internalDNSObservationSources(config.InternalDNSConfig{}) require.Error(t, err) diff --git a/docs/enhancements/internal-dns-instance-publishing.md b/docs/enhancements/internal-dns-instance-publishing.md index 51747358..b450bb99 100644 --- a/docs/enhancements/internal-dns-instance-publishing.md +++ b/docs/enhancements/internal-dns-instance-publishing.md @@ -25,15 +25,13 @@ status: ``` Enable the default-off `InternalDNSPublishing` feature gate after provisioning -DNS, project permissions, and read-only edge observation credentials. +DNS, project permissions, and read-only edge observation credentials. Compute +resolves project identities through Milo discovery, including newly created projects. ```yaml apiVersion: apiserver.config.datumapis.com/v1alpha1 kind: WorkloadOperator internalDNS: - projects: - - name: example-project - projectUID: 00000000-0000-0000-0000-000000000001 observationSources: - location: dfw # Authenticate to the edge API that owns instance observations. diff --git a/internal/config/config.go b/internal/config/config.go index d993fcf2..f5411d0f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -59,9 +59,6 @@ type WorkloadOperator struct { // +k8s:deepcopy-gen=true type InternalDNSConfig struct { - // Projects identifies the consumer project APIs. - Projects []InternalDNSProjectIdentity `json:"projects,omitempty"` - // ObservationSources selects authenticated edge APIs for live interface and // node observations. Credentials must have read-only observation permissions. ObservationSources []InternalDNSObservationSource `json:"observationSources,omitempty"` @@ -73,12 +70,6 @@ type InternalDNSObservationSource struct { KubeconfigPath string `json:"kubeconfigPath"` } -// +k8s:deepcopy-gen=true -type InternalDNSProjectIdentity struct { - Name string `json:"name"` - ProjectUID types.UID `json:"projectUID"` -} - func SetDefaults_WorkloadOperator(obj *WorkloadOperator) { if obj.LocationSource == "" { obj.LocationSource = locations.SourceNetworkServices diff --git a/internal/config/config_test.go b/internal/config/config_test.go index cd5981d8..98957865 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -38,19 +38,12 @@ internalDNS: observationSources: - location: dfw kubeconfigPath: /etc/compute/edge.kubeconfig - projects: - - name: project-a - projectUID: project-uid `) copy := cfg.DeepCopy() - copy.InternalDNS.Projects[0].ProjectUID = "changed" copy.InternalDNS.ObservationSources[0].Location = "changed" if cfg.InternalDNS.ObservationSources[0].Location != "dfw" { t.Fatal("WorkloadOperator DeepCopy aliases observation identities") } - if cfg.InternalDNS.Projects[0].ProjectUID != "project-uid" { - t.Fatal("WorkloadOperator DeepCopy aliases internal DNS project identities") - } } func TestWebhookServer_Absent(t *testing.T) { diff --git a/internal/config/zz_generated.deepcopy.go b/internal/config/zz_generated.deepcopy.go index 5928a9ad..62fc8f11 100644 --- a/internal/config/zz_generated.deepcopy.go +++ b/internal/config/zz_generated.deepcopy.go @@ -69,11 +69,6 @@ func (in *FeatureFlagsConfig) DeepCopy() *FeatureFlagsConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InternalDNSConfig) DeepCopyInto(out *InternalDNSConfig) { *out = *in - if in.Projects != nil { - in, out := &in.Projects, &out.Projects - *out = make([]InternalDNSProjectIdentity, len(*in)) - copy(*out, *in) - } if in.ObservationSources != nil { in, out := &in.ObservationSources, &out.ObservationSources *out = make([]InternalDNSObservationSource, len(*in)) @@ -106,21 +101,6 @@ func (in *InternalDNSObservationSource) DeepCopy() *InternalDNSObservationSource return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *InternalDNSProjectIdentity) DeepCopyInto(out *InternalDNSProjectIdentity) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InternalDNSProjectIdentity. -func (in *InternalDNSProjectIdentity) DeepCopy() *InternalDNSProjectIdentity { - if in == nil { - return nil - } - out := new(InternalDNSProjectIdentity) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MetricsServerConfig) DeepCopyInto(out *MetricsServerConfig) { *out = *in diff --git a/internal/controller/internaldns_publisher.go b/internal/controller/internaldns_publisher.go index 5a966d14..f66874aa 100644 --- a/internal/controller/internaldns_publisher.go +++ b/internal/controller/internaldns_publisher.go @@ -31,6 +31,7 @@ import ( computev1alpha "go.datum.net/compute/api/v1alpha" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + resourcemanagerv1alpha1 "go.miloapis.com/milo/pkg/apis/resourcemanager/v1alpha1" ) const ( @@ -103,6 +104,7 @@ type InternalDNSPublisherReconciler struct { ProjectIDForInstance func(context.Context, multicluster.ClusterName, *computev1alpha.Instance) (string, error) ProjectNamespaceForInstance func(context.Context, multicluster.ClusterName, *computev1alpha.Instance) (string, error) ProjectAccess InternalDNSProjectAccessFunc + ResolveProjectIdentity func(context.Context, string) (InternalDNSProjectIdentity, error) ProjectIdentities map[string]InternalDNSProjectIdentity MaxConcurrentReconciles int Now func() time.Time @@ -132,9 +134,9 @@ func (r *InternalDNSPublisherReconciler) Reconcile(ctx context.Context, req mcre if err != nil { return ctrl.Result{}, fmt.Errorf("resolve DNS project namespace: %w", err) } - identity, ok := r.ProjectIdentities[projectID] - if !ok || identity.ProjectUID == "" { - return ctrl.Result{}, fmt.Errorf("internal DNS identity is not configured for project %q", projectID) + identity, err := r.projectIdentity(ctx, projectID) + if err != nil { + return ctrl.Result{}, err } access, err := r.projectAccess(ctx, projectID, providerCluster) if err != nil { @@ -533,9 +535,35 @@ func (r *InternalDNSPublisherReconciler) projectAccess(ctx context.Context, proj return InternalDNSProjectAccess{Reader: providerCluster.GetAPIReader(), Writer: providerCluster.GetClient()}, nil } +// InternalDNSProjectIdentityResolver reads the current project identity from Milo. +// Unavailable or terminating projects stop publication without withdrawing records. +func InternalDNSProjectIdentityResolver(reader client.Reader) func(context.Context, string) (InternalDNSProjectIdentity, error) { + return func(ctx context.Context, projectName string) (InternalDNSProjectIdentity, error) { + var project resourcemanagerv1alpha1.Project + if err := reader.Get(ctx, client.ObjectKey{Name: projectName}, &project); err != nil { + return InternalDNSProjectIdentity{}, fmt.Errorf("read DNS project %q: %w", projectName, err) + } + if project.UID == "" || !project.DeletionTimestamp.IsZero() { + return InternalDNSProjectIdentity{}, fmt.Errorf("DNS project %q has no active identity", projectName) + } + return InternalDNSProjectIdentity{ProjectName: project.Name, ProjectUID: project.UID}, nil + } +} + +func (r *InternalDNSPublisherReconciler) projectIdentity(ctx context.Context, projectName string) (InternalDNSProjectIdentity, error) { + if r.ResolveProjectIdentity != nil { + return r.ResolveProjectIdentity(ctx, projectName) + } + identity, ok := r.ProjectIdentities[projectName] + if !ok || identity.ProjectUID == "" { + return InternalDNSProjectIdentity{}, fmt.Errorf("internal DNS identity is not configured for project %q", projectName) + } + return identity, nil +} + func (r *InternalDNSPublisherReconciler) SetupWithManager(mgr mcmanager.Manager) error { - if len(r.ProjectIdentities) == 0 { - return errors.New("at least one internal DNS project identity is required") + if r.ResolveProjectIdentity == nil && len(r.ProjectIdentities) == 0 { + return errors.New("an internal DNS project identity resolver is required") } r.mgr = mgr concurrency := r.MaxConcurrentReconciles diff --git a/internal/controller/internaldns_publisher_test.go b/internal/controller/internaldns_publisher_test.go index f631f82a..b1d1c157 100644 --- a/internal/controller/internaldns_publisher_test.go +++ b/internal/controller/internaldns_publisher_test.go @@ -4,6 +4,7 @@ package controller import ( "context" + "errors" "testing" "github.com/stretchr/testify/assert" @@ -17,6 +18,7 @@ import ( computev1alpha "go.datum.net/compute/api/v1alpha" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + resourcemanagerv1alpha1 "go.miloapis.com/milo/pkg/apis/resourcemanager/v1alpha1" ) const ( @@ -173,3 +175,53 @@ func TestEnsureDNSObjectCreatesProjectInstanceOwnershipWithoutDeleteBlock(t *tes assert.Nil(t, created.GetOwnerReferences()[0].BlockOwnerDeletion, "publisher must not require delete permission on the owning Instance") } + +func TestDNSProjectIdentityDiscoversNewProjectsAndReplacementIdentities(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, resourcemanagerv1alpha1.AddToScheme(scheme)) + cl := fake.NewClientBuilder().WithScheme(scheme).Build() + resolve := InternalDNSProjectIdentityResolver(cl) + _, err := resolve(context.Background(), "project-b") + require.Error(t, err) + + project := &resourcemanagerv1alpha1.Project{ObjectMeta: metav1.ObjectMeta{Name: "project-b", UID: "project-b-first"}} + require.NoError(t, cl.Create(context.Background(), project)) + identity, err := resolve(context.Background(), project.Name) + require.NoError(t, err) + assert.Equal(t, InternalDNSProjectIdentity{ProjectName: project.Name, ProjectUID: project.UID}, identity) + + require.NoError(t, cl.Delete(context.Background(), project)) + project.ResourceVersion = "" + project.UID = "project-b-second" + require.NoError(t, cl.Create(context.Background(), project)) + identity, err = resolve(context.Background(), project.Name) + require.NoError(t, err) + assert.Equal(t, project.UID, identity.ProjectUID) +} + +func TestDNSProjectIdentityRejectsInactiveProjectsAndReadFailures(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, resourcemanagerv1alpha1.AddToScheme(scheme)) + now := metav1.Now() + for _, project := range []*resourcemanagerv1alpha1.Project{ + {ObjectMeta: metav1.ObjectMeta{Name: "project-b"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "project-b", UID: "project-b", DeletionTimestamp: &now, Finalizers: []string{"test.datum.net/cleanup"}}}, + } { + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(project).Build() + _, err := InternalDNSProjectIdentityResolver(cl)(context.Background(), project.Name) + require.Error(t, err) + } + unavailable := errors.New("project API unavailable") + reader := &unavailableDNSProjectReader{err: unavailable} + _, err := InternalDNSProjectIdentityResolver(reader)(context.Background(), "project-b") + require.ErrorIs(t, err, unavailable) +} + +type unavailableDNSProjectReader struct { + client.Reader + err error +} + +func (r *unavailableDNSProjectReader) Get(context.Context, client.ObjectKey, client.Object, ...client.GetOption) error { + return r.err +} From afc1cf9c14ca5a61a1e008bbfb2bc65ff32b75ae Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 15:49:16 -0500 Subject: [PATCH 31/33] Name the project discovery test fixture --- .../controller/internaldns_publisher_test.go | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/internal/controller/internaldns_publisher_test.go b/internal/controller/internaldns_publisher_test.go index b1d1c157..cc3d32a8 100644 --- a/internal/controller/internaldns_publisher_test.go +++ b/internal/controller/internaldns_publisher_test.go @@ -22,18 +22,19 @@ import ( ) const ( - internalDNSTestProjectUID = "project-a" - internalDNSTestNamespace = "dns-test" - internalDNSTestDefaultDomain = "datum.internal" - internalDNSTestInstanceName = "api" - internalDNSTestInstanceUID = "dns-instance-uid" - internalDNSTestEdgeUID = "dns-edge-instance" - internalDNSTestForeignValue = "dns-unauthorized" - internalDNSTestSubject = "system:serviceaccount:dns-test:publisher" - internalDNSTestObservationNamespace = "dns-observation-project" - internalDNSTestNetworkName = "dns-application" - internalDNSTestVPCUID = "dns-test-vpc" - internalDNSTestSourceAPIUID = "dns-project-api" + internalDNSTestDiscoveredProjectName = "project-b" + internalDNSTestProjectUID = "project-a" + internalDNSTestNamespace = "dns-test" + internalDNSTestDefaultDomain = "datum.internal" + internalDNSTestInstanceName = "api" + internalDNSTestInstanceUID = "dns-instance-uid" + internalDNSTestEdgeUID = "dns-edge-instance" + internalDNSTestForeignValue = "dns-unauthorized" + internalDNSTestSubject = "system:serviceaccount:dns-test:publisher" + internalDNSTestObservationNamespace = "dns-observation-project" + internalDNSTestNetworkName = "dns-application" + internalDNSTestVPCUID = "dns-test-vpc" + internalDNSTestSourceAPIUID = "dns-project-api" ) func TestInstanceDNSAttachmentsMapsTenancyAndAddressFamilies(t *testing.T) { @@ -98,7 +99,7 @@ func TestFindResolverContextRequiresProjectAndVPCUID(t *testing.T) { require.NotNil(t, got) assert.Equal(t, "right", got.GetName()) - got, err = findResolverContext(context.Background(), cl, internalDNSTestNamespace, "project-b", internalDNSTestVPCUID) + got, err = findResolverContext(context.Background(), cl, internalDNSTestNamespace, internalDNSTestDiscoveredProjectName, internalDNSTestVPCUID) require.NoError(t, err) assert.Nil(t, got) } @@ -181,10 +182,10 @@ func TestDNSProjectIdentityDiscoversNewProjectsAndReplacementIdentities(t *testi require.NoError(t, resourcemanagerv1alpha1.AddToScheme(scheme)) cl := fake.NewClientBuilder().WithScheme(scheme).Build() resolve := InternalDNSProjectIdentityResolver(cl) - _, err := resolve(context.Background(), "project-b") + _, err := resolve(context.Background(), internalDNSTestDiscoveredProjectName) require.Error(t, err) - project := &resourcemanagerv1alpha1.Project{ObjectMeta: metav1.ObjectMeta{Name: "project-b", UID: "project-b-first"}} + project := &resourcemanagerv1alpha1.Project{ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestDiscoveredProjectName, UID: "project-b-first"}} require.NoError(t, cl.Create(context.Background(), project)) identity, err := resolve(context.Background(), project.Name) require.NoError(t, err) @@ -204,8 +205,8 @@ func TestDNSProjectIdentityRejectsInactiveProjectsAndReadFailures(t *testing.T) require.NoError(t, resourcemanagerv1alpha1.AddToScheme(scheme)) now := metav1.Now() for _, project := range []*resourcemanagerv1alpha1.Project{ - {ObjectMeta: metav1.ObjectMeta{Name: "project-b"}}, - {ObjectMeta: metav1.ObjectMeta{Name: "project-b", UID: "project-b", DeletionTimestamp: &now, Finalizers: []string{"test.datum.net/cleanup"}}}, + {ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestDiscoveredProjectName}}, + {ObjectMeta: metav1.ObjectMeta{Name: internalDNSTestDiscoveredProjectName, UID: internalDNSTestDiscoveredProjectName, DeletionTimestamp: &now, Finalizers: []string{"test.datum.net/cleanup"}}}, } { cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(project).Build() _, err := InternalDNSProjectIdentityResolver(cl)(context.Background(), project.Name) @@ -213,7 +214,7 @@ func TestDNSProjectIdentityRejectsInactiveProjectsAndReadFailures(t *testing.T) } unavailable := errors.New("project API unavailable") reader := &unavailableDNSProjectReader{err: unavailable} - _, err := InternalDNSProjectIdentityResolver(reader)(context.Background(), "project-b") + _, err := InternalDNSProjectIdentityResolver(reader)(context.Background(), internalDNSTestDiscoveredProjectName) require.ErrorIs(t, err, unavailable) } From c9adfc47504bdc8683cad7a39841ffbcdc3b793a Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 14 Oct 2026 16:17:39 -0500 Subject: [PATCH 32/33] docs: Describe current Compute feature gates Keep DNS publication and native Pod attachment help accurate. --- cmd/main.go | 2 +- internal/features/features.go | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 7b6a8a3e..769fa0f3 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -159,7 +159,7 @@ func main() { flag.StringVar(&featureGatesFlag, "feature-gates", "", "A set of key=value pairs that describe feature gates for the compute operator. "+ "Example: --feature-gates=NetworkingIntegration=true. "+ - "Available features: NetworkingIntegration, RuntimeClasses, InstanceTypes, InstanceConsoleSessions, "+ + "Available features: NetworkingIntegration, PodNetworkAttachment, RuntimeClasses, InstanceTypes, InstanceConsoleSessions, "+ "InternalDNSPublishing (all default=false).") opts := zap.Options{ diff --git a/internal/features/features.go b/internal/features/features.go index 8cd80061..154cad18 100644 --- a/internal/features/features.go +++ b/internal/features/features.go @@ -113,13 +113,7 @@ const ( // alpha: v0.1 InstanceConsoleSessions featuregate.Feature = "InstanceConsoleSessions" - // InternalDNSPublishing enables the instance identity publisher. The - // publisher reads DNS contexts and independently issued grants and writes - // registrations and leased contributions to project control planes. - // It is deliberately off by default: an installation with the gate disabled - // neither discovers DNS API types nor needs DNS RBAC. - // - // alpha: v0.1 + // InternalDNSPublishing publishes instance records into project DNS contexts. InternalDNSPublishing featuregate.Feature = "InternalDNSPublishing" ) From db0a0eda3086be9213adedc47650947d8c44dee7 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 14 Oct 2026 16:24:58 -0500 Subject: [PATCH 33/33] fix: wrap internal DNS feature help --- cmd/main.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/main.go b/cmd/main.go index 769fa0f3..14341be0 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -159,7 +159,8 @@ func main() { flag.StringVar(&featureGatesFlag, "feature-gates", "", "A set of key=value pairs that describe feature gates for the compute operator. "+ "Example: --feature-gates=NetworkingIntegration=true. "+ - "Available features: NetworkingIntegration, PodNetworkAttachment, RuntimeClasses, InstanceTypes, InstanceConsoleSessions, "+ + "Available features: NetworkingIntegration, PodNetworkAttachment, RuntimeClasses, InstanceTypes, "+ + "InstanceConsoleSessions, "+ "InternalDNSPublishing (all default=false).") opts := zap.Options{