test: Validate internal DNS on the shared Kubernetes environment

.github/workflows/e2e.yml

@@ -13,7 +13,7 @@ jobs:
     # Three test-infra kind clusters (upstream + control + edge), each with
     # Flux/cert-manager/Kyverno/Envoy Gateway, plus RustFS and a Lightningstream
     # sync interval the federation/full-chain suites poll for - give it headroom.
-    timeout-minutes: 60
+    timeout-minutes: 90
     env:
       # The Taskfile pulls in the shared datum-cloud/test-infra Taskfile as a
       # remote include; go-task gates that behind this experiment flag, and
@@ -58,13 +58,21 @@ jobs:
           make chainsaw
           echo "$GITHUB_WORKSPACE/bin" >> "$GITHUB_PATH"
 
+      - name: Bring up private DNS on the shared Kubernetes foundation
+        run: |
+          go test ./test/internaldns/kubernetes/...
+          task --yes env:internal-dns:up
+
       - name: Bring up the full DNS environment
         # Creates the upstream (replicator) + control (downstream agent +
         # PowerDNS + Lightningstream + RustFS) + edge clusters and wires the
         # whole production chain together. No sample data (env:stack-up, not
         # env:up) so the suites own all zones on the shared control PowerDNS.
         run: task --yes env:stack-up
 
+      - name: Qualify private DNS publication and serving
+        run: task --yes env:internal-dns:qualify
+
       - name: Run Chainsaw tests
         # Runs every suite under test/e2e against the environment: the
         # upstream->control replicator suites (alias, display-annotations,
@@ -100,6 +108,18 @@ jobs:
           echo "=== RustFS logs (control) ==="
           kubectl --context kind-dns-control -n rustfs-system logs -l app.kubernetes.io/instance=rustfs --tail=100 || true
 
+      - name: Collect private DNS deployment evidence
+        if: always()
+        run: test/internaldns/kubernetes/collect.sh
+
+      - name: Retain private DNS qualification evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: internal-dns-kubernetes
+          path: test/internaldns/results/kubernetes*
+          if-no-files-found: warn
+
       - name: Tear down the environment
         if: always()
         run: task --yes env:down || true

.github/workflows/internal-dns-package.yml

@@ -0,0 +1,26 @@
+name: Internal DNS package
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  package:
+    name: Qualify internal DNS runtime package
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - name: Build and qualify the runtime package
+        run: test/internaldns/qualify-package.sh
+      - name: Retain package evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: internal-dns-package
+          path: test/internaldns/results/package
+          if-no-files-found: warn

.gitignore

@@ -50,3 +50,6 @@ go.work.sum
 .claude/settings.local.json
 .claude/worktrees/
 .mcp.json
+
+/.internal-dns-e2e/
+/test/internaldns/results/

Taskfile.yaml

@@ -11,9 +11,9 @@
 version: '3'
 
 vars:
-  # Bump to the merged tag once datum-cloud/test-infra#35 lands (it removes
-  # the hardcoded NodePorts that block running two clusters concurrently).
-  TEST_INFRA_REPO_REF: 'fix/remove-hardcoded-nodeports'
+  # Shared bootstrap with dynamic NodePorts and an immutable revision for
+  # consistent qualification across environments.
+  TEST_INFRA_REPO_REF: '7622c925d5bcf2fa5bc01823775973cf6daf51ca'
   # Upstream (control-plane) cluster runs the replicator; it pushes DNSZone/
   # DNSRecordSet resources to the control cluster, which plays the "downstream"
   # agent role for the replicator and the "control" (Lightningstream source)
@@ -56,6 +56,36 @@ includes:
       REPO_REF: "{{.TEST_INFRA_REPO_REF}}"
 
 tasks:
+  env:internal-dns:up:
+    desc: "Deploy private DNS pods on the shared upstream/control/edge environment"
+    cmds:
+      - task: test-infra:ensure-repo
+      - task: test-infra:ensure-tools
+      - test/internaldns/kubernetes/prepare-kind.sh
+      - task: env:test-infra-cluster-up
+        vars:
+          CLUSTER_NAME: "{{.CONTROL_CLUSTER_NAME}}"
+          KIND_CFG: .internal-dns-e2e/kubernetes/kind-config.yaml
+      - task: env:test-infra-cluster-up
+        vars:
+          CLUSTER_NAME: "{{.UPSTREAM_CLUSTER_NAME}}"
+          KIND_CFG: .internal-dns-e2e/kubernetes/kind-config.yaml
+      - task: env:test-infra-cluster-up
+        vars:
+          CLUSTER_NAME: "{{.EDGE_CLUSTER_NAME}}"
+          KIND_CFG: .internal-dns-e2e/kubernetes/kind-config.yaml
+      - test/internaldns/kubernetes/deploy.sh
+
+  env:internal-dns:qualify:
+    desc: "Qualify private DNS through scoped product APIs and Kubernetes serving pods"
+    cmds:
+      - go run ./test/internaldns/kubernetes/qualify --environment .internal-dns-e2e/kubernetes/environment.json --results test/internaldns/results/kubernetes.json
+
+  env:internal-dns:package:
+    desc: "Build and qualify the internal DNS runtime image and deployment contracts"
+    cmds:
+      - test/internaldns/qualify-package.sh
+
   default:
     desc: "Show available commands"
     silent: true
@@ -65,6 +95,8 @@ tasks:
       - echo "  task env:stack-up    Bring up the full chain with no sample data (used by CI)"
       - echo "  task env:chainsaw    Run the full Chainsaw e2e suite against the environment"
       - echo "  task env:down        Tear down all clusters"
+      - echo "  task env:internal-dns:up       Deploy private DNS on the shared clusters"
+      - echo "  task env:internal-dns:qualify  Validate private DNS publication and isolation"
       - echo "  task env:dns-check   Query control + edge PowerDNS for the sample record"
       - echo ""
       - echo "Individual pieces:"
@@ -139,18 +171,8 @@ tasks:
   # internal: brings up a test-infra-managed kind cluster (kind cluster +
   # Flux + cert-manager + Kyverno + Envoy Gateway) for CLUSTER_NAME.
   #
-  # NOTE: this deliberately calls the individual test-infra sub-tasks
-  # (ensure-repo/ensure-tools/create-kind/install-components) instead of the
-  # composite `test-infra:cluster-up`/`up` task. As of test-infra's
-  # fix/remove-hardcoded-nodeports branch (and go-task 3.49/3.50),
-  # `cluster-up`'s trailing informational `echo "... # ..."` lines panic
-  # go-task's command parser ("reached EOF without closing quote") because
-  # go-task strips ` #...` as a shell comment even inside double quotes on
-  # single-line (non `|` block) cmd entries - a go-task parsing quirk, not
-  # a shell issue. Calling the sub-tasks directly gets the same cluster
-  # setup while skipping the broken trailing echoes. If that upstream
-  # Taskfile is fixed (or go-task fixes the quoting bug), this can go back
-  # to a single `task: test-infra:cluster-up` call.
+  # Invoke bootstrap sub-tasks directly to pass cluster variables explicitly
+  # and avoid the composite task's informational command parsing errors.
   env:test-infra-cluster-up:
     internal: true
     cmds:
@@ -163,6 +185,7 @@ tasks:
       - task: test-infra:create-kind
         vars:
           CLUSTER_NAME: "{{.CLUSTER_NAME}}"
+          KIND_CFG: '{{.KIND_CFG | default ".test-infra/cluster/kind-config.yaml"}}'
       - task: test-infra:install-components
         vars:
           CLUSTER_NAME: "{{.CLUSTER_NAME}}"

config/crd/kustomization.yaml

@@ -9,15 +9,25 @@ resources:
 
 
 - bases/dns.networking.miloapis.com_dnsnamingpolicies.yaml
+
 - bases/dns.networking.miloapis.com_dnsrecordcontributions.yaml
+
 - bases/dns.networking.miloapis.com_dnsregistrations.yaml
+
 - bases/dns.networking.miloapis.com_dnsresolveraccessbindings.yaml
+
 - bases/dns.networking.miloapis.com_dnsresolvercontexts.yaml
+
 - bases/dns.networking.miloapis.com_dnszoneassociations.yaml
+
 - bases/dns.networking.miloapis.com_dnspublicationchunks.yaml
+
 - bases/dns.networking.miloapis.com_dnspublicationmanifests.yaml
+
 - bases/dns.networking.miloapis.com_dnspublicationownerships.yaml
+
 - bases/dns.networking.miloapis.com_dnsresolverbindings.yaml
+
 - bases/dns.networking.miloapis.com_dnstransportoutboxes.yaml
 # +kubebuilder:scaffold:crdkustomizeresource
 

internal/internaldns/model/model_test.go

@@ -91,3 +91,24 @@ func TestEnvelopeRejectsObsoleteWireVersion(t *testing.T) {
 		t.Fatal("obsolete transport version accepted")
 	}
 }
+
+func TestPendingConfigurationCannotExposePartialZoneAttachments(t *testing.T) {
+	now := time.Now().UTC()
+	binding := Binding{BindingUID: "binding", ProjectUID: "project", ContextUID: "context", BindingGeneration: 1, ConfigurationRevision: 1, ConsumerAddress: "fd53::a", ClusterAddress: "fd54::a", Port: 53, Transports: []Transport{TransportUDP}, Version: AccessVersion{Epoch: 1, Sequence: 1}, ConfigurationPending: true, Zones: []ZoneAttachment{{ZoneUID: "zone", Apex: "private.internal", RequiredPublicationEpoch: 1, RequiredPublicationRevision: 1}}}
+	snapshot := ServingSnapshot{Region: "east", Shard: "shared", ConfigurationEpoch: 1, ConfigurationRevision: 1, GeneratedAt: now, Bindings: []Binding{binding}}
+	if err := snapshot.Validate(); err == nil {
+		t.Fatal("partial pending context accepted")
+	}
+	snapshot.Bindings[0].Zones = nil
+	if err := snapshot.Validate(); err != nil {
+		t.Fatal(err)
+	}
+	event, err := NewEnvelope(KindServingSnapshot, "pending", "east", "shared", "shard", 1, 1, now, snapshot)
+	if err != nil {
+		t.Fatal(err)
+	}
+	event.APIVersion = "internal.dns.networking.miloapis.com/v1alpha2"
+	if err := event.Validate(); err == nil {
+		t.Fatal("pre-pending wire format accepted")
+	}
+}

internal/internaldns/runtime/deployment_permissions_test.go

@@ -161,6 +161,18 @@ func TestEveryMemberCanCreateAllRuntimeConsumers(t *testing.T) {
 		t.Fatal(err)
 	}
 	contents := string(data)
+	start := strings.Index(contents, `user: "internal-dns-control-plane"`)
+	if start < 0 {
+		t.Fatal("missing control-plane broker identity")
+	}
+	controlPolicy := strings.SplitN(contents[start:], "\n    },", 2)[0]
+	controlACK := "$JS.ACK.DNS_PRIVATE." + durable(roleControlPlane, "us-central1", "shared-0", "acks") + ".>"
+	if !strings.Contains(controlPolicy, `"`+controlACK+`"`) {
+		t.Errorf("control plane cannot acknowledge its member-ACK consumer: %s", controlACK)
+	}
+	if strings.Contains(controlPolicy, `"$JS.ACK.DNS_PRIVATE.>"`) {
+		t.Error("control plane can acknowledge another serving member's consumers")
+	}
 	for _, member := range []string{"node-us-central1-0", "regional-front-us-central1-0", "regional-coredns-us-central1-0", "regional-coredns-us-central1-1"} {
 		start := strings.Index(contents, `user: "`+member+`"`)
 		if start < 0 {

internal/internaldns/serving/verify_test.go

@@ -122,7 +122,7 @@ func TestPublicationVerifierProvesMemberContextAndFingerprintOverBothTransports(
 					return
 				}
 				defer func() { _ = conn.Close() }()
-				header := make([]byte, 28)
+				header := make([]byte, 28, 30)
 				if _, err := io.ReadFull(conn, header); err != nil {
 					return
 				}

test/internaldns/README.md

@@ -0,0 +1,119 @@
+# Internal DNS end-to-end qualification
+
+Run private DNS on the repository's shared Kubernetes test environment. Task
+reuses the pinned `datum-cloud/test-infra` bootstrap, cluster names, add-ons, and
+cleanup. The environment also supports the existing public DNS regression chain.
+The qualification uses Go clients and Kubernetes workloads.
+
+## Run locally
+
+Install Docker, Kind, kubectl, Go, and Task. On macOS, select the suite-owned
+Colima runtime before starting the environment:
+
+```sh
+colima --profile internal-dns-e2e start --activate=false --cpu 6 --memory 16 --disk 60
+source test/internaldns/colima-env.sh
+export TASK_X_REMOTE_TASKFILES=1
+task --yes env:internal-dns:up
+task --yes env:internal-dns:qualify
+```
+
+`env:internal-dns:up` resets only the owned private DNS fixtures, then rebuilds
+and deploys them on the existing clusters. Run it again before repeating the
+qualification. It retains the shared bootstrap and public DNS workloads.
+
+Start private DNS first on fresh clusters to select the shared dual-stack Kind
+configuration. If an existing public-only environment uses IPv4, the preflight
+stops before resetting private fixtures. When ready to replace that environment,
+run `env:down`, then start `env:internal-dns:up` before `env:stack-up`.
+
+To validate public DNS alongside private DNS, use the same lifecycle as CI:
+
+```sh
+task --yes env:internal-dns:up
+task --yes env:stack-up
+task --yes env:internal-dns:qualify
+task --yes env:chainsaw
+test/internaldns/kubernetes/collect.sh
+task --yes env:down
+```
+
+`env:down` removes the three shared test clusters. Qualification results and
+sanitized deployment logs remain under `test/internaldns/results/kubernetes*`.
+Generated kubeconfigs and Secrets stay in `.internal-dns-e2e/kubernetes/` and
+are excluded from collected evidence. The qualifier restores controllers and
+broker replicas after fault injection, including a failed scenario.
+
+## Deployed path
+
+```mermaid
+flowchart LR
+  A[Project A API / scoped publisher] --> C[Two DNS controller Deployments]
+  B[Project B API / scoped publisher] --> C
+  C --> J[Three TLS JetStream replicas / PVCs]
+  J --> N[Shared node dnsdist / CoreDNS]
+  J --> R[Shared regional dnsdist / two CoreDNS members]
+  P[Separate consumer probe] -->|Service destination fixture| N --> R
+```
+
+The `dns-upstream` and `dns-edge` clusters provide independent project APIs.
+The `dns-control` cluster hosts the DNS service API, admission Service, broker,
+controllers, and fixed serving fleet. All three use the shared Flux,
+cert-manager, Kyverno, and Envoy bootstrap. Private and public DNS use separate
+namespaces on this foundation.
+
+The manifest renderer reads the shipped runtime configuration and RBAC. Each
+controller uses scoped project worker credentials; product and
+VPC integration credentials are separate. The simulated Compute publisher
+creates registrations and contributions through the project API. It discovers
+managed names from `DNSResolverContext.status.managedNamespace`.
+Contribution writes publish records and eligibility atomically. Qualification
+checks health withdrawal, stale-write conflicts, and DNS-owned status permissions.
+
+The fleet contains one node member, one regional frontend, and two regional
+CoreDNS members, each with independent persistent state. Agents validate and
+activate configuration and prove UDP/TCP publication. Local readiness verifies\napproved files and live daemons while healthy DNS survives agent restarts.
+The broker uses three persistent replicas, client and route TLS, and scoped
+subject permissions. cert-manager issues broker, client, and admission
+certificates. ACK timing matches the shipped 30-second interval and 90-second
+lease; the fixture shortens the ownership lease to 10 seconds and pod
+termination grace to 2 seconds for fault tests.
+
+## Qualification scenarios
+
+The Go qualifier covers overlapping tenant names over UDP and TCP, A/AAAA and
+negative-cache isolation, multiple zones, managed name discovery, scoped
+publication and admission denial, record updates and deletion, health
+withdrawal and recovery, fixed fleet size, regional publication proof,
+controller takeover, continued tenant-isolated UDP/TCP resolution while controllers
+and the broker are stopped, and publication after broker/controller restoration using
+the retained broker PVCs. Each assertion records its result; skipped scenarios do
+not count as passes.
+
+The separate package gate checks the shipped runtime image, all CRD schemas,
+deployment rendering, configuration, and Kubernetes/NATS permissions:
+
+```sh
+task --yes env:internal-dns:package
+```
+
+## Remaining production boundaries
+
+The deployment validates live Kubernetes APIs, admission, credentials, durable
+transport, serving pods, and DNS responses. It does not deploy Milo project
+discovery, Karmada, Galactic private
+service networking, or the actual Compute service. The independent project APIs
+and service-side IPv6 routes are explicit fixtures. Exact route/NAT exemptions
+preserve serving pod peers without widening their ACLs. The probe does not share a
+trusted serving network namespace or receive a Kubernetes credential.
+
+The [production qualification proposal](production-parity.md) covers the next
+stage: queries from attached consumer VPCs through authorized private endpoints
+into the separate DNS service VPC. Kind cannot establish physical fabric,
+microVM attachment, host-loss durability, regional independence, or production
+capacity.
+
+The suite uses wire version `v1alpha6` and persistent format 7 checkpoints.
+Compatible fleet upgrades retain those checkpoints and transport state. Roll out
+serving support before enabling new controller output; see the
+[rollout contract](../../config/internal-dns/README.md#rollout-and-recovery).

test/internaldns/colima-env.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Keep every internal-DNS qualification resource in its suite-owned Colima VM.
+# This deliberately does not activate or change the user's default Docker
+# context. Start the profile separately with:
+#   colima --profile internal-dns-e2e start --activate=false
+
+set -euo pipefail
+
+profile=${INTERNAL_DNS_COLIMA_PROFILE:-internal-dns-e2e}
+socket=${INTERNAL_DNS_DOCKER_SOCKET:-"$HOME/.colima/$profile/docker.sock"}
+expected="unix://$socket"
+
+if [[ -n ${DOCKER_HOST:-} && $DOCKER_HOST != "$expected" ]]; then
+  echo "internal DNS qualification requires DOCKER_HOST=$expected (got $DOCKER_HOST)" >&2
+  exit 2
+fi
+if [[ ! -S $socket ]]; then
+  echo "internal DNS Colima socket is unavailable: $socket" >&2
+  echo "start it with: colima --profile $profile start --activate=false" >&2
+  exit 2
+fi
+
+export INTERNAL_DNS_COLIMA_PROFILE=$profile
+export DOCKER_HOST=$expected