diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 2c0141e0..ac937a90 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -5,6 +5,7 @@ on: branches: - main pull_request: + workflow_dispatch: jobs: e2e: @@ -13,7 +14,7 @@ jobs: # Three test-infra kind clusters (upstream + control + edge), each with # Flux/cert-manager/Kyverno/Envoy Gateway, plus RustFS and a Lightningstream # sync interval the federation/full-chain suites poll for - give it headroom. - timeout-minutes: 60 + timeout-minutes: 90 env: # The Taskfile pulls in the shared datum-cloud/test-infra Taskfile as a # remote include; go-task gates that behind this experiment flag, and @@ -61,6 +62,11 @@ jobs: make chainsaw echo "$GITHUB_WORKSPACE/bin" >> "$GITHUB_PATH" + - name: Bring up private DNS on the shared Kubernetes foundation + run: | + go test ./test/internaldns/kubernetes/... + task --yes env:internal-dns:up + - name: Bring up the full DNS environment # Creates the upstream (replicator) + control (downstream agent + # PowerDNS + Lightningstream + RustFS) + edge clusters and wires the @@ -68,6 +74,9 @@ jobs: # env:up) so the suites own all zones on the shared control PowerDNS. run: task --yes env:stack-up + - name: Qualify private DNS publication and serving + run: task --yes env:internal-dns:qualify + - name: Run Chainsaw tests # Runs every suite under test/e2e against the environment: the # upstream->control replicator suites (alias, display-annotations, @@ -107,6 +116,18 @@ jobs: echo "=== RustFS logs (control) ===" kubectl --context kind-dns-control -n rustfs-system logs -l app.kubernetes.io/instance=rustfs --tail=100 || true + - name: Collect private DNS deployment evidence + if: always() + run: test/internaldns/kubernetes/collect.sh + + - name: Retain private DNS qualification evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: internal-dns-kubernetes + path: test/internaldns/results/kubernetes* + if-no-files-found: warn + - name: Tear down the environment if: always() run: task --yes env:down || true diff --git a/.github/workflows/internal-dns-package.yml b/.github/workflows/internal-dns-package.yml new file mode 100644 index 00000000..18b9c8dc --- /dev/null +++ b/.github/workflows/internal-dns-package.yml @@ -0,0 +1,27 @@ +name: Internal DNS package + +on: + pull_request: + workflow_dispatch: + push: + branches: [main] + +jobs: + package: + name: Qualify internal DNS runtime package + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + - name: Build and qualify the runtime package + run: test/internaldns/qualify-package.sh + - name: Retain package evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: internal-dns-package + path: test/internaldns/results/package + if-no-files-found: warn diff --git a/.gitignore b/.gitignore index b48f9e6e..7f4c1fa4 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,6 @@ go.work.sum .claude/settings.local.json .claude/worktrees/ .mcp.json + +/.internal-dns-e2e/ +/test/internaldns/results/ diff --git a/Taskfile.yaml b/Taskfile.yaml index d8e56347..91136f35 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -11,9 +11,9 @@ version: '3' vars: - # Bump to the merged tag once datum-cloud/test-infra#35 lands (it removes - # the hardcoded NodePorts that block running two clusters concurrently). - TEST_INFRA_REPO_REF: 'fix/remove-hardcoded-nodeports' + # Shared bootstrap with dynamic NodePorts and an immutable revision for + # consistent qualification across environments. + TEST_INFRA_REPO_REF: '7622c925d5bcf2fa5bc01823775973cf6daf51ca' # Upstream (control-plane) cluster runs the replicator; it pushes DNSZone/ # DNSRecordSet resources to the control cluster, which plays the "downstream" # agent role for the replicator and the "control" (Lightningstream source) @@ -56,6 +56,36 @@ includes: REPO_REF: "{{.TEST_INFRA_REPO_REF}}" tasks: + env:internal-dns:up: + desc: "Deploy private DNS pods on the shared upstream/control/edge environment" + cmds: + - task: test-infra:ensure-repo + - task: test-infra:ensure-tools + - test/internaldns/kubernetes/prepare-kind.sh + - task: env:test-infra-cluster-up + vars: + CLUSTER_NAME: "{{.CONTROL_CLUSTER_NAME}}" + KIND_CFG: .internal-dns-e2e/kubernetes/kind-config.yaml + - task: env:test-infra-cluster-up + vars: + CLUSTER_NAME: "{{.UPSTREAM_CLUSTER_NAME}}" + KIND_CFG: .internal-dns-e2e/kubernetes/kind-config.yaml + - task: env:test-infra-cluster-up + vars: + CLUSTER_NAME: "{{.EDGE_CLUSTER_NAME}}" + KIND_CFG: .internal-dns-e2e/kubernetes/kind-config.yaml + - test/internaldns/kubernetes/deploy.sh + + env:internal-dns:qualify: + desc: "Qualify private DNS through scoped product APIs and Kubernetes serving pods" + cmds: + - go run ./test/internaldns/kubernetes/qualify --environment .internal-dns-e2e/kubernetes/environment.json --results test/internaldns/results/kubernetes.json + + env:internal-dns:package: + desc: "Build and qualify the internal DNS runtime image and deployment contracts" + cmds: + - test/internaldns/qualify-package.sh + default: desc: "Show available commands" silent: true @@ -65,6 +95,8 @@ tasks: - echo " task env:stack-up Bring up the full chain with no sample data (used by CI)" - echo " task env:chainsaw Run the full Chainsaw e2e suite against the environment" - echo " task env:down Tear down all clusters" + - echo " task env:internal-dns:up Deploy private DNS on the shared clusters" + - echo " task env:internal-dns:qualify Validate private DNS publication and isolation" - echo " task env:dns-check Query control + edge PowerDNS for the sample record" - echo "" - echo "Individual pieces:" @@ -139,18 +171,8 @@ tasks: # internal: brings up a test-infra-managed kind cluster (kind cluster + # Flux + cert-manager + Kyverno + Envoy Gateway) for CLUSTER_NAME. # - # NOTE: this deliberately calls the individual test-infra sub-tasks - # (ensure-repo/ensure-tools/create-kind/install-components) instead of the - # composite `test-infra:cluster-up`/`up` task. As of test-infra's - # fix/remove-hardcoded-nodeports branch (and go-task 3.49/3.50), - # `cluster-up`'s trailing informational `echo "... # ..."` lines panic - # go-task's command parser ("reached EOF without closing quote") because - # go-task strips ` #...` as a shell comment even inside double quotes on - # single-line (non `|` block) cmd entries - a go-task parsing quirk, not - # a shell issue. Calling the sub-tasks directly gets the same cluster - # setup while skipping the broken trailing echoes. If that upstream - # Taskfile is fixed (or go-task fixes the quoting bug), this can go back - # to a single `task: test-infra:cluster-up` call. + # Invoke bootstrap sub-tasks directly to pass cluster variables explicitly + # and avoid the composite task's informational command parsing errors. env:test-infra-cluster-up: internal: true cmds: @@ -163,6 +185,7 @@ tasks: - task: test-infra:create-kind vars: CLUSTER_NAME: "{{.CLUSTER_NAME}}" + KIND_CFG: '{{.KIND_CFG | default ".test-infra/cluster/kind-config.yaml"}}' - task: test-infra:install-components vars: CLUSTER_NAME: "{{.CLUSTER_NAME}}" diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 7e9375dd..9e5a7df5 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -9,15 +9,25 @@ resources: - bases/dns.networking.miloapis.com_dnsnamingpolicies.yaml + - bases/dns.networking.miloapis.com_dnsrecordcontributions.yaml + - bases/dns.networking.miloapis.com_dnsregistrations.yaml + - bases/dns.networking.miloapis.com_dnsresolveraccessbindings.yaml + - bases/dns.networking.miloapis.com_dnsresolvercontexts.yaml + - bases/dns.networking.miloapis.com_dnszoneassociations.yaml + - bases/dns.networking.miloapis.com_dnspublicationchunks.yaml + - bases/dns.networking.miloapis.com_dnspublicationmanifests.yaml + - bases/dns.networking.miloapis.com_dnspublicationownerships.yaml + - bases/dns.networking.miloapis.com_dnsresolverbindings.yaml + - bases/dns.networking.miloapis.com_dnstransportoutboxes.yaml # +kubebuilder:scaffold:crdkustomizeresource diff --git a/internal/internaldns/model/model_test.go b/internal/internaldns/model/model_test.go index 470101be..783788d6 100644 --- a/internal/internaldns/model/model_test.go +++ b/internal/internaldns/model/model_test.go @@ -91,3 +91,24 @@ func TestEnvelopeRejectsObsoleteWireVersion(t *testing.T) { t.Fatal("obsolete transport version accepted") } } + +func TestPendingConfigurationCannotExposePartialZoneAttachments(t *testing.T) { + now := time.Now().UTC() + binding := Binding{BindingUID: "binding", ProjectUID: "project", ContextUID: "context", BindingGeneration: 1, ConfigurationRevision: 1, ConsumerAddress: "fd53::a", ClusterAddress: "fd54::a", Port: 53, Transports: []Transport{TransportUDP}, Version: AccessVersion{Epoch: 1, Sequence: 1}, ConfigurationPending: true, Zones: []ZoneAttachment{{ZoneUID: "zone", Apex: "private.internal", RequiredPublicationEpoch: 1, RequiredPublicationRevision: 1}}} + snapshot := ServingSnapshot{Region: "east", Shard: "shared", ConfigurationEpoch: 1, ConfigurationRevision: 1, GeneratedAt: now, Bindings: []Binding{binding}} + if err := snapshot.Validate(); err == nil { + t.Fatal("partial pending context accepted") + } + snapshot.Bindings[0].Zones = nil + if err := snapshot.Validate(); err != nil { + t.Fatal(err) + } + event, err := NewEnvelope(KindServingSnapshot, "pending", "east", "shared", "shard", 1, 1, now, snapshot) + if err != nil { + t.Fatal(err) + } + event.APIVersion = "internal.dns.networking.miloapis.com/v1alpha2" + if err := event.Validate(); err == nil { + t.Fatal("pre-pending wire format accepted") + } +} diff --git a/internal/internaldns/runtime/deployment_permissions_test.go b/internal/internaldns/runtime/deployment_permissions_test.go index e47d4536..224d03de 100644 --- a/internal/internaldns/runtime/deployment_permissions_test.go +++ b/internal/internaldns/runtime/deployment_permissions_test.go @@ -161,6 +161,18 @@ func TestEveryMemberCanCreateAllRuntimeConsumers(t *testing.T) { t.Fatal(err) } contents := string(data) + start := strings.Index(contents, `user: "internal-dns-control-plane"`) + if start < 0 { + t.Fatal("missing control-plane broker identity") + } + controlPolicy := strings.SplitN(contents[start:], "\n },", 2)[0] + controlACK := "$JS.ACK.DNS_PRIVATE." + durable(roleControlPlane, "us-central1", "shared-0", "acks") + ".>" + if !strings.Contains(controlPolicy, `"`+controlACK+`"`) { + t.Errorf("control plane cannot acknowledge its member-ACK consumer: %s", controlACK) + } + if strings.Contains(controlPolicy, `"$JS.ACK.DNS_PRIVATE.>"`) { + t.Error("control plane can acknowledge another serving member's consumers") + } for _, member := range []string{"node-us-central1-0", "regional-front-us-central1-0", "regional-coredns-us-central1-0", "regional-coredns-us-central1-1"} { start := strings.Index(contents, `user: "`+member+`"`) if start < 0 { diff --git a/internal/internaldns/serving/verify_test.go b/internal/internaldns/serving/verify_test.go index 4239c5d3..6650cc9a 100644 --- a/internal/internaldns/serving/verify_test.go +++ b/internal/internaldns/serving/verify_test.go @@ -122,7 +122,7 @@ func TestPublicationVerifierProvesMemberContextAndFingerprintOverBothTransports( return } defer func() { _ = conn.Close() }() - header := make([]byte, 28) + header := make([]byte, 28, 30) if _, err := io.ReadFull(conn, header); err != nil { return } diff --git a/test/internaldns/README.md b/test/internaldns/README.md new file mode 100644 index 00000000..21d82462 --- /dev/null +++ b/test/internaldns/README.md @@ -0,0 +1,119 @@ +# Internal DNS end-to-end qualification + +Run private DNS on the repository's shared Kubernetes test environment. Task +reuses the pinned `datum-cloud/test-infra` bootstrap, cluster names, add-ons, and +cleanup. The environment also supports the existing public DNS regression chain. +The qualification uses Go clients and Kubernetes workloads. + +## Run locally + +Install Docker, Kind, kubectl, Go, and Task. On macOS, select the suite-owned +Colima runtime before starting the environment: + +```sh +colima --profile internal-dns-e2e start --activate=false --cpu 6 --memory 16 --disk 60 +source test/internaldns/colima-env.sh +export TASK_X_REMOTE_TASKFILES=1 +task --yes env:internal-dns:up +task --yes env:internal-dns:qualify +``` + +`env:internal-dns:up` resets only the owned private DNS fixtures, then rebuilds +and deploys them on the existing clusters. Run it again before repeating the +qualification. It retains the shared bootstrap and public DNS workloads. + +Start private DNS first on fresh clusters to select the shared dual-stack Kind +configuration. If an existing public-only environment uses IPv4, the preflight +stops before resetting private fixtures. When ready to replace that environment, +run `env:down`, then start `env:internal-dns:up` before `env:stack-up`. + +To validate public DNS alongside private DNS, use the same lifecycle as CI: + +```sh +task --yes env:internal-dns:up +task --yes env:stack-up +task --yes env:internal-dns:qualify +task --yes env:chainsaw +test/internaldns/kubernetes/collect.sh +task --yes env:down +``` + +`env:down` removes the three shared test clusters. Qualification results and +sanitized deployment logs remain under `test/internaldns/results/kubernetes*`. +Generated kubeconfigs and Secrets stay in `.internal-dns-e2e/kubernetes/` and +are excluded from collected evidence. The qualifier restores controllers and +broker replicas after fault injection, including a failed scenario. + +## Deployed path + +```mermaid +flowchart LR + A[Project A API / scoped publisher] --> C[Two DNS controller Deployments] + B[Project B API / scoped publisher] --> C + C --> J[Three TLS JetStream replicas / PVCs] + J --> N[Shared node dnsdist / CoreDNS] + J --> R[Shared regional dnsdist / two CoreDNS members] + P[Separate consumer probe] -->|Service destination fixture| N --> R +``` + +The `dns-upstream` and `dns-edge` clusters provide independent project APIs. +The `dns-control` cluster hosts the DNS service API, admission Service, broker, +controllers, and fixed serving fleet. All three use the shared Flux, +cert-manager, Kyverno, and Envoy bootstrap. Private and public DNS use separate +namespaces on this foundation. + +The manifest renderer reads the shipped runtime configuration and RBAC. Each +controller uses scoped project worker credentials; product and +VPC integration credentials are separate. The simulated Compute publisher +creates registrations and contributions through the project API. It discovers +managed names from `DNSResolverContext.status.managedNamespace`. +Contribution writes publish records and eligibility atomically. Qualification +checks health withdrawal, stale-write conflicts, and DNS-owned status permissions. + +The fleet contains one node member, one regional frontend, and two regional +CoreDNS members, each with independent persistent state. Agents validate and +activate configuration and prove UDP/TCP publication. Local readiness verifies\napproved files and live daemons while healthy DNS survives agent restarts. +The broker uses three persistent replicas, client and route TLS, and scoped +subject permissions. cert-manager issues broker, client, and admission +certificates. ACK timing matches the shipped 30-second interval and 90-second +lease; the fixture shortens the ownership lease to 10 seconds and pod +termination grace to 2 seconds for fault tests. + +## Qualification scenarios + +The Go qualifier covers overlapping tenant names over UDP and TCP, A/AAAA and +negative-cache isolation, multiple zones, managed name discovery, scoped +publication and admission denial, record updates and deletion, health +withdrawal and recovery, fixed fleet size, regional publication proof, +controller takeover, continued tenant-isolated UDP/TCP resolution while controllers +and the broker are stopped, and publication after broker/controller restoration using +the retained broker PVCs. Each assertion records its result; skipped scenarios do +not count as passes. + +The separate package gate checks the shipped runtime image, all CRD schemas, +deployment rendering, configuration, and Kubernetes/NATS permissions: + +```sh +task --yes env:internal-dns:package +``` + +## Remaining production boundaries + +The deployment validates live Kubernetes APIs, admission, credentials, durable +transport, serving pods, and DNS responses. It does not deploy Milo project +discovery, Karmada, Galactic private +service networking, or the actual Compute service. The independent project APIs +and service-side IPv6 routes are explicit fixtures. Exact route/NAT exemptions +preserve serving pod peers without widening their ACLs. The probe does not share a +trusted serving network namespace or receive a Kubernetes credential. + +The [production qualification proposal](production-parity.md) covers the next +stage: queries from attached consumer VPCs through authorized private endpoints +into the separate DNS service VPC. Kind cannot establish physical fabric, +microVM attachment, host-loss durability, regional independence, or production +capacity. + +The suite uses wire version `v1alpha6` and persistent format 7 checkpoints. +Compatible fleet upgrades retain those checkpoints and transport state. Roll out +serving support before enabling new controller output; see the +[rollout contract](../../config/internal-dns/README.md#rollout-and-recovery). diff --git a/test/internaldns/colima-env.sh b/test/internaldns/colima-env.sh new file mode 100644 index 00000000..fe5c54a8 --- /dev/null +++ b/test/internaldns/colima-env.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Keep every internal-DNS qualification resource in its suite-owned Colima VM. +# This deliberately does not activate or change the user's default Docker +# context. Start the profile separately with: +# colima --profile internal-dns-e2e start --activate=false + +set -euo pipefail + +profile=${INTERNAL_DNS_COLIMA_PROFILE:-internal-dns-e2e} +socket=${INTERNAL_DNS_DOCKER_SOCKET:-"$HOME/.colima/$profile/docker.sock"} +expected="unix://$socket" + +if [[ -n ${DOCKER_HOST:-} && $DOCKER_HOST != "$expected" ]]; then + echo "internal DNS qualification requires DOCKER_HOST=$expected (got $DOCKER_HOST)" >&2 + exit 2 +fi +if [[ ! -S $socket ]]; then + echo "internal DNS Colima socket is unavailable: $socket" >&2 + echo "start it with: colima --profile $profile start --activate=false" >&2 + exit 2 +fi + +export INTERNAL_DNS_COLIMA_PROFILE=$profile +export DOCKER_HOST=$expected diff --git a/test/internaldns/fixtures/validating-webhook.yaml.tmpl b/test/internaldns/fixtures/validating-webhook.yaml.tmpl new file mode 100644 index 00000000..3731beb0 --- /dev/null +++ b/test/internaldns/fixtures/validating-webhook.yaml.tmpl @@ -0,0 +1,26 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: internal-dns-e2e + labels: + internal-dns.datum.net/qualification: "true" +webhooks: + - name: internal-dns.dns.networking.miloapis.com + admissionReviewVersions: [v1] + sideEffects: None + failurePolicy: Fail + timeoutSeconds: 5 + matchPolicy: Equivalent + clientConfig: + url: https://host.docker.internal:9443/validate-internal-dns + caBundle: ${CA_BUNDLE} + rules: + - operations: [CREATE, UPDATE] + apiGroups: [dns.networking.miloapis.com] + apiVersions: [v1alpha1] + resources: + - dnszoneassociations + - dnsregistrations + - dnsrecordcontributions + - dnsnamingpolicies + - dnsresolveraccessbindings diff --git a/test/internaldns/kubernetes/Dockerfile b/test/internaldns/kubernetes/Dockerfile new file mode 100644 index 00000000..0ddff2c4 --- /dev/null +++ b/test/internaldns/kubernetes/Dockerfile @@ -0,0 +1,29 @@ +# syntax=docker/dockerfile:1.7 +FROM --platform=$BUILDPLATFORM golang:1.26 AS build +ARG TARGETOS=linux +ARG TARGETARCH +WORKDIR /src +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/pkg/mod go mod download +COPY api ./api +COPY internal ./internal +COPY cmd/internal-dns ./cmd/internal-dns +COPY cmd/internal-dns-coredns ./cmd/internal-dns-coredns +COPY test/internaldns/kubernetes/qualify ./test/internaldns/kubernetes/qualify +RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -trimpath -o /out/internal-dns ./cmd/internal-dns && \ + CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -trimpath -o /out/coredns ./cmd/internal-dns-coredns && \ + CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -trimpath -o /out/dns-qualify ./test/internaldns/kubernetes/qualify + +FROM powerdns/dnsdist-20:2.0.10@sha256:24ac720af997aff6b98cb5b50d748754f815428e07b5368dfb0eccd5aa8d07e8 +USER 0:0 +RUN apt-get update && apt-get install --no-install-recommends -y \ + pdns-recursor iproute2 iptables procps netcat-openbsd ca-certificates && \ + rm -rf /var/lib/apt/lists/* +COPY --from=build /out/internal-dns /internal-dns +COPY --from=build /out/dns-qualify /dns-qualify +COPY --from=build /out/coredns /usr/local/bin/coredns +COPY config/internal-dns/commands/ /usr/local/sbin/ +USER 65532:65532 +RUN coredns -version +ENTRYPOINT ["/internal-dns"] diff --git a/test/internaldns/kubernetes/Dockerfile.dockerignore b/test/internaldns/kubernetes/Dockerfile.dockerignore new file mode 100644 index 00000000..5a873981 --- /dev/null +++ b/test/internaldns/kubernetes/Dockerfile.dockerignore @@ -0,0 +1,6 @@ +** +!**/*.go +**/*_test.go +!go.mod +!go.sum +!config/internal-dns/commands/** diff --git a/test/internaldns/kubernetes/collect.sh b/test/internaldns/kubernetes/collect.sh new file mode 100755 index 00000000..e93d6bfc --- /dev/null +++ b/test/internaldns/kubernetes/collect.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +cd "$root" +work="$root/.internal-dns-e2e/kubernetes" +evidence="$root/test/internaldns/results/kubernetes-artifacts" +mkdir -p "$evidence" +for file in platform project-a project-b; do + [[ -f $work/$file.kubeconfig ]] || continue + cli=(kubectl --kubeconfig "$work/$file.kubeconfig" --request-timeout=10s) + namespace=project-e2e + if [[ $file == platform ]]; then namespace=internal-dns-system; fi + "${cli[@]}" -n "$namespace" get pods,deployments,statefulsets,pvc,services,configmaps -o yaml > "$evidence/$file-deployment.yaml" 2>&1 || true + "${cli[@]}" -n "$namespace" get events -o yaml > "$evidence/$file-events.yaml" 2>&1 || true + "${cli[@]}" -n "$namespace" get dnszones,dnszoneassociations,dnsresolvercontexts,dnsresolveraccessbindings,dnsnamingpolicies,dnsregistrations,dnsrecordcontributions,dnsresolverbindings,dnspublicationmanifests,dnspublicationchunks,dnspublicationownerships,dnstransportoutboxes -o yaml > "$evidence/$file-dns.yaml" 2>&1 || true + while IFS= read -r pod; do + [[ -z $pod ]] || "${cli[@]}" -n "$namespace" logs "$pod" --all-containers --prefix --tail=1000 > "$evidence/$file-${pod#pod/}.log" 2>&1 || true + done < <("${cli[@]}" -n "$namespace" get pods -o name 2>/dev/null) +done +# Do not archive generated bootstrap manifests, Secrets, or kubeconfigs. diff --git a/test/internaldns/kubernetes/deploy.sh b/test/internaldns/kubernetes/deploy.sh new file mode 100755 index 00000000..01a22611 --- /dev/null +++ b/test/internaldns/kubernetes/deploy.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# Deploy onto the repository's shared test-infra clusters; never create a +# second Kind lifecycle or select the user's current Kubernetes context. +set -euo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +cd "$root" +work="$root/.internal-dns-e2e/kubernetes" +mkdir -p "$work" +chmod 700 "$work" + +for pair in 'dns-control platform' 'dns-upstream project-a' 'dns-edge project-b'; do + read -r cluster file <<< "$pair" + kind export kubeconfig --name "$cluster" --kubeconfig "$work/$file.kubeconfig" + kind get kubeconfig --name "$cluster" --internal > "$work/$file-internal.kubeconfig" + chmod 600 "$work/$file"*.kubeconfig +done +platform=(kubectl --kubeconfig "$work/platform.kubeconfig") +# Existing public-only clusters can use IPv4. Check the serving network before +# resetting any private fixtures; changing Kind networking requires fresh nodes. +pod_cidrs=$("${platform[@]}" get nodes -o jsonpath='{range .items[*]}{.spec.podCIDRs}{"\n"}{end}') +if ! grep -q ':' <<< "$pod_cidrs"; then + echo 'Private DNS requires dual-stack dns-control nodes. Start env:internal-dns:up before env:stack-up on fresh shared clusters; existing fixtures were not reset.' >&2 + exit 2 +fi +test/internaldns/kubernetes/reset.sh + +make kustomize +bin/kustomize build config/crd > "$work/crds.yaml" +for file in platform project-a project-b; do + kubectl --kubeconfig "$work/$file.kubeconfig" apply --server-side -f "$work/crds.yaml" +done + +docker build -f test/internaldns/kubernetes/Dockerfile \ + --iidfile "$work/image.id" -t internal-dns-kubernetes:e2e . +kind load docker-image internal-dns-kubernetes:e2e --name dns-control +go run ./test/internaldns/kubernetes/render --phase bootstrap --out "$work" +"${platform[@]}" apply -f "$work/platform-bootstrap.json" +for project in project-a project-b; do + kubectl --kubeconfig "$work/$project.kubeconfig" apply -f "$work/$project-bootstrap.json" +done +"${platform[@]}" -n internal-dns-system wait certificates --all --for=condition=Ready --timeout=180s +# Wait for network assignment before rendering concrete member addresses. The +# processes wait for ConfigMaps, so no placeholder serving configuration runs. +for pod in internal-dns-node internal-dns-regional-front internal-dns-regional-coredns-0 internal-dns-regional-coredns-1 internal-dns-probe; do + "${platform[@]}" -n internal-dns-system wait --for=jsonpath='{.status.podIPs}' "pod/$pod" --timeout=180s +done +go run ./test/internaldns/kubernetes/render --phase configure --out "$work" \ + --platform "$work/platform.kubeconfig" \ + --source-a "$work/project-a.kubeconfig" --source-b "$work/project-b.kubeconfig" \ + --source-a-internal "$work/project-a-internal.kubeconfig" --source-b-internal "$work/project-b-internal.kubeconfig" +"${platform[@]}" apply -f "$work/platform-configure.json" +"${platform[@]}" -n internal-dns-system wait pod/internal-dns-network-fixture --for=jsonpath='{.status.phase}'=Succeeded --timeout=60s +"${platform[@]}" -n internal-dns-system rollout status statefulset/internal-dns-nats --timeout=180s +for controller in control-a control-b; do + "${platform[@]}" -n internal-dns-system rollout status "deployment/$controller" --timeout=180s +done +for pod in internal-dns-node internal-dns-regional-front internal-dns-regional-coredns-0 internal-dns-regional-coredns-1; do + "${platform[@]}" -n internal-dns-system wait --for=condition=Ready "pod/$pod" --timeout=180s +done + +# The source API servers reach a real TLS admission Service over the isolated +# Kind network. +ca_bundle=$("${platform[@]}" -n internal-dns-system get secret internal-dns-admission-tls -o jsonpath='{.data.ca\.crt}') +for project in project-a project-b; do + sed -e "s|\${CA_BUNDLE}|$ca_bundle|g" \ + -e 's|host.docker.internal:9443|dns-control-control-plane:30443|g' \ + test/internaldns/fixtures/validating-webhook.yaml.tmpl > "$work/$project-webhook.yaml" + kubectl --kubeconfig "$work/$project.kubeconfig" apply -f "$work/$project-webhook.yaml" +done + +evidence="$root/test/internaldns/results/kubernetes-artifacts" +mkdir -p "$evidence" +cp "$work/crds.yaml" "$evidence/crds.yaml" +cp "$work/environment.json" "$evidence/environment.json" +cp "$work/image.id" "$evidence/image.id" +"${platform[@]}" -n internal-dns-system exec internal-dns-probe -- coredns -version > "$evidence/coredns-version.txt" +"${platform[@]}" -n internal-dns-system exec internal-dns-probe -- pdns_recursor --version > "$evidence/recursor-version.txt" 2>&1 +"${platform[@]}" -n internal-dns-system exec internal-dns-probe -- dnsdist --version > "$evidence/dnsdist-version.txt" +git rev-parse HEAD > "$evidence/source.commit" +git status --short > "$evidence/source.status" +if [[ -s $evidence/source.status ]]; then + printf 'dirty checkout; source.commit identifies the base, not the complete built source\n' > "$evidence/source.tree-state" +else + printf 'clean checkout\n' > "$evidence/source.tree-state" +fi +printf 'Kubernetes private DNS environment ready: %s\n' "$work/environment.json" diff --git a/test/internaldns/kubernetes/prepare-kind.sh b/test/internaldns/kubernetes/prepare-kind.sh new file mode 100755 index 00000000..b1017271 --- /dev/null +++ b/test/internaldns/kubernetes/prepare-kind.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +cd "$root" +mkdir -p .internal-dns-e2e/kubernetes +# Kind runs against the selected Docker host. A remote VM may not share this +# checkout's filesystem, so stage the common audit policy on that Docker host. +docker run --rm -i \ + -v /tmp/datum-internal-dns-audit:/audit \ + alpine@sha256:5291449c3df73caf6ed85e649dec1b9e818b39a5d8c871e97afc13e9cd5e8fa8 \ + sh -ec 'cat > /audit/audit-policy.yaml; chmod 644 /audit/audit-policy.yaml' \ + < .test-infra/cluster/audit-policy.yaml +sed 's|${REPO_DIR}/cluster/audit-policy.yaml|/tmp/datum-internal-dns-audit/audit-policy.yaml|g' \ + .test-infra/cluster/kind-config.yaml > .internal-dns-e2e/kubernetes/kind-config.yaml +cat >> .internal-dns-e2e/kubernetes/kind-config.yaml <<'EOF' +networking: + ipFamily: dual +EOF diff --git a/test/internaldns/kubernetes/qualify/main.go b/test/internaldns/kubernetes/qualify/main.go new file mode 100644 index 00000000..e3e2beff --- /dev/null +++ b/test/internaldns/kubernetes/qualify/main.go @@ -0,0 +1,556 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +// Command qualify publishes through project credentials and queries the shared +// Kubernetes serving fleet. The same executable runs DNS probes inside a pod. +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "reflect" + "sort" + "strings" + "time" + + "github.com/miekg/dns" + dnsv1 "go.miloapis.com/dns-operator/api/v1alpha1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type project struct { + Name string `json:"name"` + ProjectUID string `json:"projectUID"` + Namespace string `json:"namespace"` + CustomerKubeconfig string `json:"customerKubeconfig"` + ProductKubeconfig string `json:"productKubeconfig"` + IntegrationKubeconfig string `json:"integrationKubeconfig"` + ConsumerAddress string `json:"consumerAddress"` + customer, product, integration client.Client + resolverContext dnsv1.DNSResolverContext + zone dnsv1.DNSZone + access dnsv1.DNSResolverAccessBinding +} + +type environment struct { + PlatformKubeconfig string `json:"platformKubeconfig"` + Namespace string `json:"namespace"` + Region string `json:"region"` + Shard string `json:"shard"` + ProbePod string `json:"probePod"` + ProbeContainer string `json:"probeContainer"` + ProbeBinary string `json:"probeBinary"` + ControllerDeployments []string `json:"controllerDeployments"` + BrokerStatefulSet string `json:"brokerStatefulSet"` + RegionalMembers []string `json:"regionalMembers"` + Projects []project `json:"projects"` +} + +type check struct { + Name string `json:"name"` + Passed bool `json:"passed"` + Time time.Time `json:"time"` + Evidence any `json:"evidence,omitempty"` + Error string `json:"error,omitempty"` +} + +type report struct { + Suite string `json:"suite"` + Started time.Time `json:"started"` + Completed time.Time `json:"completed"` + Passed bool `json:"passed"` + Checks []check `json:"checks"` + Images map[string]string `json:"images"` + Limitations []string `json:"limitations"` + Error string `json:"error,omitempty"` +} + +type probeResult struct { + Authenticated bool `json:"authenticated"` + Recursive bool `json:"recursive"` + Rcode int `json:"rcode"` + Answers []string `json:"answers"` + Addresses []string `json:"addresses"` + TTL uint32 `json:"ttl"` + Error string `json:"error,omitempty"` +} + +type publication struct { + registration dnsv1.DNSRegistration + contribution dnsv1.DNSRecordContribution +} + +type suite struct { + env environment + platform client.Client + report report + ctx context.Context +} + +func main() { + envPath := flag.String("environment", "", "Generated environment JSON") + results := flag.String("results", "test/internaldns/results/kubernetes.json", "Qualification report") + probe := flag.Bool("probe", false, "Run one DNS query from a workload pod") + server := flag.String("server", "", "DNS server address and port") + name := flag.String("name", "", "DNS query name") + rrType := flag.String("type", "A", "DNS record type") + tcp := flag.Bool("tcp", false, "Use DNS over TCP") + flag.Parse() + if *probe { + if err := runProbe(*server, *name, *rrType, *tcp); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + return + } + s := &suite{ctx: context.Background(), report: report{ + Suite: "internal-dns-kubernetes", Started: time.Now().UTC(), Images: map[string]string{}, + Limitations: []string{ + "Kind clusters share one host; this run does not qualify regional or host failure.", + "Project APIs use independent Kubernetes servers; Milo discovery and Karmada are not exercised.", + "IPv6 context routes are a network fixture; Galactic private service connectivity and Compute attachment are not exercised.", + }, + }} + err := s.run(*envPath) + s.report.Completed = time.Now().UTC() + s.report.Passed = err == nil + if err != nil { + s.report.Error = err.Error() + s.report.Checks = append(s.report.Checks, check{Name: "qualification completed", Passed: false, Time: time.Now().UTC(), Error: err.Error()}) + } + if writeErr := writeReport(*results, s.report); writeErr != nil { + fmt.Fprintln(os.Stderr, writeErr) + os.Exit(1) + } + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + fmt.Printf("Passed %d Kubernetes qualification checks; evidence: %s\n", len(s.report.Checks), *results) +} + +func runProbe(server, name, rrType string, tcp bool) error { + t, ok := dns.StringToType[strings.ToUpper(rrType)] + if !ok || server == "" || name == "" { + return errors.New("probe requires a server, name, and valid record type") + } + message := new(dns.Msg) + message.SetQuestion(dns.Fqdn(name), t) + message.SetEdns0(1232, true) + transport := "udp" + if tcp { + transport = "tcp" + } + response, _, err := (&dns.Client{Net: transport, Timeout: 3 * time.Second}).Exchange(message, server) + if err != nil { + return json.NewEncoder(os.Stdout).Encode(probeResult{Error: err.Error()}) + } + result := probeResult{Rcode: response.Rcode, Answers: []string{}, Addresses: []string{}, Authenticated: response.AuthenticatedData, Recursive: response.RecursionAvailable} + for _, answer := range response.Answer { + result.Answers = append(result.Answers, answer.String()) + if result.TTL == 0 || answer.Header().Ttl < result.TTL { + result.TTL = answer.Header().Ttl + } + switch record := answer.(type) { + case *dns.A: + result.Addresses = append(result.Addresses, record.A.String()) + case *dns.AAAA: + result.Addresses = append(result.Addresses, record.AAAA.String()) + } + } + sort.Strings(result.Addresses) + return json.NewEncoder(os.Stdout).Encode(result) +} + +func writeReport(path string, value any) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, append(data, '\n'), 0o600) +} + +func newClient(path string) (client.Client, error) { + cfg, err := clientcmd.BuildConfigFromFlags("", path) + if err != nil { + return nil, err + } + cfg.Timeout = 8 * time.Second + scheme := runtime.NewScheme() + for _, add := range []func(*runtime.Scheme) error{corev1.AddToScheme, appsv1.AddToScheme, dnsv1.AddToScheme} { + if err := add(scheme); err != nil { + return nil, err + } + } + return client.New(cfg, client.Options{Scheme: scheme}) +} + +func (s *suite) pass(name string, evidence any) { + s.report.Checks = append(s.report.Checks, check{Name: name, Passed: true, Time: time.Now().UTC(), Evidence: evidence}) + fmt.Println("PASS", name) +} + +func (s *suite) wait(label string, duration time.Duration, test func() (bool, error)) error { + deadline := time.Now().Add(duration) + var last error + for time.Now().Before(deadline) { + passed, err := test() + if passed && err == nil { + return nil + } + if err != nil { + last = err + } + time.Sleep(500 * time.Millisecond) + } + return fmt.Errorf("timed out waiting for %s: %v", label, last) +} + +func objectRef(object client.Object) dnsv1.DNSObjectReference { + return dnsv1.DNSObjectReference{Name: object.GetName(), UID: object.GetUID(), Generation: object.GetGeneration()} +} + +func condition(conditions []metav1.Condition, name string, generation int64) bool { + c := apimeta.FindStatusCondition(conditions, name) + return c != nil && c.Status == metav1.ConditionTrue && c.ObservedGeneration == generation +} + +func (s *suite) run(path string) error { + data, err := os.ReadFile(path) + if err != nil { + return err + } + if err := json.Unmarshal(data, &s.env); err != nil { + return err + } + if len(s.env.Projects) != 2 || len(s.env.ControllerDeployments) == 0 || s.env.BrokerStatefulSet == "" { + return errors.New("qualification requires two project APIs, controllers, and a broker") + } + if s.env.ProbeBinary == "" { + s.env.ProbeBinary = "/dns-qualify" + } + if len(s.env.RegionalMembers) != 2 { + return errors.New("qualification requires two independent regional CoreDNS members") + } + s.platform, err = newClient(s.env.PlatformKubeconfig) + if err != nil { + return err + } + for i := range s.env.Projects { + p := &s.env.Projects[i] + for _, pair := range []struct { + path string + target *client.Client + }{ + {p.CustomerKubeconfig, &p.customer}, {p.ProductKubeconfig, &p.product}, + {p.IntegrationKubeconfig, &p.integration}, + } { + *pair.target, err = newClient(pair.path) + if err != nil { + return fmt.Errorf("%s client: %w", p.Name, err) + } + } + if net.ParseIP(p.ConsumerAddress) == nil || p.ProjectUID == "" { + return fmt.Errorf("%s has incomplete identity or resolver address", p.Name) + } + } + before, err := s.workloads() + if err != nil { + return err + } + if err := s.captureImages(); err != nil { + return err + } + for i := range s.env.Projects { + p := &s.env.Projects[i] + if err := s.permissions(p); err != nil { + return err + } + if err := s.prepare(p); err != nil { + return err + } + } + a, b := &s.env.Projects[0], &s.env.Projects[1] + if a.resolverContext.UID == b.resolverContext.UID || a.zone.UID == b.zone.UID || a.resolverContext.Status.ManagedNamespace.Suffix == b.resolverContext.Status.ManagedNamespace.Suffix { + return errors.New("source projects lost their distinct context, zone, or managed namespace identity") + } + s.pass("independent projects retain distinct context, zone, and managed namespace identities", map[string]string{"a": string(a.resolverContext.UID), "b": string(b.resolverContext.UID)}) + if err := s.scenarios(a, b); err != nil { + return err + } + after, err := s.workloads() + if err != nil { + return err + } + if !reflect.DeepEqual(before, after) { + return fmt.Errorf("contexts changed the shared workload count: before=%v after=%v", before, after) + } + s.pass("adding contexts and zones preserves the fixed shared workload count", after) + return nil +} + +func (s *suite) permissions(p *project) error { + contextObject := &dnsv1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "unauthorized", Namespace: p.Namespace}, Spec: dnsv1.DNSResolverContextSpec{ConsumerID: "unauthorized"}} + if err := p.product.Create(s.ctx, contextObject); !apierrors.IsForbidden(err) { + return fmt.Errorf("%s product network authorization must be forbidden: %v", p.Name, err) + } + access := &dnsv1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "unauthorized", Namespace: p.Namespace}} + if err := p.product.Create(s.ctx, access); !apierrors.IsForbidden(err) { + return fmt.Errorf("%s product access binding creation must be forbidden: %v", p.Name, err) + } + s.pass(p.Name+" product credentials cannot configure resolver access", nil) + return nil +} + +func (s *suite) prepare(p *project) error { + p.resolverContext = dnsv1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "application-network", Namespace: p.Namespace}, Spec: dnsv1.DNSResolverContextSpec{ConsumerID: "qualification/" + p.Name, ManagedNamespace: dnsv1.DNSResolverContextManagedNamespace{Enabled: true}}} + if err := p.integration.Create(s.ctx, &p.resolverContext); err != nil { + return err + } + if err := s.wait(p.Name+" managed namespace", 120*time.Second, func() (bool, error) { + err := p.product.Get(s.ctx, client.ObjectKeyFromObject(&p.resolverContext), &p.resolverContext) + return condition(p.resolverContext.Status.Conditions, "Ready", p.resolverContext.Generation) && p.resolverContext.Status.ManagedNamespace.DNSZoneRef.UID != "", err + }); err != nil { + return err + } + s.pass(p.Name+" product discovers its managed namespace through the context API", p.resolverContext.Status.ManagedNamespace) + p.zone = dnsv1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "prod", Namespace: p.Namespace}, Spec: dnsv1.DNSZoneSpec{DomainName: "prod.internal", DNSZoneClassName: "private-coredns", Visibility: dnsv1.DNSZoneVisibilityPrivate}} + if err := p.customer.Create(s.ctx, &p.zone); err != nil { + return err + } + if err := s.associate(p, &p.zone, "prod"); err != nil { + return err + } + p.access = dnsv1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "application-network-access", Namespace: p.Namespace}, Spec: dnsv1.DNSResolverAccessBindingSpec{ContextRef: objectRef(&p.resolverContext), Region: s.env.Region, DestinationAddress: p.ConsumerAddress}} + if err := p.integration.Create(s.ctx, &p.access); err != nil { + return err + } + if err := s.waitAccess(p); err != nil { + return err + } + s.pass(p.Name+" programmed destination becomes ready through shared Kubernetes members", p.ConsumerAddress) + return nil +} + +func (s *suite) associate(p *project, zone *dnsv1.DNSZone, name string) error { + association := &dnsv1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: p.Namespace}, Spec: dnsv1.DNSZoneAssociationSpec{DNSZoneRef: objectRef(zone), ResolverContextRef: objectRef(&p.resolverContext)}} + if err := p.customer.Create(s.ctx, association); err != nil { + return err + } + return s.wait("private zone association "+p.Name+"/"+name, 60*time.Second, func() (bool, error) { + err := p.customer.Get(s.ctx, client.ObjectKeyFromObject(association), association) + return condition(association.Status.Conditions, "Accepted", association.Generation), err + }) +} + +func (s *suite) waitAccess(p *project) error { + return s.wait(p.Name+" resolver access ready", 120*time.Second, func() (bool, error) { + err := p.integration.Get(s.ctx, client.ObjectKeyFromObject(&p.access), &p.access) + if err != nil { + return false, err + } + if !condition(p.access.Status.Conditions, "Ready", p.access.Generation) { + return false, fmt.Errorf("current access conditions: %v", p.access.Status.Conditions) + } + return true, nil + }) +} + +func recordSets(name, address, ipv6 string) []dnsv1.DNSContributionRecordSet { + sets := []dnsv1.DNSContributionRecordSet{{RecordType: dnsv1.RRTypeA, Records: []dnsv1.RecordEntry{{Name: name, A: &dnsv1.ARecordSpec{Content: address}}}}} + if ipv6 != "" { + sets = append(sets, dnsv1.DNSContributionRecordSet{RecordType: dnsv1.RRTypeAAAA, Records: []dnsv1.RecordEntry{{Name: name, AAAA: &dnsv1.AAAARecordSpec{Content: ipv6}}}}) + } + return sets +} + +func (s *suite) publish(p *project, zone dnsv1.DNSObjectReference, prefix, name, address, ipv6 string) (*publication, error) { + pub := &publication{registration: dnsv1.DNSRegistration{ObjectMeta: metav1.ObjectMeta{Name: prefix + "-registration", Namespace: p.Namespace}, Spec: dnsv1.DNSRegistrationSpec{DNSZoneRef: zone, Name: name, RecordTypes: []dnsv1.RRType{dnsv1.RRTypeA}, TTLSeconds: 2}}} + if ipv6 != "" { + pub.registration.Spec.RecordTypes = append(pub.registration.Spec.RecordTypes, dnsv1.RRTypeAAAA) + } + if err := p.product.Create(s.ctx, &pub.registration); err != nil { + return nil, err + } + pub.contribution = dnsv1.DNSRecordContribution{ObjectMeta: metav1.ObjectMeta{Name: prefix + "-endpoint", Namespace: p.Namespace}, Spec: dnsv1.DNSRecordContributionSpec{RegistrationRef: objectRef(&pub.registration), RecordSets: recordSets(name, address, ipv6)}} + if err := p.product.Create(s.ctx, &pub.contribution); err != nil { + return nil, err + } + if err := s.observe(p, pub, true); err != nil { + return nil, err + } + return pub, nil +} + +func (s *suite) observe(p *project, pub *publication, eligible bool) error { + if err := mutateContribution(s.ctx, p.product, &pub.contribution, func(spec *dnsv1.DNSRecordContributionSpec) { + spec.Eligible = eligible + spec.Reason = "EndpointReady" + if !eligible { + spec.Reason = "EndpointNotReady" + } + }); err != nil { + return err + } + return s.waitPublication(pub) +} + +func (s *suite) waitPublication(pub *publication) error { + return s.wait("verified publication for "+string(pub.contribution.UID), 90*time.Second, func() (bool, error) { + var publications dnsv1.DNSPublicationManifestList + if err := s.platform.List(s.ctx, &publications, client.InNamespace(s.env.Namespace)); err != nil { + return false, err + } + for _, manifest := range publications.Items { + matched := false + for _, fence := range manifest.Spec.ContributionFences { + if fence.UID == pub.contribution.UID && fence.Generation == pub.contribution.Generation { + matched = true + } + } + if !matched { + continue + } + members := map[string]bool{} + for _, ack := range manifest.Status.ReplicaAcknowledgements { + if ack.Phase == "Verified" && ack.WriterEpoch == manifest.Spec.WriterEpoch && ack.Revision == manifest.Spec.Revision && ack.ValidUntil != nil && ack.ValidUntil.After(time.Now()) { + members[ack.MemberID] = true + } + } + if members[s.env.RegionalMembers[0]] && members[s.env.RegionalMembers[1]] { + return true, nil + } + } + return false, fmt.Errorf("no fresh proof from both regional members for contribution %s generation %d", pub.contribution.UID, pub.contribution.Generation) + }) +} + +func (s *suite) query(p *project, name, rrType string, tcp bool) (probeResult, error) { + ctx, cancel := context.WithTimeout(s.ctx, 12*time.Second) + defer cancel() + args := []string{"--kubeconfig", s.env.PlatformKubeconfig, "-n", s.env.Namespace, "exec", s.env.ProbePod} + if s.env.ProbeContainer != "" { + args = append(args, "-c", s.env.ProbeContainer) + } + args = append(args, "--", s.env.ProbeBinary, "--probe", "--server", net.JoinHostPort(p.ConsumerAddress, "53"), "--name", name, "--type", rrType) + if tcp { + args = append(args, "--tcp") + } + cmd := exec.CommandContext(ctx, "kubectl", args...) + var stderr strings.Builder + cmd.Stderr = &stderr + output, err := cmd.Output() + if err != nil { + return probeResult{}, fmt.Errorf("query %s: %w: %s", name, err, stderr.String()) + } + var result probeResult + if err := json.Unmarshal(output, &result); err != nil { + return result, err + } + return result, nil +} + +func (s *suite) expect(p *project, name, rrType string, tcp bool, rcode int, addresses ...string) error { + sort.Strings(addresses) + var result probeResult + err := s.wait("DNS answer "+p.Name+" "+name, 60*time.Second, func() (bool, error) { + var err error + result, err = s.query(p, name, rrType, tcp) + if err != nil { + return false, err + } + if result.Error != "" { + return false, fmt.Errorf("DNS transport: %s", result.Error) + } + if result.Rcode != rcode { + return false, fmt.Errorf("rcode=%s answers=%v", dns.RcodeToString[result.Rcode], result.Answers) + } + if len(addresses) == 0 { + if len(result.Answers) != 0 { + return false, fmt.Errorf("unexpected answers=%v", result.Answers) + } + return true, nil + } + if !reflect.DeepEqual(result.Addresses, addresses) { + return false, fmt.Errorf("addresses=%v want=%v", result.Addresses, addresses) + } + return true, nil + }) + if err != nil { + return err + } + transport := "UDP" + if tcp { + transport = "TCP" + } + s.pass(p.Name+" "+transport+" "+rrType+" "+name, result) + return nil +} + +func (s *suite) workloads() (map[string]int32, error) { + values := map[string]int32{} + var deployments appsv1.DeploymentList + if err := s.platform.List(s.ctx, &deployments, client.InNamespace(s.env.Namespace)); err != nil { + return nil, err + } + for _, value := range deployments.Items { + if value.Spec.Replicas != nil { + values["deployment/"+value.Name] = *value.Spec.Replicas + } + } + var sets appsv1.StatefulSetList + if err := s.platform.List(s.ctx, &sets, client.InNamespace(s.env.Namespace)); err != nil { + return nil, err + } + for _, value := range sets.Items { + if value.Spec.Replicas != nil { + values["statefulset/"+value.Name] = *value.Spec.Replicas + } + } + var daemonSets appsv1.DaemonSetList + if err := s.platform.List(s.ctx, &daemonSets, client.InNamespace(s.env.Namespace)); err != nil { + return nil, err + } + for _, value := range daemonSets.Items { + values["daemonset/"+value.Name] = value.Status.DesiredNumberScheduled + } + var pods corev1.PodList + if err := s.platform.List(s.ctx, &pods, client.InNamespace(s.env.Namespace)); err != nil { + return nil, err + } + for _, pod := range pods.Items { + if len(pod.OwnerReferences) == 0 { + values["pod/"+pod.Name] = 1 + } + } + return values, nil +} + +func (s *suite) captureImages() error { + var pods corev1.PodList + if err := s.platform.List(s.ctx, &pods, client.InNamespace(s.env.Namespace)); err != nil { + return err + } + for _, pod := range pods.Items { + for _, c := range pod.Status.ContainerStatuses { + s.report.Images[pod.Name+"/"+c.Name] = c.ImageID + } + } + return nil +} diff --git a/test/internaldns/kubernetes/qualify/publisher.go b/test/internaldns/kubernetes/qualify/publisher.go new file mode 100644 index 00000000..1fc5f3f0 --- /dev/null +++ b/test/internaldns/kubernetes/qualify/publisher.go @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package main + +import ( + "context" + + dnsv1 "go.miloapis.com/dns-operator/api/v1alpha1" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func mutateContribution(ctx context.Context, source client.Client, contribution *dnsv1.DNSRecordContribution, mutate func(*dnsv1.DNSRecordContributionSpec)) error { + key := client.ObjectKeyFromObject(contribution) + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + if err := source.Get(ctx, key, contribution); err != nil { + return err + } + before := contribution.DeepCopy() + mutate(&contribution.Spec) + return source.Patch(ctx, contribution, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})) + }) +} diff --git a/test/internaldns/kubernetes/qualify/publisher_test.go b/test/internaldns/kubernetes/qualify/publisher_test.go new file mode 100644 index 00000000..d8818cff --- /dev/null +++ b/test/internaldns/kubernetes/qualify/publisher_test.go @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package main + +import ( + "context" + "errors" + "testing" + + dnsv1 "go.miloapis.com/dns-operator/api/v1alpha1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +type statusConflictClient struct { + client.Client + patches int +} + +func (c *statusConflictClient) Patch(ctx context.Context, object client.Object, patch client.Patch, opts ...client.PatchOption) error { + c.patches++ + if c.patches == 1 { + var current dnsv1.DNSRecordContribution + if err := c.Get(ctx, client.ObjectKeyFromObject(object), ¤t); err != nil { + return err + } + current.Status.PublishedRevision = 21 + if err := c.Status().Update(ctx, ¤t); err != nil { + return err + } + return apierrors.NewConflict(schema.GroupResource{Group: dnsv1.GroupVersion.Group, Resource: "dnsrecordcontributions"}, object.GetName(), errors.New("DNS readiness changed")) + } + return c.Client.Patch(ctx, object, patch, opts...) +} + +func TestContributionSpecRetryPreservesDNSReadiness(t *testing.T) { + scheme := runtime.NewScheme() + if err := dnsv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + contribution := dnsv1.DNSRecordContribution{ObjectMeta: metav1.ObjectMeta{Name: "endpoint", Namespace: "application"}, Spec: dnsv1.DNSRecordContributionSpec{RecordSets: recordSets("web", "10.42.0.10", ""), Eligible: true}} + source := &statusConflictClient{Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(&contribution).WithStatusSubresource(&dnsv1.DNSRecordContribution{}).Build()} + if err := mutateContribution(context.Background(), source, &contribution, func(spec *dnsv1.DNSRecordContributionSpec) { + spec.RecordSets = recordSets("web", "10.42.0.30", "") + spec.Eligible = false + spec.Reason = "EndpointNotReady" + }); err != nil { + t.Fatal(err) + } + if source.patches != 2 { + t.Fatalf("patch attempts = %d, want 2", source.patches) + } + var stored dnsv1.DNSRecordContribution + if err := source.Get(context.Background(), client.ObjectKeyFromObject(&contribution), &stored); err != nil { + t.Fatal(err) + } + if stored.Status.PublishedRevision != 21 { + t.Fatal("producer write changed DNS readiness status") + } + if stored.Spec.Eligible || stored.Spec.Reason != "EndpointNotReady" || len(stored.Spec.RecordSets) != 1 || len(stored.Spec.RecordSets[0].Records) != 1 || stored.Spec.RecordSets[0].Records[0].A == nil || stored.Spec.RecordSets[0].Records[0].A.Content != "10.42.0.30" { + t.Fatal("records and eligibility were not published together") + } +} diff --git a/test/internaldns/kubernetes/qualify/scenarios.go b/test/internaldns/kubernetes/qualify/scenarios.go new file mode 100644 index 00000000..1ed1d550 --- /dev/null +++ b/test/internaldns/kubernetes/qualify/scenarios.go @@ -0,0 +1,488 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package main + +import ( + "errors" + "fmt" + "os/exec" + "strings" + "time" + + "github.com/miekg/dns" + dnsv1 "go.miloapis.com/dns-operator/api/v1alpha1" + "go.miloapis.com/dns-operator/internal/internaldns/serving" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func (s *suite) scenarios(a, b *project) error { + initialDNSDist, err := s.dnsdistProcesses() + if err != nil { + return err + } + for _, p := range []*project{a, b} { + for _, tcp := range []bool{false, true} { + if err := s.wait("validated public recursion", 30*time.Second, func() (bool, error) { + result, err := s.query(p, "cloudflare.com", "A", tcp) + return err == nil && result.Error == "" && result.Rcode == dns.RcodeSuccess && len(result.Addresses) > 0 && result.Authenticated && result.Recursive, err + }); err != nil { + return err + } + s.pass("public DNSSEC recursion through PowerDNS for "+p.Name, map[string]bool{"tcp": tcp}) + } + } + for _, p := range []*project{a, b} { + if err := s.expect(p, "only-a.prod.internal", "A", false, dns.RcodeNameError); err != nil { + return err + } + } + pa, err := s.publish(a, objectRef(&a.zone), "common", "common", "10.42.0.10", "fd42::10") + if err != nil { + return err + } + pb, err := s.publish(b, objectRef(&b.zone), "common", "common", "10.42.0.20", "fd42::20") + if err != nil { + return err + } + if pa.registration.UID == pb.registration.UID || pa.contribution.UID == pb.contribution.UID { + return fmt.Errorf("same-named project publications lost their distinct identities") + } + s.pass("same-named registrations and contributions retain independent source identities", nil) + foreign := &dnsv1.DNSRecordContribution{ObjectMeta: metav1.ObjectMeta{Name: "foreign-project-endpoint", Namespace: a.Namespace}, Spec: dnsv1.DNSRecordContributionSpec{RegistrationRef: objectRef(&pb.registration), RecordSets: recordSets("common", "10.42.0.99", "")}} + if err := a.product.Create(s.ctx, foreign); !apierrors.IsForbidden(err) && !apierrors.IsInvalid(err) { + return fmt.Errorf("another project's publication references must be rejected by admission: %v", err) + } + s.pass("product admission rejects publication references from another source project", nil) + for _, tcp := range []bool{false, true} { + if err := s.expect(a, "common.prod.internal", "A", tcp, dns.RcodeSuccess, "10.42.0.10"); err != nil { + return err + } + if err := s.expect(b, "common.prod.internal", "A", tcp, dns.RcodeSuccess, "10.42.0.20"); err != nil { + return err + } + if err := s.expect(a, "common.prod.internal", "AAAA", tcp, dns.RcodeSuccess, "fd42::10"); err != nil { + return err + } + if err := s.expect(b, "common.prod.internal", "AAAA", tcp, dns.RcodeSuccess, "fd42::20"); err != nil { + return err + } + } + for _, p := range []*project{a, b} { + if err := s.expect(p, "common.prod.internal", "TXT", false, dns.RcodeSuccess); err != nil { + return err + } + } + unknown := *a + unknown.ConsumerAddress = "fd53::dead" + for _, tcp := range []bool{false, true} { + answer, err := s.query(&unknown, "common.prod.internal", "A", tcp) + if err != nil { + return err + } + if answer.Error == "" && answer.Rcode != dns.RcodeRefused && answer.Rcode != dns.RcodeServerFailure { + return fmt.Errorf("unknown destination must not resolve a tenant's record: %+v", answer) + } + } + if err := s.expect(b, "common.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.20"); err != nil { + return err + } + s.pass("an unbound destination cannot resolve either tenant's private records", nil) + if _, err := s.publish(a, objectRef(&a.zone), "only-a", "only-a", "10.42.0.20", ""); err != nil { + return err + } + if err := s.expect(a, "only-a.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.20"); err != nil { + return err + } + if err := s.expect(b, "only-a.prod.internal", "A", true, dns.RcodeNameError); err != nil { + return err + } + s.pass("negative caches and overlapping endpoint address spaces stay isolated", nil) + // Products discover their managed zone through context status. + if _, err := s.publish(a, a.resolverContext.Status.ManagedNamespace.DNSZoneRef, "managed", "web", "10.42.0.11", ""); err != nil { + return err + } + managedName := "web." + a.resolverContext.Status.ManagedNamespace.Suffix + managedNameB := "web." + b.resolverContext.Status.ManagedNamespace.Suffix + for _, tcp := range []bool{false, true} { + if err := s.expect(b, managedName, "A", tcp, dns.RcodeNameError); err != nil { + return err + } + } + if _, err := s.publish(b, b.resolverContext.Status.ManagedNamespace.DNSZoneRef, "managed", "web", "10.42.0.21", ""); err != nil { + return err + } + for _, tcp := range []bool{false, true} { + if err := s.expect(a, managedName, "A", tcp, dns.RcodeSuccess, "10.42.0.11"); err != nil { + return err + } + if err := s.expect(b, managedNameB, "A", tcp, dns.RcodeSuccess, "10.42.0.21"); err != nil { + return err + } + } + s.pass("managed names resolve within their owning contexts over UDP and TCP", map[string]string{"a": managedName, "b": managedNameB}) + apps := &dnsv1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "apps", Namespace: a.Namespace}, Spec: dnsv1.DNSZoneSpec{DomainName: "apps.internal", DNSZoneClassName: "private-coredns", Visibility: dnsv1.DNSZoneVisibilityPrivate}} + if err := a.customer.Create(s.ctx, apps); err != nil { + return err + } + if err := s.associate(a, apps, "apps"); err != nil { + return err + } + if _, err := s.publish(a, objectRef(apps), "apps", "api", "10.42.0.12", ""); err != nil { + return err + } + if err := s.expect(a, "api.apps.internal", "A", true, dns.RcodeSuccess, "10.42.0.12"); err != nil { + return err + } + s.pass("one resolver context serves its managed namespace and multiple custom zones", nil) + if err := s.observe(b, pb, true); err != nil { + return err + } + if err := s.update(a, pa, "10.42.0.30", "fd42::30"); err != nil { + return err + } + if err := s.expect(a, "common.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.30"); err != nil { + return err + } + if err := s.expect(b, "common.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.20"); err != nil { + return err + } + stale := pa.contribution.DeepCopy() + if err := s.observe(a, pa, false); err != nil { + return err + } + for _, tcp := range []bool{false, true} { + if err := s.expect(a, "common.prod.internal", "A", tcp, dns.RcodeSuccess); err != nil { + return err + } + } + s.pass("an unhealthy endpoint stops producing DNS answers", nil) + if err := s.observe(a, pa, true); err != nil { + return err + } + if err := s.expect(a, "common.prod.internal", "A", true, dns.RcodeSuccess, "10.42.0.30"); err != nil { + return err + } + if err := a.product.Get(s.ctx, client.ObjectKeyFromObject(&pa.contribution), &pa.contribution); err != nil { + return err + } + if err := a.product.Update(s.ctx, stale); !apierrors.IsConflict(err) { + return fmt.Errorf("stale producer write was not rejected: %w", err) + } + s.pass("optimistic concurrency rejects a stale producer update", nil) + if err := s.takeover(a, pa); err != nil { + return err + } + if err := s.observe(b, pb, true); err != nil { + return err + } + deleted, err := s.publish(a, objectRef(&a.zone), "deleted", "deleted", "10.42.0.14", "") + if err != nil { + return err + } + if err := s.expect(a, "deleted.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.14"); err != nil { + return err + } + if err := a.product.Delete(s.ctx, &deleted.contribution); err != nil { + return err + } + if err := s.expect(a, "deleted.prod.internal", "A", true, dns.RcodeSuccess); err != nil { + return err + } + s.pass("deleting a contribution withdraws its answer and preserves registration ownership", nil) + if err := s.offlineServing(a, b); err != nil { + return err + } + if err := s.observe(b, pb, true); err != nil { + return err + } + if err := s.expect(b, "common.prod.internal", "A", true, dns.RcodeSuccess, "10.42.0.20"); err != nil { + return err + } + s.pass("restored controllers and persistent broker publish a fresh observation", nil) + finalDNSDist, err := s.dnsdistProcesses() + if err != nil { + return err + } + for pod, id := range initialDNSDist { + if finalDNSDist[pod] != id { + return fmt.Errorf("dnsdist restarted during routing updates or agent recovery on %s", pod) + } + } + s.pass("upstream dnsdist processes survive context updates and agent restart", finalDNSDist) + return nil +} + +func (s *suite) update(p *project, pub *publication, address, ipv6 string) error { + if err := mutateContribution(s.ctx, p.product, &pub.contribution, func(spec *dnsv1.DNSRecordContributionSpec) { + spec.RecordSets = recordSets(pub.registration.Spec.Name, address, ipv6) + spec.Eligible = true + spec.Reason = "EndpointReady" + }); err != nil { + return err + } + return s.waitPublication(pub) +} + +func (s *suite) owner(zoneUID string) (dnsv1.DNSPublicationOwnership, error) { + var values dnsv1.DNSPublicationOwnershipList + if err := s.platform.List(s.ctx, &values, client.InNamespace(s.env.Namespace)); err != nil { + return dnsv1.DNSPublicationOwnership{}, err + } + for _, value := range values.Items { + if string(value.Spec.ZoneUID) == zoneUID { + return value, nil + } + } + return dnsv1.DNSPublicationOwnership{}, fmt.Errorf("zone %s has no publication owner", zoneUID) +} + +func (s *suite) scaleController(name string, replicas int32) error { + deployment := &appsv1.Deployment{} + key := client.ObjectKey{Namespace: s.env.Namespace, Name: name} + if err := s.platform.Get(s.ctx, key, deployment); err != nil { + return err + } + before := deployment.DeepCopy() + deployment.Spec.Replicas = &replicas + if err := s.platform.Patch(s.ctx, deployment, client.MergeFrom(before)); err != nil { + return err + } + return s.wait("controller scale "+name, 60*time.Second, func() (bool, error) { + var pods corev1.PodList + err := s.platform.List(s.ctx, &pods, client.InNamespace(s.env.Namespace), client.MatchingLabels(deployment.Spec.Selector.MatchLabels)) + if err != nil { + return false, err + } + if replicas == 0 { + return len(pods.Items) == 0, nil + } + if err := s.platform.Get(s.ctx, key, deployment); err != nil { + return false, err + } + return deployment.Status.ObservedGeneration >= deployment.Generation && + deployment.Status.AvailableReplicas == replicas && readyPods(pods.Items, replicas), nil + }) +} + +func (s *suite) takeover(p *project, pub *publication) (resultErr error) { + initial, err := s.owner(string(p.zone.UID)) + if err != nil { + return err + } + var target string + for _, name := range s.env.ControllerDeployments { + deployment := &appsv1.Deployment{} + if err := s.platform.Get(s.ctx, client.ObjectKey{Namespace: s.env.Namespace, Name: name}, deployment); err != nil { + return err + } + var pods corev1.PodList + if err := s.platform.List(s.ctx, &pods, client.InNamespace(s.env.Namespace), client.MatchingLabels(deployment.Spec.Selector.MatchLabels)); err != nil { + return err + } + for _, pod := range pods.Items { + if strings.HasPrefix(initial.Spec.HolderIdentity, pod.Name+"-") { + target = name + } + } + } + if target == "" { + return fmt.Errorf("cannot identify current controller owner %s", initial.Spec.HolderIdentity) + } + defer func() { + if err := s.scaleController(target, 1); err != nil { + resultErr = errors.Join(resultErr, fmt.Errorf("controller restoration: %w", err)) + } + }() + if err := s.scaleController(target, 0); err != nil { + return err + } + var current dnsv1.DNSPublicationOwnership + if err := s.wait("ownership takeover by surviving controller", 90*time.Second, func() (bool, error) { + var err error + current, err = s.owner(string(p.zone.UID)) + return current.Spec.WriterEpoch > initial.Spec.WriterEpoch && current.Spec.HolderIdentity != initial.Spec.HolderIdentity, err + }); err != nil { + return err + } + if err := s.observe(p, pub, true); err != nil { + return err + } + if err := s.expect(p, "common.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.30"); err != nil { + return err + } + s.pass("surviving Kubernetes controller takes ownership at a higher writer epoch", map[string]any{"previous": initial.Spec.HolderIdentity, "current": current.Spec.HolderIdentity, "writerEpoch": current.Spec.WriterEpoch}) + return nil +} + +func (s *suite) scaleBroker(replicas int32) error { + set := &appsv1.StatefulSet{} + key := client.ObjectKey{Namespace: s.env.Namespace, Name: s.env.BrokerStatefulSet} + if err := s.platform.Get(s.ctx, key, set); err != nil { + return err + } + before := set.DeepCopy() + set.Spec.Replicas = &replicas + if err := s.platform.Patch(s.ctx, set, client.MergeFrom(before)); err != nil { + return err + } + return s.wait("broker scale", 90*time.Second, func() (bool, error) { + var pods corev1.PodList + if err := s.platform.List(s.ctx, &pods, client.InNamespace(s.env.Namespace), + client.MatchingLabels(set.Spec.Selector.MatchLabels)); err != nil { + return false, err + } + if replicas == 0 { + return len(pods.Items) == 0, nil + } + if err := s.platform.Get(s.ctx, key, set); err != nil { + return false, err + } + return set.Status.ObservedGeneration >= set.Generation && + set.Status.ReadyReplicas == replicas && readyPods(pods.Items, replicas), nil + }) +} + +// offlineServing checks uncached queries while control-plane delivery is unavailable. +func (s *suite) offlineServing(a, b *project) (resultErr error) { + var broker appsv1.StatefulSet + if err := s.platform.Get(s.ctx, client.ObjectKey{Namespace: s.env.Namespace, Name: s.env.BrokerStatefulSet}, &broker); err != nil { + return err + } + replicas := int32(3) + if broker.Spec.Replicas != nil { + replicas = *broker.Spec.Replicas + } + defer func() { + if err := s.scaleBroker(replicas); err != nil { + resultErr = errors.Join(resultErr, fmt.Errorf("broker restoration: %w", err)) + } + for _, name := range s.env.ControllerDeployments { + if err := s.scaleController(name, 1); err != nil { + resultErr = errors.Join(resultErr, fmt.Errorf("controller restoration: %w", err)) + } + } + }() + for _, name := range s.env.ControllerDeployments { + if err := s.scaleController(name, 0); err != nil { + return err + } + } + if err := s.scaleBroker(0); err != nil { + return err + } + // The outage spans multiple DNS cache and fleet acknowledgement intervals. + until := time.Now().Add(3 * time.Minute) + for time.Now().Before(until) { + for _, tcp := range []bool{false, true} { + for _, tenant := range []struct { + project *project + address string + }{{a, "10.42.0.30"}, {b, "10.42.0.20"}} { + answer, err := s.query(tenant.project, "common.prod.internal", "A", tcp) + if err != nil { + return err + } + if answer.Error != "" || answer.Rcode != dns.RcodeSuccess || len(answer.Addresses) != 1 || answer.Addresses[0] != tenant.address { + return fmt.Errorf("outage query for %s TCP=%t: %#v", tenant.project.Name, tcp, answer) + } + } + } + select { + case <-s.ctx.Done(): + return s.ctx.Err() + case <-time.After(time.Second): + } + } + s.pass("both tenants keep resolving over UDP and TCP with controllers and messaging stopped", nil) + var pods corev1.PodList + if err := s.platform.List(s.ctx, &pods, client.InNamespace(s.env.Namespace), client.MatchingLabels{"internal-dns-role": "node"}); err != nil { + return err + } + if len(pods.Items) != 1 { + return fmt.Errorf("expected one node resolver, got %d", len(pods.Items)) + } + pod := pods.Items[0] + var restarts int32 + for _, status := range pod.Status.ContainerStatuses { + if status.Name == "agent" { + restarts = status.RestartCount + } + } + cmd := exec.CommandContext(s.ctx, "kubectl", "--kubeconfig", s.env.PlatformKubeconfig, "-n", s.env.Namespace, "exec", pod.Name, "-c", "agent", "--", "pkill", "-TERM", "-f", "^/internal-dns --role=agent") + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("restart resolver agent: %w: %s", err, output) + } + if err := s.wait("resolver agent restart without messaging", time.Minute, func() (bool, error) { + var current corev1.Pod + if err := s.platform.Get(s.ctx, client.ObjectKeyFromObject(&pod), ¤t); err != nil { + return false, err + } + for _, status := range current.Status.ContainerStatuses { + if status.Name == "agent" && status.RestartCount > restarts && status.State.Running != nil { + return true, nil + } + } + return false, nil + }); err != nil { + return err + } + for _, tcp := range []bool{false, true} { + if err := s.expect(a, "common.prod.internal", "A", tcp, dns.RcodeSuccess, "10.42.0.30"); err != nil { + return err + } + if err := s.expect(b, "common.prod.internal", "A", tcp, dns.RcodeSuccess, "10.42.0.20"); err != nil { + return err + } + } + s.pass("resolver agent restores both tenant contexts from persistent state without messaging", nil) + return nil +} + +func readyPods(pods []corev1.Pod, replicas int32) bool { + if len(pods) != int(replicas) { + return false + } + for _, pod := range pods { + if pod.DeletionTimestamp != nil { + return false + } + ready := false + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue { + ready = true + } + } + if !ready { + return false + } + } + return true +} + +func (s *suite) dnsdistProcesses() (map[string]string, error) { + result := map[string]string{} + for _, name := range []string{"internal-dns-node", "internal-dns-regional-front"} { + var pod corev1.Pod + if err := s.platform.Get(s.ctx, client.ObjectKey{Namespace: s.env.Namespace, Name: name}, &pod); err != nil { + return nil, err + } + for _, container := range pod.Spec.Containers { + if container.Name == "dnsdist" && (container.Image != serving.DNSDistImage || len(container.Command) != 1 || container.Command[0] != "/usr/local/bin/dnsdist") { + return nil, fmt.Errorf("%s must run the upstream dnsdist container directly", name) + } + } + for _, status := range pod.Status.ContainerStatuses { + if status.Name == "dnsdist" && status.State.Running != nil { + result[name] = status.ContainerID + } + } + if result[name] == "" { + return nil, fmt.Errorf("%s has no running dnsdist process", name) + } + } + return result, nil +} diff --git a/test/internaldns/kubernetes/render/main.go b/test/internaldns/kubernetes/render/main.go new file mode 100644 index 00000000..d903a786 --- /dev/null +++ b/test/internaldns/kubernetes/render/main.go @@ -0,0 +1,471 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Render the Kubernetes qualification from the shipped DNS configuration and RBAC. +package main + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/base64" + "encoding/json" + "flag" + "fmt" + "io" + "net/netip" + "os" + "path/filepath" + "strings" + + "go.miloapis.com/dns-operator/internal/internaldns/model" + runtime "go.miloapis.com/dns-operator/internal/internaldns/runtime" + "go.miloapis.com/dns-operator/internal/internaldns/serving" + authv1 "k8s.io/api/authentication/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + k8syaml "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" +) + +type object = map[string]any + +const ns = "internal-dns-system" +const projectNS = "project-e2e" +const image = "internal-dns-kubernetes:e2e" +const nodeRole = string(serving.RenderNode) +const regionalDNSDistRole = string(serving.RenderRegionalDNSDist) + +var out, phase, platform, a, b, ai, bi string +var memberNames = []string{"node-us-central1-0", "regional-front-us-central1-0", "regional-coredns-us-central1-0", "regional-coredns-us-central1-1"} +var podNames = []string{"internal-dns-node", "internal-dns-regional-front", "internal-dns-regional-coredns-0", "internal-dns-regional-coredns-1"} + +func main() { + flag.StringVar(&out, "out", ".internal-dns-e2e/kubernetes", "Private generated environment directory") + flag.StringVar(&phase, "phase", "bootstrap", "bootstrap or configure") + flag.StringVar(&platform, "platform", "", "Platform admin kubeconfig") + flag.StringVar(&a, "source-a", "", "Source A admin kubeconfig") + flag.StringVar(&b, "source-b", "", "Source B admin kubeconfig") + flag.StringVar(&ai, "source-a-internal", "", "Source A internal admin kubeconfig") + flag.StringVar(&bi, "source-b-internal", "", "Source B internal admin kubeconfig") + flag.Parse() + var err error + out, err = filepath.Abs(out) + must(err) + must(os.MkdirAll(out, 0700)) + switch phase { + case "bootstrap": + bootstrap() + case "configure": + configure() + default: + panic("unknown phase") + } +} +func must(err error) { + if err != nil { + panic(err) + } +} +func obj(api, kind, name, namespace string) object { + m := object{"name": name} + if namespace != "" { + m["namespace"] = namespace + } + if kind == "Namespace" && (name == ns || name == projectNS) { + m["labels"] = object{"internal-dns.datum.net/qualification": "true"} + } + return object{"apiVersion": api, "kind": kind, "metadata": m} +} +func write(name string, value any) { + data, err := json.MarshalIndent(value, "", " ") + must(err) + must(os.WriteFile(filepath.Join(out, name), append(data, '\n'), 0600)) +} +func list(name string, items []object) { + write(name, object{"apiVersion": "v1", "kind": "List", "items": items}) +} +func appendShippedRBAC(items []object) []object { + data, err := os.ReadFile("config/internal-dns/rbac.yaml") + must(err) + d := k8syaml.NewYAMLOrJSONDecoder(bytes.NewReader(data), 4096) + for { + var o object + err = d.Decode(&o) + if err == io.EOF { + break + } + must(err) + u := unstructured.Unstructured{Object: o} + if u.GetNamespace() == "example-project" { + u.SetNamespace(projectNS) + } + items = append(items, u.Object) + } + return items +} +func certificate(name string, dns []string, usages []string, ca bool, issuer string) object { + o := obj("cert-manager.io/v1", "Certificate", name, ns) + spec := object{"secretName": name, "commonName": name, "isCA": ca, "issuerRef": object{"name": issuer, "kind": "Issuer"}, "privateKey": object{"algorithm": "ECDSA", "size": 256}, "duration": "24h", "renewBefore": "8h"} + if len(dns) > 0 { + spec["dnsNames"] = dns + } + if len(usages) > 0 { + spec["usages"] = usages + } + o["spec"] = spec + return o +} +func bootstrap() { + items := []object{obj("v1", "Namespace", ns, ""), obj("v1", "Namespace", projectNS, "")} + items = appendShippedRBAC(items) + self := obj("cert-manager.io/v1", "Issuer", "internal-dns-selfsigned", ns) + self["spec"] = object{"selfSigned": object{}} + items = append(items, self, certificate("internal-dns-ca", nil, []string{"cert sign", "crl sign"}, true, "internal-dns-selfsigned")) + issuer := obj("cert-manager.io/v1", "Issuer", "internal-dns-ca", ns) + issuer["spec"] = object{"ca": object{"secretName": "internal-dns-ca"}} + items = append(items, issuer) + items = append(items, certificate("internal-dns-nats-server", []string{"nats." + ns + ".svc", "nats." + ns + ".svc.cluster.local", "*.internal-dns-nats." + ns + ".svc.cluster.local"}, []string{"server auth", "client auth"}, false, "internal-dns-ca")) + items = append(items, certificate("internal-dns-admission-tls", []string{"internal-dns-admission." + ns + ".svc", "dns-control-control-plane"}, []string{"server auth"}, false, "internal-dns-ca")) + identities := append([]string{"control-a", "control-b"}, memberNames...) + for _, name := range identities { + items = append(items, certificate(name+"-nats", nil, []string{"client auth"}, false, "internal-dns-ca")) + } + // Kubernetes Secrets carry the broker ACL passwords and RNDC key; none are + // emitted into ConfigMaps or test reports. + users := make([]object, 0, len(identities)) + for _, name := range identities { + password := randomSecret() + secret := obj("v1", "Secret", name+"-password", ns) + secret["stringData"] = object{"password": password} + items = append(items, secret) + allow := []string{"dns.private.serving.>", "dns.private.records.>", "$KV.DNS_PRIVATE_SNAPSHOTS.>", "$JS.API.>", "$JS.ACK.DNS_PRIVATE.control-plane-us-central1-shared-0-acks.>", "_INBOX.>"} + subscribe := []string{"dns.private.acks.>", "_INBOX.>"} + if !strings.HasPrefix(name, "control-") { + allow = make([]string, 0, 24) + allow = append(allow, model.AckSubject("us-central1", "shared-0", name), "$JS.API.STREAM.INFO.DNS_PRIVATE", "$JS.API.DIRECT.GET.KV_DNS_PRIVATE_SNAPSHOTS.>", "$JS.API.STREAM.INFO.KV_DNS_PRIVATE_SNAPSHOTS", "$JS.API.CONSUMER.CREATE.KV_DNS_PRIVATE_SNAPSHOTS.>", "$JS.API.CONSUMER.INFO.KV_DNS_PRIVATE_SNAPSHOTS.>", "$JS.API.CONSUMER.MSG.NEXT.KV_DNS_PRIVATE_SNAPSHOTS.>", "$JS.API.CONSUMER.DELETE.KV_DNS_PRIVATE_SNAPSHOTS.>", "_INBOX.>") + subscribe = []string{"_INBOX.>"} + for _, suffix := range []string{"serving", "records", "acks"} { + c := name + "-" + suffix + allow = append(allow, "$JS.API.CONSUMER.INFO.DNS_PRIVATE."+c, "$JS.API.CONSUMER.CREATE.DNS_PRIVATE."+c, "$JS.API.CONSUMER.CREATE.DNS_PRIVATE."+c+".>", "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE."+c, "$JS.ACK.DNS_PRIVATE."+c+".>") + } + } + users = append(users, object{"user": name, "password": password, "permissions": object{"publish": object{"allow": allow}, "subscribe": object{"allow": subscribe}}}) + } + tls := object{"cert_file": "/tls/tls.crt", "key_file": "/tls/tls.key", "ca_file": "/tls/ca.crt", "verify": true} + cfg := object{"port": 4222, "http_port": 8222, "jetstream": object{"store_dir": "/data"}, "tls": tls, "authorization": object{"users": users}, "cluster": object{"name": "internal-dns-e2e", "port": 6222, "tls": tls, "routes": []string{"nats-route://internal-dns-nats-0.internal-dns-nats." + ns + ".svc.cluster.local:6222", "nats-route://internal-dns-nats-1.internal-dns-nats." + ns + ".svc.cluster.local:6222", "nats-route://internal-dns-nats-2.internal-dns-nats." + ns + ".svc.cluster.local:6222"}}} + var encoded bytes.Buffer + encoder := json.NewEncoder(&encoded) + // NATS accepts JSON-style configuration but not JSON's HTML Unicode escapes. + encoder.SetEscapeHTML(false) + must(encoder.Encode(cfg)) + secret := obj("v1", "Secret", "internal-dns-nats-config", ns) + secret["stringData"] = object{"nats.conf": encoded.String()} + items = append(items, secret) + + for _, name := range []string{"nats", "internal-dns-nats"} { + s := obj("v1", "Service", name, ns) + spec := object{"selector": object{"app": "internal-dns-nats"}, "ports": []object{{"name": "client", "port": 4222}, {"name": "cluster", "port": 6222}}} + if name == "internal-dns-nats" { + spec["clusterIP"] = "None" + spec["publishNotReadyAddresses"] = true + } + s["spec"] = spec + items = append(items, s) + } + broker := obj("apps/v1", "StatefulSet", "internal-dns-nats", ns) + broker["spec"] = object{"replicas": 3, "serviceName": "internal-dns-nats", "podManagementPolicy": "Parallel", "selector": object{"matchLabels": object{"app": "internal-dns-nats"}}, "template": object{"metadata": object{"labels": object{"app": "internal-dns-nats"}}, "spec": object{"terminationGracePeriodSeconds": 2, "automountServiceAccountToken": false, "containers": []object{{"name": "nats", "image": "nats@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00", "command": []string{"nats-server"}, "args": []string{"-c", "/config/nats.conf", "-n", "$(POD_NAME)"}, "env": []object{{"name": "POD_NAME", "valueFrom": object{"fieldRef": object{"fieldPath": "metadata.name"}}}}, "volumeMounts": []object{{"name": "config", "mountPath": "/config", "readOnly": true}, {"name": "tls", "mountPath": "/tls", "readOnly": true}, {"name": "data", "mountPath": "/data"}}, "readinessProbe": object{"httpGet": object{"path": "/healthz?js-enabled-only=true", "port": 8222}}, "resources": object{"requests": object{"cpu": "100m", "memory": "128Mi"}, "limits": object{"memory": "512Mi"}}}}, "volumes": []object{{"name": "config", "secret": object{"secretName": "internal-dns-nats-config"}}, {"name": "tls", "secret": object{"secretName": "internal-dns-nats-server"}}}}}, "volumeClaimTemplates": []object{{"metadata": object{"name": "data"}, "spec": object{"accessModes": []string{"ReadWriteOnce"}, "resources": object{"requests": object{"storage": "1Gi"}}}}}} + items = append(items, broker) + public := obj("v1", "Service", "internal-dns-public-recursor", ns) + public["spec"] = object{"ipFamilies": []string{"IPv6"}, "ipFamilyPolicy": "SingleStack", "selector": object{"app": "internal-dns-public-recursor"}, "ports": []object{{"name": "dns-udp", "port": 5353, "protocol": "UDP"}, {"name": "dns-tcp", "port": 5353, "protocol": "TCP"}}} + items = append(items, public) + for i, name := range podNames { + role := []string{nodeRole, regionalDNSDistRole, "regional-coredns", "regional-coredns"}[i] + items = append(items, fleetPod(name, memberNames[i], role)) + for _, vol := range []string{"state", "config"} { + pvc := obj("v1", "PersistentVolumeClaim", name+"-"+vol, ns) + pvc["spec"] = object{"accessModes": []string{"ReadWriteOnce"}, "resources": object{"requests": object{"storage": "512Mi"}}} + items = append(items, pvc) + } + } + // Admission is a real Kubernetes TLS Service, reached by the independent + // source API servers through a Kind-network NodePort. + s := obj("v1", "Service", "internal-dns-admission", ns) + s["spec"] = object{"type": "NodePort", "selector": object{"internal-dns-role": "control-plane"}, "ports": []object{{"name": "https", "port": 443, "targetPort": 9443, "nodePort": 30443}}} + items = append(items, s) + for _, name := range []string{"control-a", "control-b"} { + items = append(items, controller(name)) + } + probe := obj("v1", "Pod", "internal-dns-probe", ns) + probe["spec"] = object{"terminationGracePeriodSeconds": 2, "automountServiceAccountToken": false, "securityContext": object{"runAsUser": 65532, "runAsGroup": 65532}, "containers": []object{{"name": "probe", "image": image, "imagePullPolicy": "Never", "command": []string{"/bin/sh", "-c", "sleep infinity"}, "securityContext": object{"allowPrivilegeEscalation": false, "readOnlyRootFilesystem": true, "capabilities": object{"drop": []string{"ALL"}}}}}} + items = append(items, probe) + list("platform-bootstrap.json", items) + for _, project := range []string{"project-a", "project-b"} { + list(project+"-bootstrap.json", sourceObjects(project)) + } +} +func randomSecret() string { + data := make([]byte, 32) + _, err := rand.Read(data) + must(err) + return base64.StdEncoding.EncodeToString(data) +} +func mount(name, path string, ro bool) object { + return object{"name": name, "mountPath": path, "readOnly": ro} +} +func security() object { + return object{"runAsNonRoot": true, "runAsUser": 65532, "runAsGroup": 65532, "allowPrivilegeEscalation": false, "readOnlyRootFilesystem": true, "capabilities": object{"drop": []string{"ALL"}}} +} +func credentialsVolume(member string) object { + return object{"name": "nats", "projected": object{"sources": []object{{"secret": object{"name": member + "-nats"}}, {"secret": object{"name": member + "-password"}}}}} +} +func fleetPod(name, member, role string) object { + o := obj("v1", "Pod", name, ns) + o["metadata"].(object)["labels"] = object{"internal-dns-role": role, "internal-dns-member": member} + mounts := []object{mount("config", "/config", true), mount("nats", "/var/run/internal-dns/nats", true), mount("state", "/var/lib/internal-dns", false), mount("daemon-config", "/etc/internal-dns", false), mount("lease", "/run/internal-dns", false), mount("runtime", "/tmp", false)} + containers := make([]object, 0, 3) + for _, process := range []string{"agent"} { + containers = append(containers, object{"name": process, "image": image, "imagePullPolicy": "Never", "command": []string{"/bin/sh", "-ec", "while [ ! -s /config/config.json ]; do sleep 1; done; exec /internal-dns --role=" + process + " --config=/config/config.json"}, "securityContext": security(), "volumeMounts": mounts, "resources": object{"requests": object{"cpu": "50m", "memory": "64Mi"}, "limits": object{"memory": "512Mi"}}}) + } + sec := security() + sec["capabilities"] = object{"drop": []string{"ALL"}, "add": []string{"NET_BIND_SERVICE"}} + if role != regionalDNSDistRole { + containers = append(containers, object{"name": "serving", "image": image, "imagePullPolicy": "Never", "command": []string{"/internal-dns", "--role=serving", "--config=/config/config.json"}, "readinessProbe": object{"exec": object{"command": []string{"/internal-dns", "--role=readiness", "--config=/config/config.json"}}, "periodSeconds": 2, "timeoutSeconds": 2}, "securityContext": sec, "volumeMounts": mounts, "resources": object{"requests": object{"cpu": "100m", "memory": "128Mi"}, "limits": object{"memory": "512Mi"}}}) + } + if role == nodeRole || role == regionalDNSDistRole { + file := "node-dnsdist.conf" + if role != nodeRole { + file = "cluster-dnsdist.conf" + containers[0]["readinessProbe"] = object{"exec": object{"command": []string{"/internal-dns", "--role=readiness", "--config=/config/config.json"}}, "periodSeconds": 2, "timeoutSeconds": 2} + } + containers = append(containers, object{"name": "dnsdist", "image": serving.DNSDistImage, "command": []string{"/usr/local/bin/dnsdist"}, "args": []string{"--supervised", "--disable-syslog", "-C", "/etc/internal-dns/" + file}, "securityContext": sec, "volumeMounts": []object{mount("daemon-config", "/etc/internal-dns", true), mount("lease", "/run/internal-dns", true), mount("runtime", "/tmp", false)}, "resources": object{"requests": object{"cpu": "50m", "memory": "64Mi"}, "limits": object{"memory": "512Mi"}}}) + } + volumes := []object{{"name": "config", "configMap": object{"name": name + "-config", "optional": true}}, credentialsVolume(member), {"name": "state", "persistentVolumeClaim": object{"claimName": name + "-state"}}, {"name": "daemon-config", "persistentVolumeClaim": object{"claimName": name + "-config"}}, {"name": "lease", "emptyDir": object{"medium": "Memory"}}, {"name": "runtime", "emptyDir": object{}}} + // The aliases model the service-side destination contract only. The separate + // probe has no ability to write routes or send trusted PROXY traffic. + init := []object{} + if role == nodeRole || role == regionalDNSDistRole { + init = append(init, object{"name": "dnsdist-config", "image": image, "imagePullPolicy": "Never", "command": []string{"/bin/sh", "-ec", "while [ ! -s /config/config.json ]; do sleep 1; done; exec /internal-dns --role=prepare --config=/config/config.json"}, "securityContext": security(), "volumeMounts": mounts}) + } + if role == nodeRole || role == regionalDNSDistRole { + prefix := "fd53::/64" + if role != nodeRole { + prefix = "fd54::/64" + } + init = append(init, object{"name": "destination-fixture", "image": image, "imagePullPolicy": "Never", "command": []string{"/bin/sh", "-ec", "ip -6 route replace local " + prefix + " dev lo"}, "securityContext": object{"runAsUser": 0, "capabilities": object{"drop": []string{"ALL"}, "add": []string{"NET_ADMIN"}}}}) + } + o["spec"] = object{"serviceAccountName": "internal-dns-fleet", "terminationGracePeriodSeconds": 2, "automountServiceAccountToken": false, "shareProcessNamespace": true, "securityContext": object{"fsGroup": 65532, "fsGroupChangePolicy": "OnRootMismatch", "sysctls": []object{{"name": "net.ipv4.ip_unprivileged_port_start", "value": "0"}}}, "containers": containers, "initContainers": init, "volumes": volumes} + return o +} +func controller(name string) object { + o := obj("apps/v1", "Deployment", name, ns) + labels := object{"app": name, "internal-dns-role": "control-plane"} + mounts := []object{mount("config", "/config", true), mount("nats", "/var/run/internal-dns/nats", true), mount("admission", "/var/run/internal-dns/admission", true), mount("sources", "/sources", true)} + o["spec"] = object{"replicas": 1, "selector": object{"matchLabels": object{"app": name}}, "template": object{"metadata": object{"labels": labels}, "spec": object{"terminationGracePeriodSeconds": 2, "serviceAccountName": "internal-dns-control-plane", "containers": []object{{"name": "control-plane", "image": image, "imagePullPolicy": "Never", "command": []string{"/bin/sh", "-ec", "while [ ! -s /config/config.json ] || [ ! -s /sources/project-a.json ]; do sleep 1; done; exec /internal-dns --role=control-plane --config=/config/config.json"}, "securityContext": security(), "volumeMounts": mounts, "ports": []object{{"name": "admission", "containerPort": 9443}}, "readinessProbe": object{"tcpSocket": object{"port": 9443}}, "resources": object{"requests": object{"cpu": "100m", "memory": "128Mi"}, "limits": object{"memory": "768Mi"}}}}, "volumes": []object{{"name": "config", "configMap": object{"name": name + "-config", "optional": true}}, credentialsVolume(name), {"name": "admission", "secret": object{"secretName": "internal-dns-admission-tls"}}, {"name": "sources", "secret": object{"secretName": "internal-dns-sources", "optional": true}}}}}} + return o +} +func sourceObjects(project string) []object { + items := []object{obj("v1", "Namespace", ns, ""), obj("v1", "Namespace", projectNS, ""), obj("v1", "Namespace", "compute-system", ""), obj("v1", "Namespace", "network-system", "")} + items = appendShippedRBAC(items) + filtered := items[:0] + for _, item := range items { + if item["kind"] == "ClusterRoleBinding" && item["metadata"].(map[string]any)["name"] == "internal-dns-control-plane" { + continue + } + filtered = append(filtered, item) + } + items = filtered + worker := obj("rbac.authorization.k8s.io/v1", "RoleBinding", "internal-dns-source-worker", projectNS) + worker["roleRef"] = object{"apiGroup": "rbac.authorization.k8s.io", "kind": "ClusterRole", "name": "internal-dns-control-plane"} + worker["subjects"] = []object{{"kind": "ServiceAccount", "name": "internal-dns-control-plane", "namespace": ns}} + items = append(items, worker) + accounts := []struct{ name, namespace, role string }{{"dns-customer", projectNS, "internal-dns-customer"}, {"dns-publisher", "compute-system", "internal-dns-product-writer"}, {"dns-vpc-integration", "network-system", "internal-dns-vpc-integration"}, {"internal-dns-control-plane", ns, ""}} + for _, account := range accounts { + if account.role != "" { + items = append(items, obj("v1", "ServiceAccount", account.name, account.namespace)) + rb := obj("rbac.authorization.k8s.io/v1", "RoleBinding", account.name, projectNS) + rb["roleRef"] = object{"apiGroup": "rbac.authorization.k8s.io", "kind": "Role", "name": account.role} + rb["subjects"] = []object{{"kind": "ServiceAccount", "name": account.name, "namespace": account.namespace}} + items = append(items, rb) + } + // Project identity in Kind is scoped to impersonation of the same + // service account and this fixed project parent. + r := obj("rbac.authorization.k8s.io/v1", "ClusterRole", account.name+"-parent", "") + r["rules"] = []object{{"apiGroups": []string{"authentication.k8s.io"}, "resources": []string{"userextras/iam.miloapis.com/parent-name"}, "resourceNames": []string{project}, "verbs": []string{"impersonate"}}} + rb := obj("rbac.authorization.k8s.io/v1", "ClusterRoleBinding", account.name+"-parent", "") + rb["roleRef"] = object{"apiGroup": "rbac.authorization.k8s.io", "kind": "ClusterRole", "name": account.name + "-parent"} + rb["subjects"] = []object{{"kind": "ServiceAccount", "name": account.name, "namespace": account.namespace}} + items = append(items, r, rb) + self := obj("rbac.authorization.k8s.io/v1", "Role", account.name+"-self", account.namespace) + self["rules"] = []object{{"apiGroups": []string{""}, "resources": []string{"serviceaccounts"}, "resourceNames": []string{account.name}, "verbs": []string{"impersonate"}}} + binding := obj("rbac.authorization.k8s.io/v1", "RoleBinding", account.name+"-self", account.namespace) + binding["roleRef"] = object{"apiGroup": "rbac.authorization.k8s.io", "kind": "Role", "name": account.name + "-self"} + binding["subjects"] = rb["subjects"] + items = append(items, self, binding) + } + c := obj("dns.networking.miloapis.com/v1alpha1", "DNSZoneClass", "private-coredns", "") + c["metadata"].(object)["labels"] = object{"internal-dns.datum.net/qualification": "true"} + c["spec"] = object{"controllerName": "coredns", "nameServerPolicy": object{"mode": "Static", "static": object{"servers": []string{"ns.internal."}}}} + items = append(items, c) + return items +} +func client(path string) *kubernetes.Clientset { + cfg, err := clientcmd.BuildConfigFromFlags("", path) + must(err) + cl, err := kubernetes.NewForConfig(cfg) + must(err) + return cl +} +func ipv6(cl *kubernetes.Clientset, name string) string { + p, err := cl.CoreV1().Pods(ns).Get(context.Background(), name, metav1.GetOptions{}) + must(err) + for _, ip := range p.Status.PodIPs { + addr, err := netip.ParseAddr(ip.IP) + if err == nil && addr.Is6() { + return ip.IP + } + } + panic("pod has no IPv6 address: " + name) +} +func tokenConfig(admin, internal, project, account, namespace, name string) []byte { + seconds := int64(7200) + req, err := client(admin).CoreV1().ServiceAccounts(namespace).CreateToken(context.Background(), account, &authv1.TokenRequest{Spec: authv1.TokenRequestSpec{ExpirationSeconds: &seconds}}, metav1.CreateOptions{}) + must(err) + cfg, err := clientcmd.LoadFromFile(admin) + must(err) + ctx := cfg.Contexts[cfg.CurrentContext] + cluster := *cfg.Clusters[ctx.Cluster] + subject := "system:serviceaccount:" + namespace + ":" + account + auth := &clientcmdapi.AuthInfo{Token: req.Status.Token, Impersonate: subject, ImpersonateUserExtra: map[string][]string{"iam.miloapis.com/parent-name": {project}}} + result := clientcmdapi.Config{Clusters: map[string]*clientcmdapi.Cluster{"source": &cluster}, Contexts: map[string]*clientcmdapi.Context{"source": {Cluster: "source", AuthInfo: "scoped", Namespace: projectNS}}, CurrentContext: "source", AuthInfos: map[string]*clientcmdapi.AuthInfo{"scoped": auth}} + host, err := clientcmd.Write(result) + must(err) + must(os.WriteFile(filepath.Join(out, project+"-"+name+".kubeconfig"), host, 0600)) + if internal != "" { + raw, err := clientcmd.LoadFromFile(internal) + must(err) + ic := raw.Contexts[raw.CurrentContext] + result.Clusters["source"] = raw.Clusters[ic.Cluster] + } + data, err := clientcmd.Write(result) + must(err) + return data +} +func exampleConfigs(filename string) map[string]runtime.Config { + data, err := os.ReadFile("config/internal-dns/" + filename) + must(err) + dec := k8syaml.NewYAMLOrJSONDecoder(bytes.NewReader(data), 4096) + result := map[string]runtime.Config{} + for { + var o object + err = dec.Decode(&o) + if err == io.EOF { + break + } + must(err) + if o["kind"] != "ConfigMap" { + continue + } + m := o["metadata"].(map[string]any) + d := o["data"].(map[string]any) + raw, ok := d["config.json"].(string) + if !ok { + continue + } + var cfg runtime.Config + must(json.Unmarshal([]byte(raw), &cfg)) + result[m["name"].(string)] = cfg + } + return result +} +func configMap(name string, cfg runtime.Config) object { + raw, err := json.Marshal(cfg) + must(err) + o := obj("v1", "ConfigMap", name, ns) + o["data"] = object{"config.json": string(raw)} + return o +} +func configure() { + cl := client(platform) + nodeIP := ipv6(cl, podNames[0]) + frontIP := ipv6(cl, podNames[1]) + bindIP := []string{ipv6(cl, podNames[2]), ipv6(cl, podNames[3])} + sources := obj("v1", "Secret", "internal-dns-sources", ns) + sourceData := object{} + projectRows := make([]object, 0, 2) + for i, project := range []string{"project-a", "project-b"} { + admin := []string{a, b}[i] + internal := []string{ai, bi}[i] + sourceData[project+".json"] = string(tokenConfig(admin, internal, project, "internal-dns-control-plane", ns, "worker")) + tokenConfig(admin, "", project, "dns-customer", projectNS, "customer") + tokenConfig(admin, "", project, "dns-publisher", "compute-system", "product") + tokenConfig(admin, "", project, "dns-vpc-integration", "network-system", "integration") + projectRows = append(projectRows, object{"name": project, "projectUID": project + "-uid", "namespace": projectNS, "customerKubeconfig": filepath.Join(out, project+"-customer.kubeconfig"), "productKubeconfig": filepath.Join(out, project+"-product.kubeconfig"), "integrationKubeconfig": filepath.Join(out, project+"-integration.kubeconfig"), "consumerAddress": []string{"fd53::a", "fd53::b"}[i]}) + } + sources["stringData"] = sourceData + items := make([]object, 0, 8) + items = append(items, sources) + recursor := obj("v1", "Pod", "internal-dns-public-recursor", ns) + recursor["metadata"].(object)["labels"] = object{"app": "internal-dns-public-recursor"} + recursor["spec"] = object{"automountServiceAccountToken": false, "containers": []object{{"name": "recursor", "image": image, "imagePullPolicy": "Never", "command": []string{"pdns_recursor", "--config-dir=/tmp", "--socket-dir=/tmp", "--local-address=::", "--local-port=5353", "--allow-from=fd00::/8", "--daemon=no", "--write-pid=no", "--dnssec=validate", "--security-poll-suffix="}, "securityContext": security(), "volumeMounts": []object{mount("runtime", "/tmp", false)}}}, "volumes": []object{{"name": "runtime", "emptyDir": object{}}}} + items = append(items, recursor) + cfg := exampleConfigs("control-plane.example.yaml")["internal-dns-control-plane-config"] + cfg.ClusterPrefix = "fd54::/64" + cfg.ManagedDomainSuffix = "datum.internal" + cfg.OwnershipLeaseSeconds = 10 + cfg.Stream.MaxBytes = 128 << 20 + configureProjectSources(&cfg) + cfg.NodeBackends = []model.Backend{{MemberID: "node-coredns-0", Address: "127.0.0.1", Port: 5300}} + cfg.ClusterBackends = []model.Backend{{MemberID: memberNames[2], Address: bindIP[0], Port: 5300}, {MemberID: memberNames[3], Address: bindIP[1], Port: 5300}} + for _, name := range []string{"control-a", "control-b"} { + c := cfg + c.NATS.Name = name + c.NATS.Username = name + must(c.Validate("control-plane")) + items = append(items, configMap(name+"-config", c)) + } + configs := exampleConfigs("fleet.example.yaml") + for i, key := range []string{"internal-dns-node-0-config", "internal-dns-regional-front-0-config", "internal-dns-regional-coredns-0-config", "internal-dns-regional-coredns-1-config"} { + c := configs[key] + if i == 0 { + c.Agent.Render.NodeDNSDist.Backends = cfg.NodeBackends + c.Agent.Render.NodeDNSDist.ACLs = []string{"fd00::/8", "::1/128"} + c.Agent.Render.NodeCoreDNS.ListenAddress = "127.0.0.1" + c.Agent.Render.NodeCoreDNS.ProxyPeers = []string{"127.0.0.1"} + } + if i == 1 { + c.Agent.Render.ClusterDNSDist.Backends = cfg.ClusterBackends + c.Agent.Render.ClusterDNSDist.ACLs = []string{nodeIP + "/128", frontIP + "/128", "::1/128"} + } + if i >= 2 { + c.Agent.Render.ClusterCoreDNS.ListenAddress = bindIP[i-2] + c.Agent.Render.ClusterCoreDNS.ProxyPeers = []string{frontIP, bindIP[i-2]} + c.Agent.Render.ClusterCoreDNS.PublicUpstreams = []string{"[" + publicRecursorIP(cl) + "]:5353"} + c.Agent.PublicationDNSProbe.Server = "[" + bindIP[i-2] + "]:5300" + } + must(c.Validate("agent")) + must(c.Validate("readiness")) + items = append(items, configMap(podNames[i]+"-config", c)) + } + routes := obj("v1", "Pod", "internal-dns-network-fixture", ns) + // Kind's default masquerading would erase the pod peer identity for service + // destinations outside its PodCIDR. Exempt only these two fixture prefixes; + // never widen the protected CoreDNS/front ACLs to the host's shared address. + routeCommand := fmt.Sprintf("ip -6 route replace fd53::/64 via %s; ip -6 route replace fd54::/64 via %s; for prefix in fd53::/64 fd54::/64; do while ip6tables -t nat -C POSTROUTING -d \"$prefix\" -j RETURN 2>/dev/null; do ip6tables -t nat -D POSTROUTING -d \"$prefix\" -j RETURN; done; ip6tables -t nat -I POSTROUTING 1 -d \"$prefix\" -j RETURN; done", nodeIP, frontIP) + routes["spec"] = object{"hostNetwork": true, "restartPolicy": "Never", "terminationGracePeriodSeconds": 2, "automountServiceAccountToken": false, "containers": []object{{"name": "routes", "image": image, "imagePullPolicy": "Never", "command": []string{"/bin/sh", "-ec", routeCommand}, "securityContext": object{"runAsUser": 0, "capabilities": object{"drop": []string{"ALL"}, "add": []string{"NET_ADMIN"}}}}}} + items = append(items, routes) + list("platform-configure.json", items) + // No tokens or certificate contents are included in the public contract. + write("environment.json", object{"platformKubeconfig": platform, "namespace": ns, "region": "us-central1", "shard": "shared-0", "probePod": "internal-dns-probe", "probeContainer": "probe", "probeBinary": "/dns-qualify", "controllerDeployments": []string{"control-a", "control-b"}, "brokerStatefulSet": "internal-dns-nats", "projects": projectRows, "regionalMembers": memberNames[2:], "members": memberNames, "limitations": []string{"Service-side route fixture, not Galactic VPC authorization", "Independent Kind project APIs, not Milo project discovery", "One host and one region, not independent regional failure"}}) +} + +func publicRecursorIP(cl *kubernetes.Clientset) string { + service, err := cl.CoreV1().Services(ns).Get(context.Background(), "internal-dns-public-recursor", metav1.GetOptions{}) + must(err) + return service.Spec.ClusterIP +} + +func configureProjectSources(cfg *runtime.Config) { + cfg.ProjectDiscovery = nil + cfg.Projects = []runtime.ProjectConfig{{Name: "project-a", ProjectUID: "project-a-uid", Namespace: projectNS, Kubeconfig: "/sources/project-a.json"}, {Name: "project-b", ProjectUID: "project-b-uid", Namespace: projectNS, Kubeconfig: "/sources/project-b.json"}} +} diff --git a/test/internaldns/kubernetes/render/main_test.go b/test/internaldns/kubernetes/render/main_test.go new file mode 100644 index 00000000..ece0726e --- /dev/null +++ b/test/internaldns/kubernetes/render/main_test.go @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: AGPL-3.0-only +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// Wildcard subjects must stay literal: NATS rejects JSON's HTML Unicode +// escapes even though a JSON decoder accepts them. +func TestBrokerConfigurationKeepsNATSWildcards(t *testing.T) { + root, err := filepath.Abs("../../../..") + if err != nil { + t.Fatal(err) + } + t.Chdir(root) + previous := out + out = t.TempDir() + t.Cleanup(func() { out = previous }) + bootstrap() + data, err := os.ReadFile(filepath.Join(out, "platform-bootstrap.json")) + if err != nil { + t.Fatal(err) + } + var list struct { + Items []object `json:"items"` + } + if err := json.Unmarshal(data, &list); err != nil { + t.Fatal(err) + } + for _, item := range list.Items { + if item["kind"] != "Secret" || item["metadata"].(map[string]any)["name"] != "internal-dns-nats-config" { + continue + } + cfg := item["stringData"].(map[string]any)["nats.conf"].(string) + var broker struct { + Authorization struct { + Users []json.RawMessage `json:"users"` + } `json:"authorization"` + } + if err := json.Unmarshal([]byte(cfg), &broker); err != nil { + t.Fatal(err) + } + if len(broker.Authorization.Users) != 6 { + t.Fatal("broker requires credentials for both controllers and all serving members") + } + for _, raw := range broker.Authorization.Users { + var user struct { + User string `json:"user"` + Permissions map[string]json.RawMessage `json:"permissions"` + } + if err := json.Unmarshal(raw, &user); err != nil { + t.Fatal(err) + } + if user.User == "" || user.Permissions["publish"] == nil || user.Permissions["subscribe"] == nil { + t.Fatal("broker users require scoped publish and subscribe permissions") + } + } + var top map[string]json.RawMessage + if err := json.Unmarshal([]byte(cfg), &top); err != nil { + t.Fatal(err) + } + if top["version"] != nil { + t.Fatal("broker configuration contains an unsupported version option") + } + if strings.Contains(cfg, `\u003e`) || !strings.Contains(cfg, `dns.private.serving.>`) { + t.Fatal("broker configuration escaped NATS wildcard subjects") + } + if !strings.Contains(cfg, "$JS.ACK.DNS_PRIVATE.control-plane-us-central1-shared-0-acks.>") { + t.Fatal("controller cannot acknowledge its member-ACK durable") + } + return + } + t.Fatal("broker configuration missing") +} + +// Project workers must not inherit the platform's cluster-wide artifact +// privileges when both APIs share the shipped ClusterRole. +func TestSourceWorkerBindingIsProjectScoped(t *testing.T) { + root, err := filepath.Abs("../../../..") + if err != nil { + t.Fatal(err) + } + t.Chdir(root) + scoped := false + for _, item := range sourceObjects("project-a") { + metadata := item["metadata"].(map[string]any) + switch { + case item["kind"] == "ClusterRoleBinding" && metadata["name"] == "internal-dns-control-plane": + t.Fatal("source worker inherited platform cluster-wide binding") + case item["kind"] == "RoleBinding" && metadata["name"] == "internal-dns-source-worker": + scoped = metadata["namespace"] == projectNS + + } + } + if !scoped { + t.Fatal("source worker requires project-scoped permissions") + } +} + +func TestFixtureSourcesUseIndependentProjectAPIs(t *testing.T) { + root, err := filepath.Abs("../../../..") + if err != nil { + t.Fatal(err) + } + t.Chdir(root) + cfg := exampleConfigs("control-plane.example.yaml")["internal-dns-control-plane-config"] + if cfg.ProjectDiscovery == nil { + t.Fatal("deployment example must exercise project discovery") + } + configureProjectSources(&cfg) + if cfg.ProjectDiscovery != nil || len(cfg.Projects) != 2 { + t.Fatal("fixture source selection mixes discovery and direct APIs") + } + for _, project := range cfg.Projects { + if project.Kubeconfig == "" || project.Namespace != projectNS { + t.Fatalf("fixture source scope missing: %#v", project) + } + } + if err := cfg.Validate("control-plane"); err != nil { + t.Fatalf("fixture source config rejected: %v", err) + } +} diff --git a/test/internaldns/kubernetes/reset.sh b/test/internaldns/kubernetes/reset.sh new file mode 100755 index 00000000..f4e40d9a --- /dev/null +++ b/test/internaldns/kubernetes/reset.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Reset only owned private DNS fixtures; retain the shared bootstrap/public DNS. +set -euo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +work="$root/.internal-dns-e2e/kubernetes" +# Remove only the owned fixture's fail-closed webhook before stopping its +# service, otherwise admission could prevent namespace cleanup. +for file in project-a project-b; do + cli=(kubectl --kubeconfig "$work/$file.kubeconfig") + owner=$("${cli[@]}" get namespace project-e2e -o jsonpath='{.metadata.labels.internal-dns\.datum\.net/qualification}' 2>/dev/null || true) + if [[ $owner == true ]]; then + "${cli[@]}" delete clusterrolebinding internal-dns-control-plane --ignore-not-found + webhook_owner=$("${cli[@]}" get validatingwebhookconfiguration internal-dns-e2e -o jsonpath='{.metadata.labels.internal-dns\.datum\.net/qualification}' 2>/dev/null || true) + if [[ -n $webhook_owner && $webhook_owner != true ]]; then + echo 'Refusing to delete an unowned admission webhook' >&2 + exit 1 + fi + "${cli[@]}" delete validatingwebhookconfiguration internal-dns-e2e --ignore-not-found + fi +done +for file in platform project-a project-b; do + cli=(kubectl --kubeconfig "$work/$file.kubeconfig") + namespace=project-e2e + if [[ $file == platform ]]; then namespace=internal-dns-system; fi + if "${cli[@]}" get namespace "$namespace" >/dev/null 2>&1; then + owner=$("${cli[@]}" get namespace "$namespace" -o jsonpath='{.metadata.labels.internal-dns\.datum\.net/qualification}') + if [[ $owner != true ]]; then + echo "Refusing to reset unowned namespace $namespace on $file" >&2 + exit 1 + fi + if [[ $file != platform ]]; then + # The fixture may have deliberately stopped both controllers. Release + # only the private-zone finalizers in this owned test namespace. + while IFS= read -r zone; do + [[ -z $zone ]] || "${cli[@]}" -n "$namespace" patch "$zone" --type merge -p '{"metadata":{"finalizers":[]}}' + done < <("${cli[@]}" -n "$namespace" get dnszones -o name) + fi + "${cli[@]}" delete namespace "$namespace" --wait=true --timeout=90s + fi +done diff --git a/test/internaldns/production-parity.md b/test/internaldns/production-parity.md new file mode 100644 index 00000000..f950b751 --- /dev/null +++ b/test/internaldns/production-parity.md @@ -0,0 +1,191 @@ +# Production-oriented internal DNS qualification + +## Proposal + +Build deployment qualification on the existing `datum-cloud/test-infra` +environment. Use the Task lifecycle, Kubernetes workloads, and Go clients for +assertions and fault injection. CI also runs the +existing public DNS Chainsaw suites on the same foundation. + +The acceptance path starts with a product publishing a record in a project +control plane and ends with a workload querying the well-known resolver address +inside its Galactic VPC. A successful direct query to a generated context +address does not qualify the consumer network path. + +## Evidence and gaps + +This assessment uses tracked infrastructure declarations at +[`datum-cloud/infra@5cc5caba5`](https://github.com/datum-cloud/infra/tree/5cc5caba5b7f886ca00e7c3ed90726834dac7fac). +Declarations establish the intended configuration; they do not establish live +cluster health or that the proposed private DNS fleet is deployed. + +- [DNS control-plane configuration](https://github.com/datum-cloud/infra/tree/5cc5caba5b7f886ca00e7c3ed90726834dac7fac/apps/dns-operator/control-plane/base) + discovers Milo projects and uses project credentials, certificate trust, and + an admission service. The Kubernetes qualification substitutes independent + Kind APIs, but uses a TLS admission Service and scoped worker credentials. +- [Karmada](https://github.com/datum-cloud/infra/blob/5cc5caba5b7f886ca00e7c3ed90726834dac7fac/infrastructure/karmada/host/base/karmada-helm-release.yaml) + provides a separate federation API. The current lab has no federation or + edge API and creates DNS access intent directly. +- [Galactic edge configuration](https://github.com/datum-cloud/infra/blob/5cc5caba5b7f886ca00e7c3ed90726834dac7fac/channels/edge/components/galactic/galactic-system.yaml) + selects the networking deployment independently. The lab uses synthetic routes + and direct context addresses; it bypasses VPC attachment, private service + authorization, service route programming, and workload resolver settings. +- [NATS infrastructure](https://github.com/datum-cloud/infra/blob/5cc5caba5b7f886ca00e7c3ed90726834dac7fac/infrastructure/databases/nats/base/nats-hr.yaml) + declares three broker replicas, persistent JetStream storage, and client and + route TLS. The Kubernetes lab uses that three-replica storage and TLS pattern. + Its full broker outage test proves local serving continuity and replica restart using + retained PVCs. Quorum loss, disk/host-loss recovery, and regional transport + still need dedicated scenarios. +- [Public edge DNS](https://github.com/datum-cloud/infra/tree/5cc5caba5b7f886ca00e7c3ed90726834dac7fac/apps/dns-operator/downstream/edge) + uses PowerDNS authoritative servers and Lightningstream. Its loopback + recursor supports ALIAS expansion. Reuse that public regression environment; + add the proposed private dnsdist/CoreDNS fleet alongside it. The public recursor + does not establish private tenant isolation. + +The existing [Task environment](../../Taskfile.yaml) already deploys public DNS +across upstream, control, and edge clusters with shared test-infra tooling. +The [CI workflow](../../.github/workflows/e2e.yml) exercises both private DNS +qualification and that public chain. The shared Task include is pinned to +`datum-cloud/test-infra@7622c925d5bcf2fa5bc01823775973cf6daf51ca`. + +## Target topology + +Use three local Kubernetes clusters: management and two edge clusters. Host +the Milo core and Karmada APIs in the management cluster. Exercise two consumer +project APIs and the DNS service's managed project through Milo's project +control-plane routing and distinct credentials; namespaces alone cannot prove +those boundaries. The current independent-source Kind APIs are a distributed +control-plane fixture, rather than the final deployment topology. + +```mermaid +flowchart LR + subgraph A[Project A control plane] + CA[Compute publisher A] --> DA[Project DNS APIs] + end + subgraph B[Project B control plane] + CB[Compute publisher B] --> DB[Project DNS APIs] + end + subgraph M[Management cluster] + CORE[Milo core API / project discovery] + DNS[DNS service managed project API] + CP[Two DNS control-plane replicas] + INT[Trusted VPC / DNS integration] + JS[Three-node TLS JetStream / persistent volumes] + K[Karmada federation API] + end + DA & DB --> CP + CORE --> CP + CP <--> DNS + CP --> JS + K --> E1 & E2 + INT -->|Access intent| DA & DB + subgraph E1[Edge A] + V1[Consumer VPC A workload] + F1[Private endpoint / well-known resolver] + G1[Galactic ServiceRoutePolicy A] + V1 --> F1 --> G1 + end + subgraph E2[Edge B] + V2[Consumer VPC B workload] + F2[Private endpoint / same resolver address] + G2[Galactic ServiceRoutePolicy B] + V2 --> F2 --> G2 + end + subgraph SV[Producer DNS service VPC] + EP1[ServiceEndpoint / node listener A] + EP2[ServiceEndpoint / node listener B] + N1[Edge A shared node dnsdist / CoreDNS] + N2[Edge B shared node dnsdist / CoreDNS] + R[Shared regional dnsdist / two CoreDNS members] + EP1 --> N1 --> R + EP2 --> N2 --> R + end + G1 --> EP1 + G2 --> EP2 + INT -->|Private service authorization| K + JS --> N1 & N2 & R +``` + +The two edge clusters model separate edge control planes on one host in one region. +The DNS service runs in its own producer VPC. Both node and regional resolver +tiers belong to that service VPC; node members run on shared edge capacity, +not in consumer VPCs. Galactic `ServiceEndpoint` resources select the producer +listeners. Consumer `ServiceRoutePolicy` resources expose those listeners as +private endpoints at the well-known resolver address in each consumer VPC. +Galactic owns VPC authorization and destination translation. The DNS integration +publishes short-lived access intent through the DNS APIs; DNS workers do not +watch Galactic resources. Queries carry the authorized service destination to +dnsdist, which supplies PROXYv2 only across protected serving links. Clients +cannot choose another tenant's destination or supply a trusted PROXYv2 header. + +Keep fleet size fixed when adding VPCs or zones. Two contexts must share the +same node and regional workloads, with independent regional member storage and +identities. Deploy controllers, agents, dnsdist, and CoreDNS as real +pods from the runtime package, with the same reload and fail-closed helpers +intended for staging. Do not mount the Docker socket into those workloads. + +## Qualification stages + +1. **Package gate (implemented).** Build the runtime image through + `config/internal-dns/Dockerfile`, smoke-test its executable as the image's + non-root user, render all CRDs and deployment examples with the installer's + pinned Kustomize, and validate their JSON + configuration, CRD schemas installed in an RBAC-enabled test API, Kubernetes + permissions, and NATS subject permissions in CI. + This gate checks packaging; the examples still require deployment overlays + and serving images. It makes no live Kubernetes or consumer-path claim. +2. **Kubernetes deployment (implemented).** Reuse pinned test-infra, serving + images and command helpers, TLS admission, scoped worker/product credentials, + two controllers, three TLS brokers, and persistent serving state. Run + publication, isolation, takeover, and outage checks against real pods. ACK + timing is 30/90 seconds; ownership lease and termination grace are shortened + to 10 and 2 seconds. CI runs the public regression chain in the same job. + Publication resumes after broker/controller restoration using retained + broker PVCs. Member restart, stale-envelope replay, broker quorum, and API-partition fault + scenarios remain to be added to this deployment suite. +3. **Consumer network path.** Add real Milo project discovery and the DNS + service managed project, Karmada and edge resource propagation, Galactic + VPCs, the separate producer DNS service VPC, its `ServiceEndpoint` listeners, + consumer private endpoints through `ServiceRoutePolicy`, and the trusted + access integration. Run Go publisher jobs with product credentials and query from + attached workloads without manually programming context routes. Also test + an actual feature-flagged Compute build; a publisher fixture alone does not + qualify the product integration. +4. **Staging and regional resilience.** Reuse those overlays in staging-lab + for actual Compute attachments and edge networking. Add a second region only + after defining and deploying its broker and delivery topology; a second edge + cluster connected to the same broker is not regional replication proof. + +## Acceptance and evidence + +Each stage must retain the exact Git commits, rendered manifests, image digests, +topology, non-secret identity metadata, observed resource versions, failure +timestamps, DNS responses, and pod logs. Mark each scenario as passed, failed, +or not exercised. Do not count a skipped integration as a passing check. + +Required deployment and consumer-path scenarios: + +- Overlapping zones, names, and endpoint addresses in two VPCs return distinct + answers through private endpoints into the same producer DNS service VPC over + UDP and TCP, including positive and negative cache isolation. +- Unauthorized access, forged destinations and PROXYv2 headers, and a withdrawn + service route cannot return another VPC's records. Product credentials cannot + configure resolver access. +- Endpoint health withdrawal and recovery, deletion, source API loss, and + stale-observation replay preserve current source fences and configured TTLs. + Records remain available during control-plane outages until an explicit update + withdraws them. +- Either controller can take ownership without stale publication; a healthy + project continues publishing while another project's API is unavailable. +- Broker leader loss retains publication with quorum. Agent restart during + quorum loss restores verified checkpoints without broker access. Member and node + failures stop unsafe local serving through independent health checks. +- Certificate rotation, webhook unavailability, rolling member replacement, + and PVC reuse do not bypass admission or regress checkpoint authority. +- Adding contexts leaves the workload count fixed, and public authoritative + DNS still resolves its test records without exposing private zones. + +Kind cannot qualify physical fabric, microVM attachment, disk durability across +host loss, production scale, or independent regional failure. Staging-lab owns +those checks. Capacity tests must measure reload/proof latency, query continuity, +and incomplete-activation recovery before increasing shard limits. diff --git a/test/internaldns/qualify-package.sh b/test/internaldns/qualify-package.sh new file mode 100755 index 00000000..423d5dd6 --- /dev/null +++ b/test/internaldns/qualify-package.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Qualify the shipped runtime artifact without claiming deployment parity. +set -euo pipefail + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +cd "$root" +evidence_dir=${INTERNAL_DNS_PACKAGE_EVIDENCE:-test/internaldns/results/package} +mkdir -p "$evidence_dir" +evidence_dir=$(cd "$evidence_dir" && pwd) + +# The RBAC test skips without a real API server. Require its binaries rather +# than accepting that skip as deployment qualification. +if [[ -z ${KUBEBUILDER_ASSETS:-} ]]; then + make setup-envtest > "$evidence_dir/envtest.log" 2>&1 + k8s_version=$(go list -m -f '{{.Version}}' k8s.io/api | awk -F'[v.]' '{printf "1.%d", $3}') + KUBEBUILDER_ASSETS=$(bin/setup-envtest use "$k8s_version" --bin-dir bin -p path) + export KUBEBUILDER_ASSETS +fi +# go test runs from the package directory, not this script's repository root. +# setup-envtest can return a relative path when --bin-dir is relative. +KUBEBUILDER_ASSETS=$(cd "$KUBEBUILDER_ASSETS" && pwd) +export KUBEBUILDER_ASSETS +test -x "$KUBEBUILDER_ASSETS/kube-apiserver" +test -x "$KUBEBUILDER_ASSETS/etcd" +go test ./internal/internaldns/runtime \ + -run '^(TestDeploymentExampleConfigsValidate|TestDeploymentRolesAuthorizeWorkerAndDiscoveryBoundaries|TestNATSExampleUsesExactMemberAckSubjects|TestEveryMemberCanCreateAllRuntimeConsumers|TestReadOnlyServingImageHasWritableCoreDNSWorkingDirectory)$' \ + -count=1 -v > "$evidence_dir/contracts.log" 2>&1 +# Use the same pinned Kustomize binary as installation and the public E2E job. +make kustomize > "$evidence_dir/kustomize.log" 2>&1 +bin/kustomize build config/crd > "$evidence_dir/crds.yaml" +crd_files=(config/crd/bases/*.yaml) +test "$(grep -c '^kind: CustomResourceDefinition$' "$evidence_dir/crds.yaml")" -eq "${#crd_files[@]}" +bin/kustomize build config/internal-dns > "$evidence_dir/deployment-examples.yaml" + +# Run by immutable local image ID, so an unrelated tag cannot replace the +# artifact between build and verification. Docker selects the host architecture. +docker build --file config/internal-dns/Dockerfile \ + --iidfile "$evidence_dir/image.id" . > "$evidence_dir/build.log" 2>&1 +image_id=$(cat "$evidence_dir/image.id") +docker image inspect "$image_id" > "$evidence_dir/image.json" +test "$(docker image inspect --format '{{.Config.User}}' "$image_id")" = '65532:65532' +docker run --rm --network none --read-only --cap-drop ALL \ + --security-opt no-new-privileges "$image_id" --help \ + > "$evidence_dir/help.log" 2>&1 +# Confirm this is the internal DNS entry point rather than an image that only +# happens to exit successfully. +grep -q 'Process role: control-plane, agent, serving, readiness, or prepare' "$evidence_dir/help.log" +git rev-parse HEAD > "$evidence_dir/source.commit" +git status --porcelain > "$evidence_dir/source.status" +printf 'Internal DNS package qualification passed; evidence: %s\n' "$evidence_dir" diff --git a/test/internaldns/run.sh b/test/internaldns/run.sh new file mode 100755 index 00000000..cc910f90 --- /dev/null +++ b/test/internaldns/run.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Task entry point for the shared Kubernetes qualification. +set -euo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +cd "$root" +if (( $# )); then + echo 'Use Task env:internal-dns:up / env:internal-dns:qualify; this wrapper takes no arguments.' >&2 + exit 2 +fi +task --yes env:internal-dns:up +task --yes env:internal-dns:qualify