diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 052276cf..ba9996d9 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -43,3 +43,53 @@ jobs: # release would not even match each other. image-overlays: config/manager,config/agent,config/components/dns-mcp secrets: inherit + + publish-internal-dns-runtime: + permissions: + id-token: write + contents: read + packages: write + attestations: write + uses: datum-cloud/actions/.github/workflows/publish-docker.yaml@v1.14.0 + with: + image-name: internal-dns + dockerfile-path: config/internal-dns/Dockerfile + target: runtime + platforms: linux/amd64,linux/arm64 + secrets: inherit + + publish-internal-dns-serving: + permissions: + id-token: write + contents: read + packages: write + attestations: write + uses: datum-cloud/actions/.github/workflows/publish-docker.yaml@v1.14.0 + with: + image-name: internal-dns-serving + dockerfile-path: config/internal-dns/Dockerfile + target: serving + platforms: linux/amd64,linux/arm64 + secrets: inherit + + publish-internal-dns-builds: + if: github.event_name == 'release' || (github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/fix/internal-dns-name-references' || startsWith(github.ref, 'refs/tags/'))) + needs: [publish-internal-dns-runtime, publish-internal-dns-serving, publish-kustomize-bundles] + permissions: + contents: read + packages: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v4 + - uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: fluxcd/flux2/action@v2.7.5 + - name: Publish the runtime and serving image pair + env: + RUNTIME_TAG: ${{ needs.publish-internal-dns-runtime.outputs.tag }} + SERVING_TAG: ${{ needs.publish-internal-dns-serving.outputs.tag }} + run: hack/publish-internal-dns-channel.sh diff --git a/cmd/internal-dns-coredns/main.go b/cmd/internal-dns-coredns/main.go new file mode 100644 index 00000000..b87dbcdc --- /dev/null +++ b/cmd/internal-dns-coredns/main.go @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package main + +import ( + "context" + "flag" + "fmt" + "os" + "os/signal" + "syscall" + + "runtime/debug" + + "go.miloapis.com/dns-operator/internal/internaldns/coredns" +) + +func main() { + flags := flag.NewFlagSet("coredns", flag.ExitOnError) + var path string + var validate, version bool + flags.StringVar(&path, "config", "", "Absolute path to the DNS context snapshot") + flags.BoolVar(&validate, "validate", false, "Validate the DNS context snapshot and exit") + flags.BoolVar(&version, "version", false, "Print the CoreDNS version and exit") + if err := flags.Parse(os.Args[1:]); err != nil { + os.Exit(2) + } + if version { + version := "unknown" + if info, ok := debug.ReadBuildInfo(); ok { + for _, dep := range info.Deps { + if dep.Path == "github.com/coredns/coredns" { + version = dep.Version + } + } + } + fmt.Printf("CoreDNS %s; dnscontext plugin\n", version) + return + } + var err error + if validate { + _, err = coredns.Load(path) + } else { + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + err = coredns.Run(ctx, path) + } + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "CoreDNS: %v\n", err) + os.Exit(1) + } +} diff --git a/cmd/internal-dns/main.go b/cmd/internal-dns/main.go new file mode 100644 index 00000000..41a98c9a --- /dev/null +++ b/cmd/internal-dns/main.go @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package main + +import ( + "flag" + "fmt" + "os" + + internruntime "go.miloapis.com/dns-operator/internal/internaldns/runtime" + _ "k8s.io/client-go/plugin/pkg/client/auth" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +func main() { + var role, configPath, kubeconfig string + flag.StringVar(&role, "role", "", "Process role: control-plane, agent, serving, readiness, or prepare") + flag.StringVar(&configPath, "config", "", "Absolute path to the internal DNS JSON configuration") + // controller-runtime/client-go may install this standard flag from an init + // hook. Reuse it when present so the command keeps one stable contract. + if flag.Lookup("kubeconfig") == nil { + flag.StringVar(&kubeconfig, "kubeconfig", "", "Control-plane Kubernetes configuration; ignored by serving processes") + } + logOptions := zap.Options{Development: false} + logOptions.BindFlags(flag.CommandLine) + flag.Parse() + if existing := flag.Lookup("kubeconfig"); existing != nil { + kubeconfig = existing.Value.String() + } + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&logOptions))) + + cfg, err := internruntime.LoadConfig(configPath) + if err == nil { + err = internruntime.Run(ctrl.SetupSignalHandler(), role, cfg, kubeconfig) + } + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "internal-dns: %v\n", err) + os.Exit(1) + } +} diff --git a/config/internal-dns/Dockerfile b/config/internal-dns/Dockerfile new file mode 100644 index 00000000..1c86bf29 --- /dev/null +++ b/config/internal-dns/Dockerfile @@ -0,0 +1,39 @@ +# syntax=docker/dockerfile:1.7 + +FROM --platform=$BUILDPLATFORM golang:1.26 AS build +ARG TARGETOS=linux +ARG TARGETARCH +WORKDIR /src +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/pkg/mod go mod download +COPY cmd/internal-dns ./cmd/internal-dns +COPY cmd/internal-dns-coredns ./cmd/internal-dns-coredns +COPY api ./api +COPY internal ./internal +RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH \ + go build -trimpath -tags netgo,osusergo -ldflags='-s -w' \ + -o /out/internal-dns ./cmd/internal-dns && \ + CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -trimpath -o /out/coredns ./cmd/internal-dns-coredns + +FROM scratch AS runtime +COPY --from=build /out/internal-dns /internal-dns +USER 65532:65532 +ENTRYPOINT ["/internal-dns"] + +FROM powerdns/dnsdist-20:2.0.10@sha256:24ac720af997aff6b98cb5b50d748754f815428e07b5368dfb0eccd5aa8d07e8 AS serving +USER 0:0 +RUN apt-get update && apt-get install --no-install-recommends -y \ + pdns-recursor iproute2 procps netcat-openbsd ca-certificates bash && \ + rm -rf /var/lib/apt/lists/* +COPY --from=build /out/internal-dns /internal-dns +COPY --from=build /out/coredns /usr/local/bin/coredns +COPY config/internal-dns/commands/ /usr/local/sbin/ +USER 65532:65532 +RUN for tool in bash ip nc coredns dnsdist pdns_recursor; do \ + command -v "$tool" >/dev/null || exit 1; \ + done && test -x /internal-dns && coredns -version +RUN for helper in /usr/local/sbin/internal-dns-*; do \ + test -x "$helper" && bash -n "$helper" || exit 1; \ + done +ENTRYPOINT ["/internal-dns"] diff --git a/config/internal-dns/Dockerfile.dockerignore b/config/internal-dns/Dockerfile.dockerignore new file mode 100644 index 00000000..5a873981 --- /dev/null +++ b/config/internal-dns/Dockerfile.dockerignore @@ -0,0 +1,6 @@ +** +!**/*.go +**/*_test.go +!go.mod +!go.sum +!config/internal-dns/commands/** diff --git a/config/internal-dns/README.md b/config/internal-dns/README.md new file mode 100644 index 00000000..5449bc3f --- /dev/null +++ b/config/internal-dns/README.md @@ -0,0 +1,235 @@ +# Internal DNS regional fleet deployment + +These manifests show the process boundaries for one shared regional shard. +Replace the example images, addresses, certificates, storage classes, admission +CA injection, and site command wrappers before deployment. + +CI publishes `ghcr.io/datum-cloud/internal-dns` for the control plane and +`ghcr.io/datum-cloud/internal-dns-serving` for fleet agents and CoreDNS. dnsdist runs directly from its upstream image. +The serving image includes the validation, activation, and gating +commands in `config/internal-dns/commands`. The Kubernetes qualification +environment uses the same commands. + +After both images and the API bundle publish, CI publishes +`oci://ghcr.io/datum-cloud/internal-dns-builds` with the same version tag. +This Flux artifact contains the `internal-dns-images` ConfigMap with both +image manifest digests and the source revision. Staging selects timestamped +feature-branch tags and uses these immutable image references. + +The fixed fleet has three roles: + +- A `node` resolver agent configures the shared node dnsdist/CoreDNS pair. +- A `regional-dnsdist` resolver agent configures the shared regional frontend and its + pool of regional CoreDNS members. +- Two or more `regional-coredns` combined agents independently compile the complete + DNS context snapshot for their member, validate it with + `coredns -validate`, load it atomically into the context plugin, prove the + exact version over UDP and TCP, and publish a leased ACK. + +Contexts, access bindings, and zones add destinations and record data to +these processes. They never create workloads per VPC or zone. Use `node`, +`regional-dnsdist`, and `regional-coredns` roles for the multi-member production +layout. The `cluster` and `both` roles are limited to development. + +Each member needs a distinct member ID, NATS identity and durable consumers, +checkpoint PVC, configuration PVC, and durable activation state. Treat that +set as a single-writer unit. A rolling update must allocate a new member +identity and storage; do not run two pods against one checkpoint. + +Kubernetes manages the dnsdist process. An init container creates a private, +pod-local control key and an initial configuration that refuses every context. +The agent validates complete routing snapshots and replaces the rule chain over +the authenticated loopback console. Failed validation retains active rules. +Backend eligibility uses immutable pools shared by identical member sets; +context identity remains the original trusted destination. Listener or backend +topology changes require a replacement member with independent storage. Agent and broker outages retain applied +routing. The control key is generated locally and is never a DNS API field. + +`routers.example.yaml` supplies the node and regional frontend workloads. +`fleet.example.yaml` supplies their configurations and the regional CoreDNS +workload template. Give every member independent configuration and checkpoint +storage. CoreDNS process activation uses the serving role; dnsdist runs directly +as its container process. + +The agent runs as UID 65532; CoreDNS and dnsdist run as UID 65533. Query +containers do not mount broker credentials and cannot inspect the agent's +filesystem through the shared process namespace. Group 65532 can read accepted +configuration, checkpoints, and the pod-local console key. Only agents that +manage CoreDNS receive `KILL` to stop query processes during activation; no +container receives `SYS_PTRACE`. + +## Control-plane and API boundaries + + +Control-plane replicas discover ready Milo projects through the core API and +watch every namespace in each project control plane. New projects are included +without restarting the service. `projectDiscovery.kubeconfig` authenticates +discovery and project API access; the process `--kubeconfig` authenticates the +DNS service project. Generated kubeconfig Secrets use the `value` key. +Unavailable discovery retains active sources and applied DNS configuration. +Compilation requires the live source project identity to match its worker. +Admission resolves the authenticated project parent against its live identity, +and acknowledgement projection follows the discovered source clients. + +Control-plane replicas can concurrently watch the platform API and source +projects. Publication and shard ownership use resource-version checks and +increasing writer epochs on takeover. Each source project has an authenticated +client and optional request limits. An unavailable source must not prevent ACK +projection or publication for healthy sources. + +DNS always consumes `DNSResolverContext` and `DNSResolverAccessBinding` from +the project API. It does not watch a networking API. A trusted integration +principal bound to `internal-dns-vpc-integration` creates a context and writes +regional access intent. RBAC separates network integration, product publication, and DNS controller +permissions. Product publishers cannot modify resolver access. +DNS pins the context and access lifetimes in the internal `DNSResolverBinding` +and generates mapping versions. Access persists until an explicit withdrawal. + +The context directly owns its requested managed zone and association. +Compute discovers the context status and publishes through +`DNSRegistration` and `DNSRecordContribution`. It does not choose a zone for the managed-name path and receives no NATS credentials. +Publishers write addresses and eligibility together in contribution spec using +optimistic concurrency. DNS owns status and uses API-assigned generations to +reject stale desired state across takeover and replay. + +## Serving identity and DNS contexts + +Consumer and regional listener prefixes must be local routes in their dnsdist +network namespaces. Galactic authorizes and translates the consumer path to a +service-side destination. dnsdist preserves that destination in a trusted +PROXYv2 header. Node CoreDNS selects a context and forwards to its regional +destination. Regional dnsdist selects the same context and load-balances the +healthy CoreDNS members. Every regional CoreDNS member hosts the context's +private zones. Public queries forward to a shared PowerDNS Recursor service. + +An owned name without eligible answers carries service-only `TYPE65280` +metadata at that exact name. This private-use record preserves NODATA without +adding a child name or conflicting with a recovered CNAME. Tenant declarations +cannot use that type. Dynamic registrations require TTLs from 1 to 30 seconds; +client caches can retain withdrawn answers for at most that TTL. Static records +retain their explicit RRset TTL, or the default of 300 seconds. Entries in one +static RRset must use the same TTL. + +A context without private zones proves recursion with the root SOA over UDP and +TCP. `verifier.publicName` can override this probe. CoreDNS members need a writable +`/tmp` runtime volume even when the image root filesystem is read-only. + +Private packet caching stays disabled in dnsdist and CoreDNS. The context plugin +answers private queries from the accepted in-memory zone data. Endpoint changes +replace that data without retaining cached private answers. Each context has a +separate public cache. PowerDNS Recursor validates DNSSEC for public queries. +Unknown destinations are rejected before cache lookup. + +Regional publication proof connects directly to that member's CoreDNS listener, +supplies the exact context destination in a protected PROXYv2 header over both +UDP and TCP, and verifies the installed SOA serial and fingerprint. Restrict +`proxyPeers` to regional dnsdist and the member's local publication agent +network. Local health probes also require the CoreDNS listener address in +`proxyPeers` and loopback in dnsdist ACLs. These peers must remain inside the +protected serving network. + +## Transactions, health, and outage recovery + +Each attached private apex has independent readiness. A pending zone returns +SERVFAIL while published zones continue serving. Its names cannot fall through +to public recursion or cached answers. If the private apex is unknown, the +context stays gated until its configuration is complete. Completing publication +restores service with the same access mapping. Resolver access readiness +reports the applied path independently of each zone's publication status. Explicit access withdrawal +requires a newer access version to reactivate. + +The agent stages a complete JSON snapshot containing context destinations, +private zone data, and upstream destinations. The CoreDNS validator parses every +zone before installation. Atomic rename exposes one complete snapshot; the +plugin builds its replacement state before swapping the lookup table. Invalid +updates retain the accepted state. A record update does not restart CoreDNS. +The agent proves the installed publication over UDP and TCP before acknowledging it. + +Publication acknowledgements report verified configuration versions. The fleet +retains accepted proofs during control plane or broker outages and continues +using local DNS probes and backend health checks. Status can report degraded +update delivery without withdrawing an applied resolver binding. + +Records and access bindings have no expiry deadline. Product services withdraw +endpoints through explicit health updates or deletion. When delivery stops, DNS +retains the last reported endpoint state. Agents restore applied configuration +from persistent checkpoints before waiting for messaging connectivity. New +nodes require a complete configuration before accepting queries. Replay cannot +regress writer or configuration versions or restore a withdrawn lifetime. + +Readiness verifies the approved checkpoint and installed files, then queries +local DNS daemons over UDP and TCP. Healthy daemons continue through normal +agent stops and restarts. Staging and validation can continue while approved +configuration serves. A rejected expansion retains that configuration; a +withdrawal that cannot be applied gates serving. The serving supervisor accepts +approved configuration independently of the agent. Incomplete installation +keeps a durable activation marker; candidate verification requires a live +transaction lock until the approved checkpoint is committed. Orphaned +activation remains gated until recovery verifies the saved configuration. +Readiness and activation gates use local state without Kubernetes or NATS. + +Mount the transaction stage, active CoreDNS configuration, checkpoint, +and activation state on storage local to the serving host. Replication uses +the durable publication stream and independent regional members; each member +has its own crash-safe checkpoint volume. + +The transaction stage and active configuration must share one filesystem: +installation uses atomic rename. The examples put the stage at +`/etc/internal-dns/stage` on the configuration PVC and keep checkpoints on the +separate state PVC. + +## Capacity and transport + +Set `maxBindings` from measured CoreDNS context memory, snapshot activation time, +dnsdist rule cost, NATS payload size, and activation recovery time. The example caps a +shard at 200 contexts. Add another fixed shard before +raising those bounds. + +NATS uses TLS with a verified server name, client certificate, and one scoped +identity per process. Control-plane credentials publish serving/publication +subjects and consume ACKs. Each serving member can consume its assigned durable +subjects and publish only its own hashed ACK subject. Broker partitions delay +updates while the fleet continues serving its applied configuration. + +`fleet.example.yaml` provides node, regional frontend, and regional CoreDNS JSON +configurations plus one Deployment template. Instantiate the template for each +member and create a second regional CoreDNS member with distinct addresses and +storage. `control-plane.example.yaml` lists both regional members in +`clusterBackends` and `members`. + +## Resource access + +The project API server authorizes DNS resource and status operations. Bind the +roles in `rbac.yaml` to service principals in each project namespace. Customer +access uses the protected resources and DNS roles in `config/iam`. + +The webhook validates live resource relationships. Project RBAC controls who +can publish; the compiler validates registration ownership, record types, and +observation versions. Publishers preserve DNS-owned publication status during +optimistically locked observation updates. + +## Rollout and recovery + +Project APIs select zones by name and report names and readiness. DNS stores +resolved ownership identities in reference ConfigMaps in the service project. +The collector checks source resources before removing those records. + +The supported envelope version is `v1alpha6` and checkpoint format is 7. Keep +reference records, transport state, and persistent checkpoints across compatible upgrades. Deploy +serving agents before controllers that emit per-zone pending state. Enable the +controller change after every serving member supports that contract; a matching +wire version alone does not establish feature compatibility. + +The checkpoint retains desired version fences and the activated configuration +separately. Restart can restore the activated configuration during a rejected +expansion or broker outage. Explicit withdrawal invalidates that fallback before +installation. Unknown checkpoint or wire formats preserve files and reject +activation. Format conversion requires a tested migration that retains lifetime, +withdrawal, configuration, and publisher fences; never reset those fences to +complete an upgrade. + +Snapshot budgets admit established bindings before new allocations. Overflow +reports `CapacityAvailable=False` for the affected binding while healthy +contexts and withdrawals continue. A pending zone fails only queries beneath +its private apex; its names never fall through to public recursion. Bindings, +publication targets, and context status use the same stable shard assignment. diff --git a/config/internal-dns/admission.example.yaml b/config/internal-dns/admission.example.yaml new file mode 100644 index 00000000..6d08e766 --- /dev/null +++ b/config/internal-dns/admission.example.yaml @@ -0,0 +1,32 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: internal-dns-admission + annotations: + cert-manager.io/inject-ca-from: internal-dns-system/internal-dns-admission-tls +webhooks: +- name: validation.internal-dns.miloapis.com + admissionReviewVersions: [v1] + sideEffects: None + failurePolicy: Fail + matchPolicy: Equivalent + timeoutSeconds: 5 + clientConfig: + service: + namespace: internal-dns-system + name: internal-dns-admission + path: /validate-internal-dns + port: 443 + rules: + # Live DNS references require service validation. + # The API server authorizes resource and status operations. + - apiGroups: [dns.networking.miloapis.com] + apiVersions: [v1alpha1] + operations: [CREATE, UPDATE] + resources: + - dnszoneassociations + - dnsregistrations + - dnsrecordcontributions + - dnsnamingpolicies + - dnsresolveraccessbindings + scope: Namespaced diff --git a/config/internal-dns/commands/internal-dns-gate b/config/internal-dns/commands/internal-dns-gate new file mode 100755 index 00000000..68862c6b --- /dev/null +++ b/config/internal-dns/commands/internal-dns-gate @@ -0,0 +1,20 @@ +#!/bin/sh +set -eu +case "$1" in + node) config=/etc/internal-dns/node-dnsdist.conf; port=5199 ;; + regional-front) config=/etc/internal-dns/cluster-dnsdist.conf; port=5299 ;; + regional-coredns) config= ;; + *) echo 'unknown fleet role' >&2; exit 2 ;; +esac +if [ -n "$config" ] && nc -z -w 1 127.0.0.1 "$port" >/dev/null 2>&1; then + output=$(timeout 5s dnsdist -C "$config" -c -e 'setRules({newRuleAction(AllRule(),RCodeAction(DNSRCode.SERVFAIL))}); return "INTERNAL_DNS_GATED"') + [ "$output" = 'INTERNAL_DNS_GATED' ] || { printf '%s\n' "$output" >&2; exit 1; } +fi +rm -f /run/internal-dns/allow-coredns +pkill -TERM -x coredns || true +for attempt in $(seq 1 40); do + if ! pgrep -x coredns >/dev/null; then exit 0; fi + sleep 0.05 +done +pkill -KILL -x coredns || true +if pgrep -x coredns >/dev/null; then echo 'CoreDNS did not stop' >&2; exit 1; fi diff --git a/config/internal-dns/commands/internal-dns-reload-coredns b/config/internal-dns/commands/internal-dns-reload-coredns new file mode 100755 index 00000000..a00050af --- /dev/null +++ b/config/internal-dns/commands/internal-dns-reload-coredns @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +touch /run/internal-dns/allow-coredns diff --git a/config/internal-dns/commands/internal-dns-reload-dnsdist b/config/internal-dns/commands/internal-dns-reload-dnsdist new file mode 100755 index 00000000..717bd223 --- /dev/null +++ b/config/internal-dns/commands/internal-dns-reload-dnsdist @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +case "$1" in + node) config=/etc/internal-dns/node-dnsdist.conf ;; + regional) config=/etc/internal-dns/cluster-dnsdist.conf ;; + *) echo 'unknown dnsdist role' >&2; exit 2 ;; +esac +output=$(timeout 5s dnsdist -C "$config" -c -e "dofile(\"$config\"); return \"INTERNAL_DNS_APPLIED\"") +[ "$output" = 'INTERNAL_DNS_APPLIED' ] || { printf '%s\n' "$output" >&2; exit 1; } diff --git a/config/internal-dns/commands/internal-dns-validate-coredns b/config/internal-dns/commands/internal-dns-validate-coredns new file mode 100755 index 00000000..3a47c06d --- /dev/null +++ b/config/internal-dns/commands/internal-dns-validate-coredns @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +exec coredns -validate -config "$2" diff --git a/config/internal-dns/commands/internal-dns-validate-dnsdist b/config/internal-dns/commands/internal-dns-validate-dnsdist new file mode 100755 index 00000000..77e1f323 --- /dev/null +++ b/config/internal-dns/commands/internal-dns-validate-dnsdist @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +INTERNAL_DNS_VALIDATE=1 exec dnsdist --check-config -C "$2" diff --git a/config/internal-dns/control-plane.example.yaml b/config/internal-dns/control-plane.example.yaml new file mode 100644 index 00000000..58162d92 --- /dev/null +++ b/config/internal-dns/control-plane.example.yaml @@ -0,0 +1,127 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: internal-dns-control-plane-config + namespace: internal-dns-system +data: + config.json: | + { + "region": "us-central1", + "shard": "shared-0", + "platformNamespace": "internal-dns-system", + "platformKubeAPI": {"qps":50,"burst":100,"requestTimeoutSeconds":10}, + "projectDiscovery": { + "kubeconfig": "/var/run/internal-dns/discovery/kubeconfig", + "intervalSeconds": 10, + "kubeAPI": {"qps":20,"burst":40,"requestTimeoutSeconds":10} + }, + "nodeBackends": [ + {"memberID":"node-coredns-0","address":"fd70:10::10","port":5300} + ], + "clusterBackends": [ + {"memberID":"regional-coredns-us-central1-0","address":"fd70:20::10","port":5300}, + {"memberID":"regional-coredns-us-central1-1","address":"fd70:20::11","port":5300} + ], + "members": [ + {"memberID":"node-us-central1-0","replicaID":"node-us-central1-0","role":"node"}, + {"memberID":"regional-front-us-central1-0","replicaID":"regional-front-us-central1-0","role":"resolver"}, + {"memberID":"regional-coredns-us-central1-0","replicaID":"regional-coredns-us-central1-0","role":"regional"}, + {"memberID":"regional-coredns-us-central1-1","replicaID":"regional-coredns-us-central1-1","role":"regional"} + ], + "clusterPrefix": "fd70:200::/48", + "privateZoneClassName": "private-coredns", + "ownershipLeaseSeconds": 30, + "shards": [ + {"region":"us-central1","shard":"shared-0","maxBindings":200,"maxSnapshotBytes":524288} + ], + "nats": { + "url": "tls://nats.internal-dns-system.svc:4222", + "name": "internal-dns-control-plane", + "username": "internal-dns-control-plane", + "passwordFile": "/var/run/internal-dns/nats/password", + "tls": { + "caFile": "/var/run/internal-dns/nats/ca.crt", + "certificateFile": "/var/run/internal-dns/nats/tls.crt", + "keyFile": "/var/run/internal-dns/nats/tls.key", + "serverName": "nats.internal-dns-system.svc" + } + }, + "stream": { + "name": "DNS_PRIVATE", + "subjects": ["dns.private.serving.>","dns.private.records.>","dns.private.acks.>"], + "maxAge": 86400000000000, + "maxBytes": 10737418240, + "replicas": 3, + "snapshotsBucket": "DNS_PRIVATE_SNAPSHOTS" + }, + "ensureStream": true, + "admission": { + "enabled": true, + "port": 9443, + "certDir": "/var/run/internal-dns/admission" + } + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: internal-dns-control-plane + namespace: internal-dns-system +spec: + replicas: 1 + selector: + matchLabels: {app.kubernetes.io/name: internal-dns-control-plane} + template: + metadata: + labels: {app.kubernetes.io/name: internal-dns-control-plane} + spec: + serviceAccountName: internal-dns-control-plane + containers: + - name: control-plane + image: example.invalid/dns-operator/internal-dns:VERSION + args: ["--role=control-plane","--config=/etc/internal-dns/config.json","--kubeconfig=/var/run/internal-dns/platform/kubeconfig"] + ports: + - {name: admission, containerPort: 9443} + resources: + requests: {cpu: 100m, memory: 256Mi} + limits: {memory: 2Gi} + readinessProbe: + httpGet: {scheme: HTTPS, path: /readyz, port: admission} + periodSeconds: 5 + timeoutSeconds: 2 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + capabilities: {drop: [ALL]} + volumeMounts: + - {name: discovery, mountPath: /var/run/internal-dns/discovery, readOnly: true} + - {name: platform, mountPath: /var/run/internal-dns/platform, readOnly: true} + - {name: config, mountPath: /etc/internal-dns, readOnly: true} + - {name: nats, mountPath: /var/run/internal-dns/nats, readOnly: true} + - {name: admission, mountPath: /var/run/internal-dns/admission, readOnly: true} + volumes: + - name: discovery + secret: + secretName: internal-dns-discovery-kubeconfig + items: [{key: value, path: kubeconfig}] + - name: platform + secret: + secretName: internal-dns-service-kubeconfig + items: [{key: value, path: kubeconfig}] + - name: config + configMap: {name: internal-dns-control-plane-config} + - name: nats + secret: {secretName: internal-dns-control-plane-nats} + - name: admission + secret: {secretName: internal-dns-admission-tls} +--- +apiVersion: v1 +kind: Service +metadata: + name: internal-dns-admission + namespace: internal-dns-system +spec: + selector: {app.kubernetes.io/name: internal-dns-control-plane} + ports: + - {name: https, port: 443, targetPort: admission} diff --git a/config/internal-dns/fleet.example.yaml b/config/internal-dns/fleet.example.yaml new file mode 100644 index 00000000..c8503f94 --- /dev/null +++ b/config/internal-dns/fleet.example.yaml @@ -0,0 +1,122 @@ +# Fixed shared fleet members for one regional shard. Deploy at least two copies +# of the regional-coredns member with distinct IDs, addresses, state, config, NATS +# credentials, and activation state. Contexts and zones never create workloads. +apiVersion: v1 +kind: ConfigMap +metadata: {name: internal-dns-node-0-config, namespace: internal-dns-system} +data: + config.json: | + { + "nats":{"url":"tls://nats.internal-dns-system.svc:4222","name":"node-us-central1-0","username":"node-us-central1-0","passwordFile":"/var/run/internal-dns/nats/password","tls":{"caFile":"/var/run/internal-dns/nats/ca.crt","certificateFile":"/var/run/internal-dns/nats/tls.crt","keyFile":"/var/run/internal-dns/nats/tls.key","serverName":"nats.internal-dns-system.svc"}}, + "stream":{"name":"DNS_PRIVATE","snapshotsBucket":"DNS_PRIVATE_SNAPSHOTS"}, + "agent":{ + "region":"us-central1","shard":"shared-0","memberID":"node-us-central1-0","replicaID":"node-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"resolver", + "render":{"role":"node","nodeDNSDist":{"path":"/etc/internal-dns/node-dnsdist.conf","listenAddress":"[::]:53","acls":["fd71:100::/64","::1/128"],"poolName":"node_shared","backends":[{"memberID":"node-coredns-0","address":"fd70:10::10","port":5300}]},"nodeCoreDNS":{"path":"/etc/internal-dns/node-coredns.json","listenAddress":"fd70:10::10","port":5300,"proxyPeers":["fd70:10::20","fd70:10::10"],"publicUpstreams":["[fd70:30::10]:5353"]}}, + "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-dnsdist","args":["node","{stageDir}/node-dnsdist.conf"],"whenChanged":["/etc/internal-dns/node-dnsdist.conf"]},{"path":"/usr/local/sbin/internal-dns-validate-coredns","args":["node","{stageDir}/node-coredns.json"],"whenChanged":["/etc/internal-dns/node-coredns.json"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-coredns","args":["node"],"whenChanged":["/etc/internal-dns/node-coredns.json"]},{"path":"/usr/local/sbin/internal-dns-reload-dnsdist","args":["node"],"whenChanged":["/etc/internal-dns/node-dnsdist.conf"]}],"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["node"]}]}, + "reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"resolverDNSProbe":{"timeout":3000000000},"resolverVerifyConcurrency":16 + } + } +--- +apiVersion: v1 +kind: ConfigMap +metadata: {name: internal-dns-regional-front-0-config, namespace: internal-dns-system} +data: + config.json: | + { + "nats":{"url":"tls://nats.internal-dns-system.svc:4222","name":"regional-front-us-central1-0","username":"regional-front-us-central1-0","passwordFile":"/var/run/internal-dns/nats/password","tls":{"caFile":"/var/run/internal-dns/nats/ca.crt","certificateFile":"/var/run/internal-dns/nats/tls.crt","keyFile":"/var/run/internal-dns/nats/tls.key","serverName":"nats.internal-dns-system.svc"}}, + "stream":{"name":"DNS_PRIVATE","snapshotsBucket":"DNS_PRIVATE_SNAPSHOTS"}, + "agent":{ + "region":"us-central1","shard":"shared-0","memberID":"regional-front-us-central1-0","replicaID":"regional-front-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"resolver", + "render":{"role":"regional-dnsdist","clusterDNSDist":{"path":"/etc/internal-dns/cluster-dnsdist.conf","listenAddress":"[::]:53","acls":["fd71:200::/64","::1/128"],"poolName":"regional_shared","backends":[{"memberID":"regional-coredns-us-central1-0","address":"fd70:20::10","port":5300},{"memberID":"regional-coredns-us-central1-1","address":"fd70:20::11","port":5300}]}}, + "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-dnsdist","args":["regional","{stageDir}/cluster-dnsdist.conf"],"whenChanged":["/etc/internal-dns/cluster-dnsdist.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-dnsdist","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-dnsdist.conf"]}],"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["regional-front"]}]}, + "reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"resolverDNSProbe":{"timeout":3000000000},"resolverVerifyConcurrency":16 + } + } +--- +apiVersion: v1 +kind: ConfigMap +metadata: {name: internal-dns-regional-coredns-0-config, namespace: internal-dns-system} +data: + config.json: | + { + "nats":{"url":"tls://nats.internal-dns-system.svc:4222","name":"regional-coredns-us-central1-0","username":"regional-coredns-us-central1-0","passwordFile":"/var/run/internal-dns/nats/password","tls":{"caFile":"/var/run/internal-dns/nats/ca.crt","certificateFile":"/var/run/internal-dns/nats/tls.crt","keyFile":"/var/run/internal-dns/nats/tls.key","serverName":"nats.internal-dns-system.svc"}}, + "stream":{"name":"DNS_PRIVATE","snapshotsBucket":"DNS_PRIVATE_SNAPSHOTS"}, + "agent":{ + "region":"us-central1","shard":"shared-0","memberID":"regional-coredns-us-central1-0","replicaID":"regional-coredns-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"combined", + "render":{"role":"regional-coredns","clusterCoreDNS":{"path":"/etc/internal-dns/cluster-coredns.json","listenAddress":"fd70:20::10","port":5300,"proxyPeers":["fd70:20::20","fd70:20::30","fd70:20::10"],"publicUpstreams":["[fd70:30::10]:5353"]}}, + "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-coredns","args":["regional","{stageDir}/cluster-coredns.json"],"whenChanged":["/etc/internal-dns/cluster-coredns.json"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-coredns","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-coredns.json"]}],"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["regional-coredns"]}]}, + "reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"publicationDNSProbe":{"server":"[fd70:20::10]:5300","timeout":3000000000} + } + } +--- +# The second regional member has independent identity, listener, state, +# checkpoint, and activation storage. Its backend memberID exactly matches the +# second clusterBackends entry in the control-plane configuration. +apiVersion: v1 +kind: ConfigMap +metadata: {name: internal-dns-regional-coredns-1-config, namespace: internal-dns-system} +data: + config.json: | + { + "nats":{"url":"tls://nats.internal-dns-system.svc:4222","name":"regional-coredns-us-central1-1","username":"regional-coredns-us-central1-1","passwordFile":"/var/run/internal-dns/nats/password","tls":{"caFile":"/var/run/internal-dns/nats/ca.crt","certificateFile":"/var/run/internal-dns/nats/tls.crt","keyFile":"/var/run/internal-dns/nats/tls.key","serverName":"nats.internal-dns-system.svc"}}, + "stream":{"name":"DNS_PRIVATE","snapshotsBucket":"DNS_PRIVATE_SNAPSHOTS"}, + "agent":{ + "region":"us-central1","shard":"shared-0","memberID":"regional-coredns-us-central1-1","replicaID":"regional-coredns-us-central1-1","stateDir":"/var/lib/internal-dns","mode":"combined", + "render":{"role":"regional-coredns","clusterCoreDNS":{"path":"/etc/internal-dns/cluster-coredns.json","listenAddress":"fd70:20::11","port":5300,"proxyPeers":["fd70:20::20","fd70:20::31","fd70:20::11"],"publicUpstreams":["[fd70:30::10]:5353"]}}, + "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-coredns","args":["regional","{stageDir}/cluster-coredns.json"],"whenChanged":["/etc/internal-dns/cluster-coredns.json"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-coredns","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-coredns.json"]}],"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["regional-coredns"]}]}, + "reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"publicationDNSProbe":{"server":"[fd70:20::11]:5300","timeout":3000000000} + } + } +--- +# This Deployment template is instantiated once for each ConfigMap above and a +# second time for regional-coredns-1. The serving image contains the local dnsdist +# and/or CoreDNS process selected by MEMBER_ROLE plus the validation/reload/gate +# wrappers referenced by that member's config. +apiVersion: apps/v1 +kind: Deployment +metadata: {name: internal-dns-regional-coredns-0, namespace: internal-dns-system} +spec: + replicas: 1 + strategy: {type: Recreate} + selector: {matchLabels: {app.kubernetes.io/name: internal-dns-regional-coredns-0}} + template: + metadata: {labels: {app.kubernetes.io/name: internal-dns-regional-coredns-0, internal-dns.datum.net/tier: regional}} + spec: + serviceAccountName: internal-dns-fleet + automountServiceAccountToken: false + shareProcessNamespace: true + securityContext: {fsGroup: 65532, fsGroupChangePolicy: OnRootMismatch} + containers: + - name: agent + image: example.invalid/dns-operator/internal-dns-serving:VERSION + args: ["--role=agent","--config=/config/config.json"] + securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, runAsNonRoot: true, runAsUser: 65532, runAsGroup: 65532, capabilities: {drop: [ALL], add: [KILL]}} + volumeMounts: &member-mounts + - {name: config, mountPath: /config, readOnly: true} + - {name: nats, mountPath: /var/run/internal-dns/nats, readOnly: true} + - {name: activation, mountPath: /run/internal-dns} + - {name: state, mountPath: /var/lib/internal-dns} + - {name: daemon-config, mountPath: /etc/internal-dns} + - name: serving + image: example.invalid/dns-operator/internal-dns-serving:VERSION + command: ["/internal-dns"] + args: ["--role=serving","--config=/config/config.json"] + readinessProbe: + exec: {command: ["/internal-dns", "--role=readiness", "--config=/config/config.json"]} + periodSeconds: 2 + timeoutSeconds: 2 + securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, runAsUser: 65533, runAsGroup: 65532, capabilities: {drop: [ALL], add: [NET_BIND_SERVICE]}} + volumeMounts: + - {name: config, mountPath: /config, readOnly: true} + - {name: state, mountPath: /var/lib/internal-dns} + - {name: activation, mountPath: /run/internal-dns} + - {name: daemon-config, mountPath: /etc/internal-dns} + # CoreDNS requires a writable working directory with a read-only root filesystem. + - {name: runtime, mountPath: /tmp} + volumes: + - {name: config, configMap: {name: internal-dns-regional-coredns-0-config}} + - {name: nats, secret: {secretName: internal-dns-regional-coredns-0-nats}} + - {name: activation, emptyDir: {medium: Memory}} + - {name: runtime, emptyDir: {}} + - {name: daemon-config, persistentVolumeClaim: {claimName: internal-dns-regional-coredns-0-config}} + - {name: state, persistentVolumeClaim: {claimName: internal-dns-regional-coredns-0-state}} diff --git a/config/internal-dns/kustomization.yaml b/config/internal-dns/kustomization.yaml new file mode 100644 index 00000000..4fca26ba --- /dev/null +++ b/config/internal-dns/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- namespace.yaml +- rbac.yaml +- control-plane.example.yaml +- fleet.example.yaml +- routers.example.yaml +- admission.example.yaml diff --git a/config/internal-dns/namespace.yaml b/config/internal-dns/namespace.yaml new file mode 100644 index 00000000..13650a17 --- /dev/null +++ b/config/internal-dns/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: internal-dns-system + labels: + app.kubernetes.io/part-of: internal-dns diff --git a/config/internal-dns/nats-server.conf.example b/config/internal-dns/nats-server.conf.example new file mode 100644 index 00000000..573d8eac --- /dev/null +++ b/config/internal-dns/nats-server.conf.example @@ -0,0 +1,152 @@ +# This file demonstrates application-subject isolation. Password variables are +# injected by the broker secret manager; use NKey/JWT credentials in a NATS +# operator deployment. No product workload receives a NATS identity. +server_name: internal-dns-1 +jetstream { + store_dir: "/var/lib/nats/jetstream" +} +tls { + cert_file: "/var/run/nats/tls/tls.crt" + key_file: "/var/run/nats/tls/tls.key" + ca_file: "/var/run/nats/tls/ca.crt" + verify: true + timeout: 2 +} +authorization { + users: [ + { + user: "internal-dns-control-plane" + password: $CONTROL_PLANE_PASSWORD + permissions: { + publish: { allow: [ + "dns.private.serving.>", "dns.private.records.>", + "$KV.DNS_PRIVATE_SNAPSHOTS.>", "$JS.API.>", + # Match the control-plane ACK durable for this region and shard. + "$JS.ACK.DNS_PRIVATE.control-plane-us-central1-shared-0-acks.>", + "_INBOX.>" + ] } + subscribe: { allow: ["dns.private.acks.>", "_INBOX.>"] } + } + }, + { + user: "node-us-central1-0" + password: $NODE_RESOLVER_0_PASSWORD + permissions: { + publish: { allow: [ + "dns.private.acks.us-central1.shared-0.0195a8650ce9d5c927c91495", + "$JS.API.STREAM.INFO.DNS_PRIVATE", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.node-us-central1-0-serving", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.node-us-central1-0-records", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.node-us-central1-0-acks", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.node-us-central1-0-serving", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.node-us-central1-0-serving.>", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.node-us-central1-0-records", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.node-us-central1-0-records.>", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.node-us-central1-0-acks", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.node-us-central1-0-acks.>", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.node-us-central1-0-serving", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.node-us-central1-0-records", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.node-us-central1-0-acks", + "$JS.ACK.DNS_PRIVATE.node-us-central1-0-serving.>", + "$JS.ACK.DNS_PRIVATE.node-us-central1-0-records.>", + "$JS.ACK.DNS_PRIVATE.node-us-central1-0-acks.>", + "$JS.API.DIRECT.GET.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.STREAM.INFO.KV_DNS_PRIVATE_SNAPSHOTS", + "$JS.API.CONSUMER.CREATE.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.CONSUMER.INFO.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.CONSUMER.MSG.NEXT.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.CONSUMER.DELETE.KV_DNS_PRIVATE_SNAPSHOTS.>", + "_INBOX.>" + ] } + subscribe: { allow: ["_INBOX.>"] } + } + }, + { + user: "regional-front-us-central1-0" + password: $REGIONAL_FRONT_0_PASSWORD + permissions: { + publish: { allow: [ + "dns.private.acks.us-central1.shared-0.92106c62061ed0349bc66280", + "$JS.API.STREAM.INFO.DNS_PRIVATE", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-front-us-central1-0-serving", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-front-us-central1-0-records", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-front-us-central1-0-acks", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-front-us-central1-0-serving", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-front-us-central1-0-serving.>", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-front-us-central1-0-records", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-front-us-central1-0-records.>", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-front-us-central1-0-acks", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-front-us-central1-0-acks.>", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-front-us-central1-0-serving", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-front-us-central1-0-records", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-front-us-central1-0-acks", + "$JS.ACK.DNS_PRIVATE.regional-front-us-central1-0-serving.>", + "$JS.ACK.DNS_PRIVATE.regional-front-us-central1-0-records.>", + "$JS.ACK.DNS_PRIVATE.regional-front-us-central1-0-acks.>", + "$JS.API.DIRECT.GET.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.STREAM.INFO.KV_DNS_PRIVATE_SNAPSHOTS", + "$JS.API.CONSUMER.CREATE.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.CONSUMER.INFO.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.CONSUMER.MSG.NEXT.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.CONSUMER.DELETE.KV_DNS_PRIVATE_SNAPSHOTS.>", + "_INBOX.>" + ] } + subscribe: { allow: ["_INBOX.>"] } + } + }, + { + user: "regional-coredns-us-central1-0" + password: $REGIONAL_COREDNS_0_PASSWORD + permissions: { + publish: { allow: [ + "dns.private.acks.us-central1.shared-0.7c90d3973cc5ead390c680d8", + "$JS.API.STREAM.INFO.DNS_PRIVATE", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-coredns-us-central1-0-serving", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-coredns-us-central1-0-records", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-coredns-us-central1-0-acks", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-coredns-us-central1-0-serving", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-coredns-us-central1-0-serving.>", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-coredns-us-central1-0-records", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-coredns-us-central1-0-records.>", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-coredns-us-central1-0-acks", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-coredns-us-central1-0-acks.>", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-coredns-us-central1-0-serving", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-coredns-us-central1-0-records", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-coredns-us-central1-0-acks", + "$JS.ACK.DNS_PRIVATE.regional-coredns-us-central1-0-serving.>", + "$JS.ACK.DNS_PRIVATE.regional-coredns-us-central1-0-records.>", + "$JS.ACK.DNS_PRIVATE.regional-coredns-us-central1-0-acks.>", + "$JS.API.DIRECT.GET.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.STREAM.INFO.KV_DNS_PRIVATE_SNAPSHOTS", + "$JS.API.CONSUMER.CREATE.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.CONSUMER.INFO.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.CONSUMER.MSG.NEXT.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.CONSUMER.DELETE.KV_DNS_PRIVATE_SNAPSHOTS.>", + "_INBOX.>" + ] } + subscribe: { allow: ["_INBOX.>"] } + } + }, + { + user: "regional-coredns-us-central1-1" + password: $REGIONAL_COREDNS_1_PASSWORD + permissions: { + publish: { allow: [ + "dns.private.acks.us-central1.shared-0.f7f1afc061d6adfe80acb36b", + "$JS.API.STREAM.INFO.DNS_PRIVATE", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-coredns-us-central1-1-serving", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-coredns-us-central1-1-records", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-coredns-us-central1-1-acks", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-coredns-us-central1-1-serving", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-coredns-us-central1-1-serving.>", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-coredns-us-central1-1-records", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-coredns-us-central1-1-records.>", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-coredns-us-central1-1-acks", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-coredns-us-central1-1-acks.>", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-coredns-us-central1-1-serving", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-coredns-us-central1-1-records", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-coredns-us-central1-1-acks", + "$JS.ACK.DNS_PRIVATE.regional-coredns-us-central1-1-serving.>", + "$JS.ACK.DNS_PRIVATE.regional-coredns-us-central1-1-records.>", + "$JS.ACK.DNS_PRIVATE.regional-coredns-us-central1-1-acks.>", + "$JS.API.DIRECT.GET.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.STREAM.INFO.KV_DNS_PRIVATE_SNAPSHOTS", + "$JS.API.CONSUMER.CREATE.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.CONSUMER.INFO.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.CONSUMER.MSG.NEXT.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.CONSUMER.DELETE.KV_DNS_PRIVATE_SNAPSHOTS.>", + "_INBOX.>" + ] } + subscribe: { allow: ["_INBOX.>"] } + } + } + ] +} diff --git a/config/internal-dns/rbac.yaml b/config/internal-dns/rbac.yaml new file mode 100644 index 00000000..73f5394e --- /dev/null +++ b/config/internal-dns/rbac.yaml @@ -0,0 +1,138 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: internal-dns-control-plane + namespace: internal-dns-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: internal-dns-fleet + namespace: internal-dns-system +automountServiceAccountToken: false +--- +# Bind this Role only to the separately deployed VPC integration principal. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: internal-dns-vpc-integration + namespace: example-project +rules: +- apiGroups: [dns.networking.miloapis.com] + resources: [dnsresolvercontexts, dnsresolveraccessbindings] + verbs: [get, list, watch, create, update, patch, delete] +- apiGroups: [dns.networking.miloapis.com] + resources: [dnsresolvercontexts/status, dnsresolveraccessbindings/status] + verbs: [get, list, watch] +--- +# Project products atomically publish endpoint records and eligibility. +# Product writers cannot modify resolver access or publication artifacts. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: internal-dns-product-writer + namespace: example-project +rules: +- apiGroups: [dns.networking.miloapis.com] + resources: [dnsregistrations, dnsrecordcontributions] + verbs: [get, list, watch, create, update, patch, delete] +- apiGroups: [dns.networking.miloapis.com] + resources: [dnsresolvercontexts, dnsnamingpolicies] + verbs: [get, list, watch] +--- +# Customers configure private zones and naming within their project namespace. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: internal-dns-customer + namespace: example-project +rules: +- apiGroups: [dns.networking.miloapis.com] + resources: [dnszones, dnsrecordsets, dnszoneassociations, dnsnamingpolicies, dnsregistrations] + verbs: [get, list, watch, create, update, patch, delete] +- apiGroups: [dns.networking.miloapis.com] + resources: [dnsresolvercontexts] + verbs: [get, list, watch] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: internal-dns-control-plane +rules: +- apiGroups: [dns.networking.miloapis.com] + resources: + - dnszones + - dnsrecordsets + - dnszoneassociations + - dnsregistrations + - dnsrecordcontributions + - dnsnamingpolicies + - dnsresolvercontexts + - dnsresolveraccessbindings + verbs: [get, list, watch] +# Managed zone creation and the private-zone publication finalizer. +- apiGroups: [dns.networking.miloapis.com] + resources: [dnszones, dnszoneassociations] + verbs: [create, patch] +- apiGroups: [dns.networking.miloapis.com] + resources: + - dnsresolverbindings + - dnspublicationmanifests + - dnspublicationchunks + - dnstransportoutboxes + - dnspublicationownerships + verbs: [get, list, watch, create, update, patch, delete] +- apiGroups: [dns.networking.miloapis.com] + resources: + - dnsresolverbindings/status + - dnspublicationmanifests/status + - dnstransportoutboxes/status + - dnspublicationownerships/status + - dnszones/status + - dnszoneassociations/status + - dnsnamingpolicies/status + - dnsregistrations/status + - dnsrecordcontributions/status + - dnsresolvercontexts/status + - dnsresolveraccessbindings/status + verbs: [get, update, patch] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: internal-dns-control-plane +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: internal-dns-control-plane +subjects: +- kind: ServiceAccount + name: internal-dns-control-plane + namespace: internal-dns-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: internal-dns-control-plane-state + namespace: internal-dns-system +rules: +- apiGroups: [coordination.k8s.io] + resources: [leases] + verbs: [get, list, watch, create, update, patch] +- apiGroups: [""] + resources: [configmaps] + verbs: [get, list, watch, create, update, patch, delete] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: internal-dns-control-plane-state + namespace: internal-dns-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: internal-dns-control-plane-state +subjects: +- kind: ServiceAccount + name: internal-dns-control-plane + namespace: internal-dns-system diff --git a/config/internal-dns/routers.example.yaml b/config/internal-dns/routers.example.yaml new file mode 100644 index 00000000..25b27c68 --- /dev/null +++ b/config/internal-dns/routers.example.yaml @@ -0,0 +1,292 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: internal-dns-node-0 + namespace: internal-dns-system +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: internal-dns-node-0 + template: + metadata: + labels: + app.kubernetes.io/name: internal-dns-node-0 + internal-dns.datum.net/tier: node + spec: + serviceAccountName: internal-dns-fleet + automountServiceAccountToken: false + shareProcessNamespace: true + securityContext: + fsGroup: 65532 + fsGroupChangePolicy: OnRootMismatch + containers: + - name: agent + image: example.invalid/dns-operator/internal-dns-serving:VERSION + args: + - "--role=agent" + - "--config=/config/config.json" + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + capabilities: + drop: + - ALL + add: + - KILL + volumeMounts: + - name: config + mountPath: "/config" + readOnly: true + - name: nats + mountPath: "/var/run/internal-dns/nats" + readOnly: true + - name: activation + mountPath: "/run/internal-dns" + - name: state + mountPath: "/var/lib/internal-dns" + - name: daemon-config + mountPath: "/etc/internal-dns" + - name: serving + image: example.invalid/dns-operator/internal-dns-serving:VERSION + command: + - "/internal-dns" + args: + - "--role=serving" + - "--config=/config/config.json" + readinessProbe: + exec: + command: + - "/internal-dns" + - "--role=readiness" + - "--config=/config/config.json" + periodSeconds: 2 + timeoutSeconds: 2 + securityContext: &1 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsUser: 65533 + runAsGroup: 65532 + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + volumeMounts: + - name: config + mountPath: "/config" + readOnly: true + - name: state + mountPath: "/var/lib/internal-dns" + - name: activation + mountPath: "/run/internal-dns" + - name: daemon-config + mountPath: "/etc/internal-dns" + - name: runtime + mountPath: "/tmp" + - name: dnsdist + image: powerdns/dnsdist-20:2.0.10@sha256:24ac720af997aff6b98cb5b50d748754f815428e07b5368dfb0eccd5aa8d07e8 + command: + - "/usr/local/bin/dnsdist" + args: + - "--supervised" + - "--disable-syslog" + - "-C" + - "/etc/internal-dns/node-dnsdist.conf" + securityContext: *1 + volumeMounts: + - name: daemon-config + mountPath: "/etc/internal-dns" + readOnly: true + - name: activation + mountPath: "/run/internal-dns" + readOnly: true + - name: runtime + mountPath: "/tmp" + volumes: + - name: config + configMap: + name: internal-dns-node-0-config + - name: nats + secret: + secretName: internal-dns-node-0-nats + - name: activation + emptyDir: + medium: Memory + - name: runtime + emptyDir: {} + - name: daemon-config + persistentVolumeClaim: + claimName: internal-dns-node-0-config + - name: state + persistentVolumeClaim: + claimName: internal-dns-node-0-state + initContainers: + - name: dnsdist-config + image: example.invalid/dns-operator/internal-dns-serving:VERSION + args: + - "--role=prepare" + - "--config=/config/config.json" + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + capabilities: + drop: + - ALL + volumeMounts: + - name: config + mountPath: "/config" + readOnly: true + - name: nats + mountPath: "/var/run/internal-dns/nats" + readOnly: true + - name: activation + mountPath: "/run/internal-dns" + - name: state + mountPath: "/var/lib/internal-dns" + - name: daemon-config + mountPath: "/etc/internal-dns" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: internal-dns-regional-front-0 + namespace: internal-dns-system +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: internal-dns-regional-front-0 + template: + metadata: + labels: + app.kubernetes.io/name: internal-dns-regional-front-0 + internal-dns.datum.net/tier: regional + spec: + serviceAccountName: internal-dns-fleet + automountServiceAccountToken: false + shareProcessNamespace: true + securityContext: + fsGroup: 65532 + fsGroupChangePolicy: OnRootMismatch + containers: + - name: agent + image: example.invalid/dns-operator/internal-dns-serving:VERSION + args: + - "--role=agent" + - "--config=/config/config.json" + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + capabilities: + drop: + - ALL + volumeMounts: + - name: config + mountPath: "/config" + readOnly: true + - name: nats + mountPath: "/var/run/internal-dns/nats" + readOnly: true + - name: activation + mountPath: "/run/internal-dns" + - name: state + mountPath: "/var/lib/internal-dns" + - name: daemon-config + mountPath: "/etc/internal-dns" + readinessProbe: + exec: + command: + - "/internal-dns" + - "--role=readiness" + - "--config=/config/config.json" + periodSeconds: 2 + timeoutSeconds: 2 + - name: dnsdist + image: powerdns/dnsdist-20:2.0.10@sha256:24ac720af997aff6b98cb5b50d748754f815428e07b5368dfb0eccd5aa8d07e8 + command: + - "/usr/local/bin/dnsdist" + args: + - "--supervised" + - "--disable-syslog" + - "-C" + - "/etc/internal-dns/cluster-dnsdist.conf" + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsUser: 65533 + runAsGroup: 65532 + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + volumeMounts: + - name: daemon-config + mountPath: "/etc/internal-dns" + readOnly: true + - name: activation + mountPath: "/run/internal-dns" + readOnly: true + - name: runtime + mountPath: "/tmp" + volumes: + - name: config + configMap: + name: internal-dns-regional-front-0-config + - name: nats + secret: + secretName: internal-dns-regional-front-0-nats + - name: activation + emptyDir: + medium: Memory + - name: runtime + emptyDir: {} + - name: daemon-config + persistentVolumeClaim: + claimName: internal-dns-regional-front-0-config + - name: state + persistentVolumeClaim: + claimName: internal-dns-regional-front-0-state + initContainers: + - name: dnsdist-config + image: example.invalid/dns-operator/internal-dns-serving:VERSION + args: + - "--role=prepare" + - "--config=/config/config.json" + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + capabilities: + drop: + - ALL + volumeMounts: + - name: config + mountPath: "/config" + readOnly: true + - name: nats + mountPath: "/var/run/internal-dns/nats" + readOnly: true + - name: activation + mountPath: "/run/internal-dns" + - name: state + mountPath: "/var/lib/internal-dns" + - name: daemon-config + mountPath: "/etc/internal-dns" diff --git a/hack/publish-internal-dns-channel.sh b/hack/publish-internal-dns-channel.sh new file mode 100755 index 00000000..3d56f02f --- /dev/null +++ b/hack/publish-internal-dns-channel.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${RUNTIME_TAG:?runtime build tag is required}" +: "${SERVING_TAG:?serving build tag is required}" +: "${RUNNER_TEMP:?runner temporary directory is required}" +: "${GITHUB_REPOSITORY:?source repository is required}" +: "${GITHUB_SHA:?source revision is required}" +if [[ "$RUNTIME_TAG" != "$SERVING_TAG" || ! "$RUNTIME_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+[-a-zA-Z0-9.]*$ ]]; then + echo 'Runtime and serving builds must share a valid version tag' >&2 + exit 1 +fi + +runtime_digest=$(docker buildx imagetools inspect "ghcr.io/datum-cloud/internal-dns:$RUNTIME_TAG" --format '{{.Manifest.Digest}}') +serving_digest=$(docker buildx imagetools inspect "ghcr.io/datum-cloud/internal-dns-serving:$SERVING_TAG" --format '{{.Manifest.Digest}}') +for digest in "$runtime_digest" "$serving_digest"; do + if [[ ! "$digest" =~ ^sha256:[a-f0-9]{64}$ ]]; then + echo 'A published image manifest digest is required' >&2 + exit 1 + fi +done + +channel_dir=$(mktemp -d "$RUNNER_TEMP/internal-dns-channel.XXXXXX") +trap 'rm -rf "$channel_dir"' EXIT +cat > "$channel_dir/kustomization.yaml" <<'EOF' +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - images.yaml +EOF +cat > "$channel_dir/images.yaml" < 1000 { + return fmt.Errorf("%s.qps must be greater than 0 and at most 1000", name) + } + if c.Burst <= 0 || c.Burst > 2000 { + return fmt.Errorf("%s.burst must be greater than 0 and at most 2000", name) + } + if c.RequestTimeoutSeconds <= 0 || c.RequestTimeoutSeconds > 120 { + return fmt.Errorf("%s.requestTimeoutSeconds must be greater than 0 and at most 120", name) + } + return nil +} + +func (c *Config) Validate(role string) error { + c.defaults() + switch role { + case roleControlPlane, roleAgent, roleReadiness, roleServing, rolePrepare: + default: + return fmt.Errorf("unsupported role %q", role) + } + if role == roleReadiness || role == roleServing || role == rolePrepare { + if c.Agent.Region == "" { + c.Agent.Region = c.Region + } + if c.Agent.Shard == "" { + c.Agent.Shard = c.Shard + } + return c.Agent.Validate() + } + if len(c.RegionalTransports) == 0 { + if err := validateTransportConfig(c.NATS, c.Stream, c.EnsureStream); err != nil { + return err + } + } else { + seen := map[string]bool{} + for i := range c.RegionalTransports { + regional := &c.RegionalTransports[i] + if regional.NATS.Name == "" { + regional.NATS.Name = "internal-dns-" + model.SafeToken(regional.Region) + } + if regional.Region == "" || seen[regional.Region] { + return fmt.Errorf("regional transport %d has missing or duplicate region", i) + } + seen[regional.Region] = true + if err := validateTransportConfig(regional.NATS, regional.Stream, regional.EnsureStream); err != nil { + return fmt.Errorf("regional transport %s: %w", regional.Region, err) + } + } + } + if role == roleAgent { + if c.Agent.Region == "" { + c.Agent.Region = c.Region + } + if c.Agent.Shard == "" { + c.Agent.Shard = c.Shard + } + if _, _, _, err := c.transportForRegion(c.Agent.Region); err != nil { + return err + } + return c.Agent.Validate() + } + if c.PlatformNamespace == "" || (len(c.Projects) == 0 && c.ProjectDiscovery == nil) { + return errors.New("control-plane requires platformNamespace and project discovery or projects") + } + if err := validateKubeAPI("platformKubeAPI", c.PlatformKubeAPI); err != nil { + return err + } + if c.ProjectDiscovery != nil { + if len(c.Projects) != 0 { + return errors.New("projectDiscovery and projects are mutually exclusive") + } + if !filepath.IsAbs(c.ProjectDiscovery.Kubeconfig) { + return errors.New("projectDiscovery.kubeconfig must be absolute") + } + if c.ProjectDiscovery.IntervalSeconds < 1 || c.ProjectDiscovery.IntervalSeconds > 300 { + return errors.New("projectDiscovery.intervalSeconds must be between 1 and 300") + } + if err := validateKubeAPI("projectDiscovery.kubeAPI", c.ProjectDiscovery.KubeAPI); err != nil { + return err + } + } + names := map[string]bool{} + uids := map[types.UID]bool{} + namespaces := map[string]bool{} + for i, p := range c.Projects { + if p.ProjectUID == "" || p.Namespace == "" { + return fmt.Errorf("project %d requires projectUID and namespace", i) + } + if p.Namespace == c.PlatformNamespace { + return fmt.Errorf("project %d namespace collides with platformNamespace", i) + } + if err := validateKubeAPI(fmt.Sprintf("projects[%d].kubeAPI", i), p.KubeAPI); err != nil { + return err + } + if uids[p.ProjectUID] { + return fmt.Errorf("duplicate project UID %q", p.ProjectUID) + } + uids[p.ProjectUID] = true + if p.Name != "" { + if names[p.Name] { + return fmt.Errorf("duplicate project name %q", p.Name) + } + names[p.Name] = true + } + if p.Kubeconfig == "" && namespaces[p.Namespace] { + return fmt.Errorf("duplicate project namespace %q on the platform cluster", p.Namespace) + } + namespaces[p.Namespace] = true + } + if len(c.Projects) > 1 { + for _, p := range c.Projects { + if p.Name == "" { + return errors.New("every project needs name when several projects are configured") + } + } + } + if c.Admission.Enabled && !filepath.IsAbs(c.Admission.CertDir) { + return errors.New("admission.certDir must be absolute when admission is enabled") + } + if err := validateMembers(c); err != nil { + return err + } + shardPairs := map[string]bool{} + for i := range c.Shards { + if c.Shards[i].Region == "" || c.Shards[i].Shard == "" { + return fmt.Errorf("shard %d requires region and shard", i) + } + key := c.Shards[i].Region + "/" + c.Shards[i].Shard + if shardPairs[key] { + return fmt.Errorf("duplicate shard %q", key) + } + shardPairs[key] = true + } + for _, planner := range c.plannerConfigs() { + if _, _, _, err := c.transportForRegion(planner.Region); err != nil { + return err + } + count := 0 + for _, member := range planner.Members { + if member.Role == memberRoleRegional || member.Role == memberRoleCluster { + count++ + } + } + if count == 0 { + return fmt.Errorf("shard %s/%s has %d regional resolver members; at least one is required", planner.Region, planner.Shard, count) + } + } + return nil +} + +func validateTransportConfig(nats transport.Config, stream transport.StreamConfig, ensure bool) error { + if nats.URL == "" || stream.Name == "" { + return errors.New("nats.url and stream.name are required") + } + if ensure && len(stream.Subjects) == 0 { + return errors.New("stream.subjects are required when ensureStream is enabled") + } + if stream.SnapshotsBucket == "" { + return errors.New("stream.snapshotsBucket is required for restart bootstrap") + } + return nil +} + +func (c Config) transportForRegion(region string) (transport.Config, transport.StreamConfig, bool, error) { + if len(c.RegionalTransports) == 0 { + return c.NATS, c.Stream, c.EnsureStream, nil + } + for _, regional := range c.RegionalTransports { + if regional.Region == region { + return regional.NATS, regional.Stream, regional.EnsureStream, nil + } + } + return transport.Config{}, transport.StreamConfig{}, false, fmt.Errorf("no transport configured for region %q", region) +} + +func validateMembers(c *Config) error { + seen := map[string]bool{} + check := func(ms []model.Member) error { + for _, m := range ms { + if m.MemberID == "" || m.Role == "" { + return errors.New("member ID and role are required") + } + if m.Role != "node" && m.Role != memberRoleRegional && m.Role != memberRoleCluster && m.Role != "resolver" { + return fmt.Errorf("member %s has unsupported resolver role %s", m.MemberID, m.Role) + } + if seen[m.MemberID] { + return fmt.Errorf("member ID %q is not globally unique", m.MemberID) + } + seen[m.MemberID] = true + } + return nil + } + if len(c.Shards) == 0 { + return check(c.Members) + } + for _, s := range c.Shards { + members := s.Members + if len(members) == 0 { + members = c.Members + } + if err := check(members); err != nil { + return err + } + } + return nil +} + +func (c KubeAPIClientConfig) requestTimeout() time.Duration { + return time.Duration(c.RequestTimeoutSeconds) * time.Second +} +func (c Config) ownershipLease() time.Duration { + return time.Duration(c.OwnershipLeaseSeconds) * time.Second +} diff --git a/internal/internaldns/runtime/config_test.go b/internal/internaldns/runtime/config_test.go new file mode 100644 index 00000000..0457c89e --- /dev/null +++ b/internal/internaldns/runtime/config_test.go @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package runtime + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" + "go.miloapis.com/dns-operator/internal/internaldns/serving" + "go.miloapis.com/dns-operator/internal/internaldns/transport" + "k8s.io/client-go/rest" +) + +func TestAgentUsesRootRegionAndDoesNotNeedKubernetes(t *testing.T) { + c := Config{ + Region: "us-central", Shard: "shared-1", + NATS: transport.Config{URL: "nats://fixture:4222", AllowInsecure: true}, + Stream: transport.StreamConfig{Name: "DNS", SnapshotsBucket: "DNS_BOOTSTRAP"}, + Agent: serving.Config{MemberID: "resolver-1", ReplicaID: "resolver-1", StateDir: t.TempDir(), Mode: serving.ModeResolver, + Render: serving.RenderConfig{Role: serving.RenderNode, + NodeDNSDist: serving.DNSDistRenderConfig{Path: t.TempDir() + "/dnsdist.conf", ListenAddress: "[::1]:53", PoolName: "node"}, + NodeCoreDNS: serving.CoreDNSRenderConfig{Path: t.TempDir() + "/named.conf", ListenAddress: "::1", Port: 5353, ProxyPeers: []string{"::1"}}}, + Transaction: serving.TransactionConfig{StageDir: t.TempDir()}, UnsafeAllowNoFailClosed: true, UnsafeAllowNoDNSVerify: true}, + } + if err := c.Validate("agent"); err != nil { + t.Fatal(err) + } + if c.Agent.Region != c.Region || c.Agent.Shard != c.Shard { + t.Fatalf("agent did not inherit shard: %#v", c.Agent) + } +} + +func TestKubeAPIRateLimitDefaultsOverridesAndValidation(t *testing.T) { + base := Config{ + Region: "central", Shard: "one", + NATS: transport.Config{URL: "nats://fixture:4222", AllowInsecure: true}, + Stream: transport.StreamConfig{Name: "DNS", SnapshotsBucket: "DNS_BOOTSTRAP"}, + Projects: []ProjectConfig{{Name: "project", ProjectUID: "p1", Namespace: "project", + KubeAPI: KubeAPIClientConfig{QPS: 30, Burst: 60}}}, + Members: []model.Member{{MemberID: "regional-1", Role: "regional"}}, + } + if err := base.Validate("control-plane"); err != nil { + t.Fatal(err) + } + if base.PlatformKubeAPI.QPS != 50 || base.PlatformKubeAPI.Burst != 100 { + t.Fatalf("unexpected platform defaults: %#v", base.PlatformKubeAPI) + } + if base.Projects[0].KubeAPI.QPS != 30 || base.Projects[0].KubeAPI.Burst != 60 { + t.Fatalf("source override was not preserved: %#v", base.Projects[0].KubeAPI) + } + restConfig := &rest.Config{} + applyKubeAPILimit(restConfig, base.Projects[0].KubeAPI) + if restConfig.QPS != 30 || restConfig.Burst != 60 || restConfig.RateLimiter == nil { + t.Fatalf("REST client did not receive shared rate limiter: %#v", restConfig) + } + invalid := base + invalid.PlatformKubeAPI.QPS = -1 + if err := invalid.Validate("control-plane"); err == nil { + t.Fatal("negative platform Kubernetes API QPS was accepted") + } + invalid = base + invalid.Projects = append([]ProjectConfig(nil), base.Projects...) + invalid.Projects[0].KubeAPI.Burst = 2001 + if err := invalid.Validate("control-plane"); err == nil { + t.Fatal("excessive source Kubernetes API burst was accepted") + } +} + +func TestProjectSourceFailureDoesNotStopHealthySource(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + failedStarted := make(chan struct{}, 1) + healthyStarted := make(chan struct{}, 1) + projects := []projectRuntime{ + {config: ProjectConfig{Name: "offline", ProjectUID: "p-offline"}, start: func(context.Context) error { + failedStarted <- struct{}{} + return errors.New("source API unavailable") + }}, + {config: ProjectConfig{Name: "healthy", ProjectUID: "p-healthy"}, start: func(ctx context.Context) error { + healthyStarted <- struct{}{} + <-ctx.Done() + return ctx.Err() + }}, + } + startProjectSources(ctx, projects) + for name, started := range map[string]<-chan struct{}{"offline": failedStarted, "healthy": healthyStarted} { + select { + case <-started: + case <-time.After(time.Second): + t.Fatalf("%s source did not start independently", name) + } + } +} + +func TestControlPlaneRejectsAmbiguousProjectsAndMemberIDs(t *testing.T) { + base := Config{ + NATS: transport.Config{URL: "nats://fixture:4222", AllowInsecure: true}, + Stream: transport.StreamConfig{Name: "DNS", SnapshotsBucket: "DNS_BOOTSTRAP"}, + Projects: []ProjectConfig{{ProjectUID: "p1", Namespace: "n1"}, {ProjectUID: "p2", Namespace: "n2"}}, + } + if err := base.Validate("control-plane"); err == nil { + t.Fatal("unnamed multi-project configuration was accepted") + } + base.Projects[0].Name, base.Projects[1].Name = "one", "two" + base.Members = []model.Member{{MemberID: "same", Role: "resolver"}, {MemberID: "same", Role: "regional"}} + if err := base.Validate("control-plane"); err == nil { + t.Fatal("duplicate fleet member identity was accepted") + } +} + +func TestConfigLoaderRejectsLegacyNetworkingIntegration(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.json") + if err := os.WriteFile(path, []byte(`{"networking":{"enabled":true}}`), 0600); err != nil { + t.Fatal(err) + } + if _, err := LoadConfig(path); err == nil { + t.Fatal("legacy DNS-owned networking integration was accepted") + } +} + +func TestConfigLoaderRejectsRemovedPrivateServingFields(t *testing.T) { + for _, field := range []string{"authoritativeBackends", "sourcePrefix", "minAuthoritativeReplicas", "resolverContexts", "cacheSize", "authorizationTTLSeconds"} { + t.Run(field, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.json") + if err := os.WriteFile(path, []byte("{\""+field+"\":null}"), 0600); err != nil { + t.Fatal(err) + } + _, err := LoadConfig(path) + if err == nil || !strings.Contains(err.Error(), "unknown field") { + t.Fatalf("removed field accepted: %v", err) + } + }) + } + path := filepath.Join(t.TempDir(), "config.json") + if err := os.WriteFile(path, []byte("{\"region\":\"central\",\"shard\":\"shared\"}"), 0600); err != nil { + t.Fatal(err) + } + loaded, err := LoadConfig(path) + if err != nil || loaded.Region != "central" { + t.Fatalf("current config rejected: %#v %v", loaded, err) + } + if err := os.WriteFile(path, []byte("{} {}"), 0600); err != nil { + t.Fatal(err) + } + if _, err := LoadConfig(path); err == nil { + t.Fatal("multiple config objects accepted") + } +} + +func TestAgentRestoresCheckpointWhileBrokerUnavailable(t *testing.T) { + dir := t.TempDir() + cfg := Config{Region: "central", Shard: "one", NATS: transport.Config{URL: "nats://127.0.0.1:1", AllowInsecure: true}, Stream: transport.StreamConfig{Name: "DNS", SnapshotsBucket: "DNS_BOOTSTRAP"}, Agent: serving.Config{MemberID: "node-0", StateDir: filepath.Join(dir, "state"), Mode: serving.ModeResolver, Render: serving.RenderConfig{Role: serving.RenderNode, NodeDNSDist: serving.DNSDistRenderConfig{Path: filepath.Join(dir, "dnsdist.conf"), ListenAddress: "[::1]:53", PoolName: "node"}, NodeCoreDNS: serving.CoreDNSRenderConfig{Path: filepath.Join(dir, "named.conf"), ListenAddress: "::1", Port: 5353, ProxyPeers: []string{"::1"}}}, Transaction: serving.TransactionConfig{StageDir: filepath.Join(dir, "stage")}, UnsafeAllowNoFailClosed: true, UnsafeAllowNoDNSVerify: true, ReconcileInterval: 10 * time.Millisecond}} + if err := cfg.Validate(roleAgent); err != nil { + t.Fatal(err) + } + seed, err := serving.NewAgent(cfg.Agent, serving.Dependencies{}) + if err != nil { + t.Fatal(err) + } + snapshot := model.ServingSnapshot{Region: "central", Shard: "one", ConfigurationEpoch: 1, ConfigurationRevision: 1, GeneratedAt: time.Now(), Bindings: []model.Binding{{BindingUID: "binding", ProjectUID: "project", ContextUID: "context", BindingGeneration: 1, ConfigurationRevision: 1, ConsumerAddress: "fd53::1", ClusterAddress: "fd53::2", Port: 53, Transports: []model.Transport{model.TransportUDP, model.TransportTCP}, Version: model.AccessVersion{Epoch: 1, Sequence: 1}}}} + env, err := model.NewEnvelope(model.KindServingSnapshot, "initial", "central", "one", "shard", 1, 1, time.Now(), snapshot) + if err != nil { + t.Fatal(err) + } + if err := seed.Handle(context.Background(), env); err != nil { + t.Fatal(err) + } + if err := os.Remove(cfg.Agent.Render.NodeDNSDist.Path); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- runAgent(ctx, cfg) }() + deadline := time.Now().Add(3 * time.Second) + for { + if _, err := os.Stat(cfg.Agent.Render.NodeDNSDist.Path); err == nil { + break + } + if time.Now().After(deadline) { + t.Fatal("checkpoint activation waited for broker connectivity") + } + time.Sleep(10 * time.Millisecond) + } + data, err := os.ReadFile(cfg.Agent.Render.NodeDNSDist.Path) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), "fd53::1") || !strings.Contains(string(data), "ready=true") { + t.Fatalf("persisted resolver mapping was not restored: %s", data) + } + cancel() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(3 * time.Second): + t.Fatal("agent did not stop") + } +} + +func TestControlPlaneProjectDiscoveryValidation(t *testing.T) { + base := Config{Region: "central", Shard: "one", NATS: transport.Config{URL: "nats://fixture:4222", AllowInsecure: true}, Stream: transport.StreamConfig{Name: "DNS", SnapshotsBucket: "DNS_BOOTSTRAP"}, Members: []model.Member{{MemberID: "regional", Role: "regional"}}, ProjectDiscovery: &ProjectDiscoveryConfig{Kubeconfig: "/var/run/discovery/kubeconfig"}} + if err := base.Validate(roleControlPlane); err != nil { + t.Fatal(err) + } + if base.ProjectDiscovery.IntervalSeconds != 10 || base.ProjectDiscovery.KubeAPI.RequestTimeoutSeconds != 10 { + t.Fatal("discovery defaults missing") + } + mixed := base + mixed.Projects = []ProjectConfig{{ProjectUID: "project", Namespace: "project"}} + if err := mixed.Validate(roleControlPlane); err == nil { + t.Fatal("mixed discovery and static sources accepted") + } + relative := base + relative.ProjectDiscovery = &ProjectDiscoveryConfig{Kubeconfig: "relative"} + if err := relative.Validate(roleControlPlane); err == nil { + t.Fatal("relative discovery credential accepted") + } +} + +func TestLocalServingRolesDoNotRequireDeliveryConfiguration(t *testing.T) { + for _, role := range []string{"serving", "readiness"} { + t.Run(role, func(t *testing.T) { + dir := t.TempDir() + cfg := Config{Region: "central", Shard: "one", Agent: serving.Config{MemberID: "node", StateDir: filepath.Join(dir, "state"), Mode: serving.ModeResolver, Render: serving.RenderConfig{Role: serving.RenderNode, NodeDNSDist: serving.DNSDistRenderConfig{Path: filepath.Join(dir, "dnsdist.conf"), ListenAddress: "[::1]:53", PoolName: "node"}, NodeCoreDNS: serving.CoreDNSRenderConfig{Path: filepath.Join(dir, "named.conf"), ListenAddress: "::1", Port: 5353, ProxyPeers: []string{"::1"}}}, Transaction: serving.TransactionConfig{StageDir: filepath.Join(dir, "stage"), FailClosed: []serving.Command{{Path: "/bin/true"}}}}} + if err := cfg.Validate(role); err != nil { + t.Fatalf("local role required messaging or control-plane configuration: %v", err) + } + }) + } +} diff --git a/internal/internaldns/runtime/deadline_client.go b/internal/internaldns/runtime/deadline_client.go new file mode 100644 index 00000000..d3717798 --- /dev/null +++ b/internal/internaldns/runtime/deadline_client.go @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package runtime + +import ( + "context" + "time" + + k8sruntime "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// deadlineClient bounds every direct source API operation without imposing a +// short lifetime on the manager's long-running watch connections. +type deadlineClient struct { + client.Client + Timeout time.Duration +} + +func (c *deadlineClient) bounded(ctx context.Context) (context.Context, context.CancelFunc) { + if c.Timeout <= 0 { + return context.WithCancel(ctx) + } + if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) <= c.Timeout { + return context.WithCancel(ctx) + } + return context.WithTimeout(ctx, c.Timeout) +} + +func (c *deadlineClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + ctx, cancel := c.bounded(ctx) + defer cancel() + return c.Client.Get(ctx, key, obj, opts...) +} +func (c *deadlineClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + ctx, cancel := c.bounded(ctx) + defer cancel() + return c.Client.List(ctx, list, opts...) +} +func (c *deadlineClient) Apply(ctx context.Context, obj k8sruntime.ApplyConfiguration, opts ...client.ApplyOption) error { + ctx, cancel := c.bounded(ctx) + defer cancel() + return c.Client.Apply(ctx, obj, opts...) +} +func (c *deadlineClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + ctx, cancel := c.bounded(ctx) + defer cancel() + return c.Client.Create(ctx, obj, opts...) +} +func (c *deadlineClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + ctx, cancel := c.bounded(ctx) + defer cancel() + return c.Client.Delete(ctx, obj, opts...) +} +func (c *deadlineClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + ctx, cancel := c.bounded(ctx) + defer cancel() + return c.Client.Update(ctx, obj, opts...) +} +func (c *deadlineClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + ctx, cancel := c.bounded(ctx) + defer cancel() + return c.Client.Patch(ctx, obj, patch, opts...) +} +func (c *deadlineClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + ctx, cancel := c.bounded(ctx) + defer cancel() + return c.Client.DeleteAllOf(ctx, obj, opts...) +} +func (c *deadlineClient) Status() client.SubResourceWriter { + return &deadlineSubresourceWriter{client: c, subresource: c.Client.Status()} +} +func (c *deadlineClient) SubResource(name string) client.SubResourceClient { + return &deadlineSubresource{client: c, subresource: c.Client.SubResource(name)} +} + +type deadlineSubresource struct { + client *deadlineClient + subresource client.SubResourceClient +} + +type deadlineSubresourceWriter struct { + client *deadlineClient + subresource client.SubResourceWriter +} + +func (s *deadlineSubresourceWriter) Create(ctx context.Context, obj client.Object, sub client.Object, opts ...client.SubResourceCreateOption) error { + ctx, cancel := s.client.bounded(ctx) + defer cancel() + return s.subresource.Create(ctx, obj, sub, opts...) +} +func (s *deadlineSubresourceWriter) Update(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error { + ctx, cancel := s.client.bounded(ctx) + defer cancel() + return s.subresource.Update(ctx, obj, opts...) +} +func (s *deadlineSubresourceWriter) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + ctx, cancel := s.client.bounded(ctx) + defer cancel() + return s.subresource.Patch(ctx, obj, patch, opts...) +} +func (s *deadlineSubresourceWriter) Apply(ctx context.Context, obj k8sruntime.ApplyConfiguration, opts ...client.SubResourceApplyOption) error { + ctx, cancel := s.client.bounded(ctx) + defer cancel() + return s.subresource.Apply(ctx, obj, opts...) +} + +func (s *deadlineSubresource) Get(ctx context.Context, obj client.Object, sub client.Object, opts ...client.SubResourceGetOption) error { + ctx, cancel := s.client.bounded(ctx) + defer cancel() + return s.subresource.Get(ctx, obj, sub, opts...) +} +func (s *deadlineSubresource) Create(ctx context.Context, obj client.Object, sub client.Object, opts ...client.SubResourceCreateOption) error { + ctx, cancel := s.client.bounded(ctx) + defer cancel() + return s.subresource.Create(ctx, obj, sub, opts...) +} +func (s *deadlineSubresource) Update(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error { + ctx, cancel := s.client.bounded(ctx) + defer cancel() + return s.subresource.Update(ctx, obj, opts...) +} +func (s *deadlineSubresource) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + ctx, cancel := s.client.bounded(ctx) + defer cancel() + return s.subresource.Patch(ctx, obj, patch, opts...) +} +func (s *deadlineSubresource) Apply(ctx context.Context, obj k8sruntime.ApplyConfiguration, opts ...client.SubResourceApplyOption) error { + ctx, cancel := s.client.bounded(ctx) + defer cancel() + return s.subresource.Apply(ctx, obj, opts...) +} + +var _ client.Client = (*deadlineClient)(nil) diff --git a/internal/internaldns/runtime/deadline_client_test.go b/internal/internaldns/runtime/deadline_client_test.go new file mode 100644 index 00000000..7d307d99 --- /dev/null +++ b/internal/internaldns/runtime/deadline_client_test.go @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package runtime + +import ( + "context" + "errors" + "testing" + "time" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type blockedSourceClient struct{ client.Client } + +func (blockedSourceClient) List(ctx context.Context, _ client.ObjectList, _ ...client.ListOption) error { + <-ctx.Done() + return ctx.Err() +} + +func TestDeadlineClientBoundsPartitionedSourceRequests(t *testing.T) { + c := &deadlineClient{Client: blockedSourceClient{}, Timeout: 25 * time.Millisecond} + started := time.Now() + err := c.List(context.Background(), nil) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("partitioned source request returned %v", err) + } + if elapsed := time.Since(started); elapsed > 250*time.Millisecond { + t.Fatalf("partitioned source request exceeded its bound: %s", elapsed) + } +} diff --git a/internal/internaldns/runtime/deployment_examples_test.go b/internal/internaldns/runtime/deployment_examples_test.go new file mode 100644 index 00000000..cb288ac1 --- /dev/null +++ b/internal/internaldns/runtime/deployment_examples_test.go @@ -0,0 +1,141 @@ +package runtime + +import ( + "bytes" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "go.miloapis.com/dns-operator/internal/internaldns/model" + appsv1 "k8s.io/api/apps/v1" + k8syaml "k8s.io/apimachinery/pkg/util/yaml" +) + +func TestDeploymentExampleConfigsValidate(t *testing.T) { + t.Parallel() + root := filepath.Join("..", "..", "..", "config", "internal-dns") + cases := []struct { + file string + namePrefix string + roles []string + }{ + {file: "control-plane.example.yaml", namePrefix: "internal-dns-control-plane-config", roles: []string{"control-plane"}}, + {file: "fleet.example.yaml", namePrefix: "internal-dns-", roles: []string{"agent", "readiness"}}, + } + for _, tc := range cases { + t.Run(tc.file, func(t *testing.T) { + contents, err := os.ReadFile(filepath.Join(root, tc.file)) + if err != nil { + t.Fatal(err) + } + decoder := k8syaml.NewYAMLOrJSONDecoder(bytes.NewReader(contents), 4096) + found := 0 + for { + var object map[string]any + if err := decoder.Decode(&object); err == io.EOF { + break + } else if err != nil { + t.Fatal(err) + } + if object["kind"] != "ConfigMap" { + continue + } + metadata, _ := object["metadata"].(map[string]any) + name, _ := metadata["name"].(string) + if !strings.HasPrefix(name, tc.namePrefix) { + continue + } + data, _ := object["data"].(map[string]any) + raw, _ := data["config.json"].(string) + if raw == "" { + continue + } + found++ + for _, role := range tc.roles { + configPath := filepath.Join(t.TempDir(), "config.json") + if err := os.WriteFile(configPath, []byte(raw), 0600); err != nil { + t.Fatal(err) + } + config, err := LoadConfig(configPath) + if err != nil { + t.Fatalf("%s config JSON: %v", name, err) + } + if err := config.Validate(role); err != nil { + t.Errorf("%s role %s: %v", name, role, err) + } + } + } + if found == 0 { + t.Fatal("no example runtime ConfigMaps found") + } + }) + } +} + +func TestNATSExampleUsesExactMemberAckSubjects(t *testing.T) { + t.Parallel() + path := filepath.Join("..", "..", "..", "config", "internal-dns", "nats-server.conf.example") + contents, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + for _, memberID := range []string{"node-us-central1-0", "regional-front-us-central1-0", "regional-coredns-us-central1-0", "regional-coredns-us-central1-1"} { + subject := model.AckSubject("us-central1", "shared-0", memberID) + if !bytes.Contains(contents, []byte(`"`+subject+`"`)) { + t.Errorf("NATS policy is missing exact ACK subject %q", subject) + } + } +} + +func TestQueryDaemonsCannotShareCredentialAgentIdentity(t *testing.T) { + for _, file := range []string{"fleet.example.yaml", "routers.example.yaml"} { + contents, err := os.ReadFile(filepath.Join("..", "..", "..", "config", "internal-dns", file)) + if err != nil { + t.Fatal(err) + } + decoder := k8syaml.NewYAMLOrJSONDecoder(bytes.NewReader(contents), 4096) + count := 0 + for { + var deployment appsv1.Deployment + if err := decoder.Decode(&deployment); err == io.EOF { + break + } else if err != nil { + t.Fatal(err) + } + if deployment.Kind != "Deployment" { + continue + } + for _, container := range deployment.Spec.Template.Spec.Containers { + if container.Name != "agent" && container.Name != "serving" && container.Name != "dnsdist" { + continue + } + count++ + expected := int64(65533) + if container.Name == "agent" { + expected = 65532 + } + sc := container.SecurityContext + if sc == nil || sc.RunAsUser == nil || *sc.RunAsUser != expected { + t.Fatalf("%s/%s must use UID %d", deployment.Name, container.Name, expected) + } + for _, capability := range sc.Capabilities.Add { + if capability == "SYS_PTRACE" { + t.Fatal("query Pods must not grant process inspection") + } + } + if container.Name != "agent" { + for _, mount := range container.VolumeMounts { + if mount.Name == "nats" { + t.Fatal("query daemon mounts broker credentials") + } + } + } + } + } + if count == 0 { + t.Fatalf("no query containers in %s", file) + } + } +} diff --git a/internal/internaldns/runtime/deployment_permissions_test.go b/internal/internaldns/runtime/deployment_permissions_test.go new file mode 100644 index 00000000..6cfab9ee --- /dev/null +++ b/internal/internaldns/runtime/deployment_permissions_test.go @@ -0,0 +1,301 @@ +package runtime + +import ( + "bytes" + "context" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" + authorizationv1 "k8s.io/api/authorization/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + k8syaml "k8s.io/apimachinery/pkg/util/yaml" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +func deploymentObjects(t *testing.T, name string) []*unstructured.Unstructured { + t.Helper() + data, err := os.ReadFile(filepath.Join("..", "..", "..", "config", "internal-dns", name)) + if err != nil { + t.Fatal(err) + } + decoder := k8syaml.NewYAMLOrJSONDecoder(bytes.NewReader(data), 4096) + var objects []*unstructured.Unstructured + for { + var object unstructured.Unstructured + err := decoder.Decode(&object.Object) + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + if object.GetKind() != "" { + objects = append(objects, &object) + } + } + return objects +} + +// Use the Kubernetes RBAC authorizer and the actual shipped Role/ClusterRole, +// rather than accepting reconciliation under an administrator kubeconfig. +func TestDeploymentRolesAuthorizeWorkerAndDiscoveryBoundaries(t *testing.T) { + assets := os.Getenv("KUBEBUILDER_ASSETS") + if assets == "" { + t.Skip("set KUBEBUILDER_ASSETS to verify packaged roles with the Kubernetes RBAC authorizer") + } + environment := &envtest.Environment{BinaryAssetsDirectory: assets, CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, ErrorIfCRDPathMissing: true} + environment.ControlPlane.GetAPIServer().Configure().Set("authorization-mode", "RBAC") + cfg, err := environment.Start() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := environment.Stop(); err != nil { + t.Error(err) + } + }) + scheme, err := newScheme() + if err != nil { + t.Fatal(err) + } + if err := authorizationv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + cl, err := client.New(cfg, client.Options{Scheme: scheme}) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + for _, namespace := range []string{"example-project", "internal-dns-system"} { + if err := cl.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}); err != nil { + t.Fatal(err) + } + } + verifyProjectReferenceSchemas(t, ctx, cl) + for _, object := range deploymentObjects(t, "rbac.yaml") { + if err := cl.Create(ctx, object); err != nil { + t.Fatal(err) + } + } + verifyStateNamespaceBoundary(t, ctx, cl) + for _, identity := range []struct{ name, role string }{{"product", "internal-dns-product-writer"}, {"integration", "internal-dns-vpc-integration"}, {"customer", "internal-dns-customer"}} { + binding := &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: identity.name + "-test", Namespace: "example-project"}, Subjects: []rbacv1.Subject{{Kind: "ServiceAccount", Name: identity.name, Namespace: "example-project"}}, RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "Role", Name: identity.role}} + if err := cl.Create(ctx, binding); err != nil { + t.Fatal(err) + } + } + cases := []struct { + principal, resource, subresource, verb string + allowed bool + }{ + {"internal-dns-control-plane", "dnsrecordsets", "", "list", true}, + {"internal-dns-control-plane", "dnsrecordsets", "", "watch", true}, + {"internal-dns-control-plane", "dnszones", "", "create", true}, + {"internal-dns-control-plane", "dnszones", "", "patch", true}, + {"internal-dns-control-plane", "dnszoneassociations", "", "create", true}, + {"internal-dns-control-plane", "dnszones", "status", "patch", true}, + {"internal-dns-control-plane", "dnszoneassociations", "status", "patch", true}, + {"internal-dns-control-plane", "dnsnamingpolicies", "status", "patch", true}, + {"internal-dns-control-plane", "dnsregistrations", "status", "patch", true}, + {"internal-dns-control-plane", "dnspublicationmanifests", "", "create", true}, + {"product", "dnsresolvercontexts", "", "get", true}, + {"product", "dnsresolvercontexts", "", "list", true}, + {"product", "dnsnamingpolicies", "", "get", true}, + {"product", "dnsnamingpolicies", "", "list", true}, + {"product", "dnsnamingpolicies", "", "patch", false}, + {"product", "dnsnamingpolicies", "status", "patch", false}, + {"product", "dnsregistrations", "", "create", true}, + {"product", "dnsrecordcontributions", "status", "patch", false}, + {"product", "dnsrecordcontributions", "", "patch", true}, + {"product", "dnsresolvercontexts", "", "patch", false}, + {"product", "dnsresolveraccessbindings", "", "create", false}, + {"product", "dnspublicationmanifests", "", "create", false}, + {"customer", "dnszones", "", "create", true}, + {"customer", "dnszoneassociations", "", "create", true}, + {"customer", "dnsnamingpolicies", "", "create", true}, + {"customer", "dnsresolvercontexts", "", "get", true}, + {"customer", "dnsresolvercontexts", "", "patch", false}, + {"customer", "dnsresolveraccessbindings", "", "create", false}, + {"customer", "dnszoneassociations", "status", "patch", false}, + {"customer", "dnspublicationmanifests", "", "create", false}, + {"integration", "dnsresolvercontexts", "", "create", true}, + {"integration", "dnsresolveraccessbindings", "", "create", true}, + {"integration", "dnsresolvercontexts", "status", "patch", false}, + {"integration", "dnspublicationmanifests", "", "create", false}, + } + for _, tc := range cases { + namespace := "example-project" + if tc.principal == "internal-dns-control-plane" { + namespace = "internal-dns-system" + } + user := "system:serviceaccount:" + namespace + ":" + tc.principal + deadline := time.Now().Add(5 * time.Second) + for { + sar := &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{User: user, Groups: []string{"system:authenticated", "system:serviceaccounts", "system:serviceaccounts:" + namespace}, ResourceAttributes: &authorizationv1.ResourceAttributes{Namespace: "example-project", Group: "dns.networking.miloapis.com", Resource: tc.resource, Subresource: tc.subresource, Verb: tc.verb}}} + if err := cl.Create(ctx, sar); err != nil { + t.Fatal(err) + } + if sar.Status.Allowed == tc.allowed { + break + } + if time.Now().After(deadline) { + t.Fatalf("%s %s %s/%s allowed=%v want=%v: %s", user, tc.verb, tc.resource, tc.subresource, sar.Status.Allowed, tc.allowed, sar.Status.Reason) + } + time.Sleep(20 * time.Millisecond) + } + } +} + +func TestEveryMemberCanCreateAllRuntimeConsumers(t *testing.T) { + data, err := os.ReadFile(filepath.Join("..", "..", "..", "config", "internal-dns", "nats-server.conf.example")) + if err != nil { + t.Fatal(err) + } + contents := string(data) + for _, member := range []string{"node-us-central1-0", "regional-front-us-central1-0", "regional-coredns-us-central1-0", "regional-coredns-us-central1-1"} { + start := strings.Index(contents, `user: "`+member+`"`) + if start < 0 { + t.Fatalf("missing member %s", member) + } + policy := strings.SplitN(contents[start:], "\n },", 2)[0] + for _, suffix := range []string{"serving", "records", "acks"} { + consumer := member + "-" + suffix + for _, subject := range []string{ + "$JS.API.CONSUMER.INFO.DNS_PRIVATE." + consumer, + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE." + consumer, + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE." + consumer + ".>", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE." + consumer, + "$JS.ACK.DNS_PRIVATE." + consumer + ".>", + } { + if !strings.Contains(policy, `"`+subject+`"`) { + t.Errorf("%s lacks startup/fetch permission %s", member, subject) + } + } + } + for _, broad := range []string{`"$JS.API.>"`, `"dns.private.records.>"`, `"dns.private.serving.>"`, `"dns.private.acks.>"`} { + if strings.Contains(policy, broad) { + t.Errorf("member %s has authority to write %s", member, broad) + } + } + } +} + +func TestReadOnlyServingImageHasWritableCoreDNSWorkingDirectory(t *testing.T) { + found := false + for _, object := range deploymentObjects(t, "fleet.example.yaml") { + if object.GetKind() != "Deployment" { + continue + } + containers, _, err := unstructured.NestedSlice(object.Object, "spec", "template", "spec", "containers") + if err != nil { + t.Fatal(err) + } + for _, value := range containers { + container := value.(map[string]interface{}) + if container["name"] != "serving" { + continue + } + found = true + mounts, _, err := unstructured.NestedSlice(container, "volumeMounts") + if err != nil { + t.Fatal(err) + } + runtimeVolume := "" + for _, value := range mounts { + mount := value.(map[string]interface{}) + if mount["mountPath"] == "/tmp" && mount["readOnly"] != true { + runtimeVolume, _ = mount["name"].(string) + } + } + if runtimeVolume == "" { + t.Fatal("read-only serving image lacks writable CoreDNS /tmp mount") + } + volumes, _, err := unstructured.NestedSlice(object.Object, "spec", "template", "spec", "volumes") + if err != nil { + t.Fatal(err) + } + writable := false + for _, value := range volumes { + volume := value.(map[string]interface{}) + if volume["name"] == runtimeVolume && volume["emptyDir"] != nil { + writable = true + } + } + if !writable { + t.Fatal("CoreDNS working-directory mount is not a per-member writable runtime volume") + } + } + } + if !found { + t.Fatal("no serving container was checked") + } +} + +func verifyProjectReferenceSchemas(t *testing.T, ctx context.Context, cl client.Client) { + t.Helper() + zone := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "private", Namespace: "example-project"}, Spec: dnsv1alpha1.DNSZoneSpec{DomainName: "schema.internal", Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate, DNSZoneClassName: "private"}} + if err := cl.Create(ctx, zone); err != nil { + t.Fatal(err) + } + ref := dnsv1alpha1.DNSObjectReference{Name: zone.Name, UID: zone.UID} + contextRef := dnsv1alpha1.DNSObjectReference{Name: "context", UID: "context-uid"} + cases := []struct { + name string + object client.Object + valid bool + }{ + {"registration-valid", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: ref, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 30}}, true}, + {"registration-unpinned", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name}, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 30}}, false}, + {"registration-ttl31", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: ref, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 31}}, false}, + {"association-valid", &dnsv1alpha1.DNSZoneAssociation{Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: ref, ResolverContextRef: contextRef}}, true}, + {"association-unpinned", &dnsv1alpha1.DNSZoneAssociation{Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name}, ResolverContextRef: contextRef}}, false}, + {"naming-valid", &dnsv1alpha1.DNSNamingPolicy{Spec: dnsv1alpha1.DNSNamingPolicySpec{ResolverContextRef: contextRef, AdditionalNames: []dnsv1alpha1.DNSAdditionalNameRule{{Selector: nil, DNSZoneRef: ref, NamePrefix: "services"}}}}, true}, + {"naming-unpinned", &dnsv1alpha1.DNSNamingPolicy{Spec: dnsv1alpha1.DNSNamingPolicySpec{ResolverContextRef: contextRef, AdditionalNames: []dnsv1alpha1.DNSAdditionalNameRule{{Selector: nil, DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name}, NamePrefix: "services"}}}}, false}, + } + for _, tc := range cases { + tc.object.SetName(tc.name) + tc.object.SetNamespace("example-project") + err := cl.Create(ctx, tc.object) + if tc.valid && err != nil { + t.Errorf("valid %s rejected: %v", tc.name, err) + } + if !tc.valid && !apierrors.IsInvalid(err) { + t.Errorf("invalid %s should fail API schema validation: %v", tc.name, err) + } + } +} + +func verifyStateNamespaceBoundary(t *testing.T, ctx context.Context, cl client.Client) { + t.Helper() + for _, resource := range []struct{ group, name string }{{"", "configmaps"}, {"coordination.k8s.io", "leases"}} { + for _, namespace := range []string{"internal-dns-system", "example-project"} { + for _, verb := range []string{"get", "create", "patch"} { + allowed := namespace == "internal-dns-system" + deadline := time.Now().Add(5 * time.Second) + for { + review := &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{User: "system:serviceaccount:internal-dns-system:internal-dns-control-plane", Groups: []string{"system:authenticated", "system:serviceaccounts", "system:serviceaccounts:internal-dns-system"}, ResourceAttributes: &authorizationv1.ResourceAttributes{Namespace: namespace, Group: resource.group, Resource: resource.name, Verb: verb}}} + if err := cl.Create(ctx, review); err != nil { + t.Fatal(err) + } + if review.Status.Allowed == allowed { + break + } + if time.Now().After(deadline) { + t.Fatalf("state boundary %s %s in %s allowed=%t want=%t", verb, resource.name, namespace, review.Status.Allowed, allowed) + } + time.Sleep(20 * time.Millisecond) + } + } + } + } +} diff --git a/internal/internaldns/runtime/identity_test.go b/internal/internaldns/runtime/identity_test.go new file mode 100644 index 00000000..228b1043 --- /dev/null +++ b/internal/internaldns/runtime/identity_test.go @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package runtime + +import ( + "fmt" + "os" + "os/exec" + "strings" + "testing" +) + +func TestHolderIdentityHelper(t *testing.T) { + if os.Getenv("INTERNAL_DNS_IDENTITY_HELPER") != "1" { + return + } + fmt.Printf("%s\n%s\n", holderIdentity(), holderIdentity()) +} + +func TestHolderIdentityIsStablePerProcessAndUniqueAcrossSameHostProcesses(t *testing.T) { + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + run := func() string { + cmd := exec.Command(executable, "-test.run=^TestHolderIdentityHelper$") + cmd.Env = append(os.Environ(), "INTERNAL_DNS_IDENTITY_HELPER=1") + output, err := cmd.Output() + if err != nil { + t.Fatal(err) + } + identities := strings.Fields(string(output)) + if len(identities) < 2 || identities[0] != identities[1] { + t.Fatalf("holder identity changed within one process: %q", output) + } + return identities[0] + } + first, second := run(), run() + if first == second { + t.Fatalf("same-host processes shared holder identity %q", first) + } +} diff --git a/internal/internaldns/runtime/project_discovery.go b/internal/internaldns/runtime/project_discovery.go new file mode 100644 index 00000000..62cb1349 --- /dev/null +++ b/internal/internaldns/runtime/project_discovery.go @@ -0,0 +1,193 @@ +package runtime + +import ( + "context" + "errors" + "fmt" + "net/url" + "path" + "sort" + "sync" + + "go.miloapis.com/dns-operator/internal/internaldns/platform" + resourcemanagerv1alpha1 "go.miloapis.com/milo/pkg/apis/resourcemanager/v1alpha1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + kubeadmission "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +type directoryProject struct { + projectRuntime + cancel context.CancelFunc +} + +type projectDirectory struct { + mu sync.RWMutex + projects map[string]directoryProject + discover client.Reader +} + +func newProjectDirectory(projects []projectRuntime) *projectDirectory { + d := &projectDirectory{projects: map[string]directoryProject{}} + for _, p := range projects { + d.projects[p.config.Name] = directoryProject{projectRuntime: p} + } + return d +} + +func (d *projectDirectory) scopes() []platform.ProjectScope { + d.mu.RLock() + defer d.mu.RUnlock() + scopes := make([]platform.ProjectScope, 0, len(d.projects)) + names := make([]string, 0, len(d.projects)) + for name := range d.projects { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + p := d.projects[name] + scopes = append(scopes, platform.ProjectScope{Client: p.direct, Namespace: p.config.Namespace, RequestTimeout: p.config.KubeAPI.requestTimeout()}) + } + return scopes +} + +func (d *projectDirectory) selectProject(ctx context.Context, req kubeadmission.Request) (client.Client, error) { + parents := req.UserInfo.Extra[parentNameExtra] + d.mu.RLock() + if d.discover == nil { + projects := make([]projectRuntime, 0, len(d.projects)) + for _, p := range d.projects { + projects = append(projects, p.projectRuntime) + } + d.mu.RUnlock() + return projectSelector(projects)(ctx, req) + } + reader := d.discover + d.mu.RUnlock() + if len(parents) != 1 || req.Namespace == "" { + return nil, errors.New("authenticated project parent is missing or ambiguous") + } + var project resourcemanagerv1alpha1.Project + if err := reader.Get(ctx, client.ObjectKey{Name: parents[0]}, &project); err != nil { + return nil, fmt.Errorf("observe authenticated project: %w", err) + } + if !project.DeletionTimestamp.IsZero() || !apimeta.IsStatusConditionTrue(project.Status.Conditions, resourcemanagerv1alpha1.ProjectReady) { + return nil, errors.New("authenticated project is unavailable") + } + d.mu.RLock() + defer d.mu.RUnlock() + p, found := d.projects[parents[0]] + if !found || p.config.ProjectUID != project.UID { + return nil, errors.New("authenticated project source is not ready") + } + return p.direct, nil +} + +type projectDiscovery struct { + client client.Client + directory *projectDirectory + rest *rest.Config + kubeAPI KubeAPIClientConfig + makeProject func(ProjectConfig, *rest.Config) (projectRuntime, error) +} + +func (d *projectDiscovery) step(ctx context.Context) error { + d.directory.mu.Lock() + d.directory.discover = d.client + d.directory.mu.Unlock() + var projects resourcemanagerv1alpha1.ProjectList + if err := d.client.List(ctx, &projects); err != nil { + return err + } + seen := map[string]bool{} + var failures error + for i := range projects.Items { + p := &projects.Items[i] + seen[p.Name] = true + if p.UID == "" { + failures = errors.Join(failures, fmt.Errorf("project %s has no identity", p.Name)) + continue + } + d.directory.mu.RLock() + existing, exists := d.directory.projects[p.Name] + d.directory.mu.RUnlock() + ready := apimeta.IsStatusConditionTrue(p.Status.Conditions, resourcemanagerv1alpha1.ProjectReady) + if exists && existing.config.ProjectUID != p.UID { + d.remove(p.Name) + exists = false + } + if !ready || exists { + continue + } + projectREST, err := projectRESTConfig(d.rest, p.Name) + if err != nil { + failures = errors.Join(failures, err) + continue + } + worker, err := d.makeProject(ProjectConfig{Name: p.Name, ProjectUID: p.UID, KubeAPI: d.kubeAPI}, projectREST) + if err != nil { + failures = errors.Join(failures, err) + continue + } + sourceCtx, cancel := context.WithCancel(ctx) + d.directory.mu.Lock() + d.directory.projects[p.Name] = directoryProject{projectRuntime: worker, cancel: cancel} + d.directory.mu.Unlock() + go runProjectSource(sourceCtx, worker) + } + d.directory.mu.RLock() + names := make([]string, 0, len(d.directory.projects)) + for name := range d.directory.projects { + names = append(names, name) + } + d.directory.mu.RUnlock() + for _, name := range names { + if !seen[name] { + d.remove(name) + } + } + return failures +} + +func (d *projectDiscovery) remove(name string) { + d.directory.mu.Lock() + defer d.directory.mu.Unlock() + if p, found := d.directory.projects[name]; found { + if p.cancel != nil { + p.cancel() + } + delete(d.directory.projects, name) + } +} + +func projectRESTConfig(base *rest.Config, name string) (*rest.Config, error) { + endpoint, err := url.Parse(base.Host) + if err != nil || endpoint.Scheme == "" || endpoint.Host == "" { + return nil, fmt.Errorf("invalid project discovery API endpoint %q", base.Host) + } + endpoint.Path = path.Join(endpoint.Path, "apis/resourcemanager.miloapis.com/v1alpha1/projects", name, "control-plane") + config := rest.CopyConfig(base) + config.Host = endpoint.String() + return config, nil +} + +func projectCache(namespace string) cache.Options { + if namespace == "" { + return cache.Options{} + } + return cache.Options{DefaultNamespaces: map[string]cache.Config{namespace: {}}} +} + +func verifyProjectIdentity(ctx context.Context, reader client.Reader, name string, expected types.UID) error { + var project resourcemanagerv1alpha1.Project + if err := reader.Get(ctx, client.ObjectKey{Name: name}, &project); err != nil { + return fmt.Errorf("observe source project identity: %w", err) + } + if project.UID != expected { + return errors.New("source project lifetime changed") + } + return nil +} diff --git a/internal/internaldns/runtime/project_discovery_test.go b/internal/internaldns/runtime/project_discovery_test.go new file mode 100644 index 00000000..9440587e --- /dev/null +++ b/internal/internaldns/runtime/project_discovery_test.go @@ -0,0 +1,186 @@ +package runtime + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + dnsv1 "go.miloapis.com/dns-operator/api/v1alpha1" + resourcemanagerv1alpha1 "go.miloapis.com/milo/pkg/apis/resourcemanager/v1alpha1" + admissionv1 "k8s.io/api/admission/v1" + authenticationv1 "k8s.io/api/authentication/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + kubeadmission "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +type discoveryFixtureClient struct { + client.Client + unavailable bool +} + +func (c *discoveryFixtureClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if c.unavailable { + return errors.New("discovery unavailable") + } + return c.Client.Get(ctx, key, obj, opts...) +} + +func (c *discoveryFixtureClient) List(ctx context.Context, obj client.ObjectList, opts ...client.ListOption) error { + if c.unavailable { + return errors.New("discovery unavailable") + } + return c.Client.List(ctx, obj, opts...) +} + +func TestProjectDiscoveryAddsSourcesAndPreservesThemDuringOutage(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + scheme, err := newScheme() + if err != nil { + t.Fatal(err) + } + first := readyDiscoveryProject("first", "first-uid") + core := &discoveryFixtureClient{Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(first).Build()} + directory := newProjectDirectory(nil) + starts := make(chan types.UID, 4) + d := &projectDiscovery{client: core, directory: directory, rest: &rest.Config{Host: "https://milo.example:6443"}, makeProject: func(config ProjectConfig, sourceREST *rest.Config) (projectRuntime, error) { + if config.Namespace != "" || !strings.HasSuffix(sourceREST.Host, "/projects/"+config.Name+"/control-plane") { + t.Errorf("unexpected discovery scope: %#v %s", config, sourceREST.Host) + } + direct := fake.NewClientBuilder().WithScheme(scheme).Build() + return projectRuntime{config: config, direct: direct, start: func(workerCtx context.Context) error { + starts <- config.ProjectUID + <-workerCtx.Done() + return workerCtx.Err() + }}, nil + }} + if err := d.step(ctx); err != nil { + t.Fatal(err) + } + awaitDiscoveredSource(t, starts, first.UID) + second := readyDiscoveryProject("second", "second-uid") + if err := core.Create(ctx, second); err != nil { + t.Fatal(err) + } + if err := d.step(ctx); err != nil { + t.Fatal(err) + } + awaitDiscoveredSource(t, starts, second.UID) + scopes := directory.scopes() + if len(scopes) != 2 || scopes[0].Namespace != "" || scopes[1].Namespace != "" { + t.Fatalf("ACK scopes did not follow discovery: %#v", scopes) + } + if err := verifyProjectIdentity(ctx, core, first.Name, first.UID); err != nil { + t.Fatal(err) + } + core.unavailable = true + if err := verifyProjectIdentity(ctx, core, first.Name, first.UID); err == nil { + t.Fatal("unobserved source identity permitted compilation") + } + if err := d.step(ctx); err == nil { + t.Fatal("expected discovery error") + } + if len(directory.scopes()) != 2 { + t.Fatal("discovery outage discarded existing sources") + } + core.unavailable = false + if err := core.Get(ctx, client.ObjectKeyFromObject(first), first); err != nil { + t.Fatal(err) + } + first.Status.Conditions[0].Status = metav1.ConditionFalse + if err := core.Update(ctx, first); err != nil { + t.Fatal(err) + } + if err := d.step(ctx); err != nil { + t.Fatal(err) + } + if len(directory.scopes()) != 2 { + t.Fatal("unavailable project discarded its existing source") + } + request := kubeadmission.Request{AdmissionRequest: admissionv1.AdmissionRequest{Namespace: "internal-dns-platform", UserInfo: authenticationv1.UserInfo{Extra: map[string]authenticationv1.ExtraValue{parentNameExtra: {second.Name}}}}} + selected, err := directory.selectProject(ctx, request) + if err != nil || selected != scopes[1].Client { + t.Fatalf("dynamic admission did not map all namespaces: %v", err) + } + if err := core.Delete(ctx, second); err != nil { + t.Fatal(err) + } + if _, err := directory.selectProject(ctx, request); err == nil { + t.Fatal("deleted project passed admission") + } + recreated := readyDiscoveryProject(second.Name, "replacement-uid") + if err := core.Create(ctx, recreated); err != nil { + t.Fatal(err) + } + if err := verifyProjectIdentity(ctx, core, recreated.Name, second.UID); err == nil { + t.Fatal("recreated project used previous compilation identity") + } + if _, err := directory.selectProject(ctx, request); err == nil { + t.Fatal("recreated project used previous source") + } + if err := d.step(ctx); err != nil { + t.Fatal(err) + } + awaitDiscoveredSource(t, starts, recreated.UID) + if _, err := directory.selectProject(ctx, request); err != nil { + t.Fatal(err) + } + if err := core.Delete(ctx, recreated); err != nil { + t.Fatal(err) + } + if err := d.step(ctx); err != nil { + t.Fatal(err) + } + if len(directory.scopes()) != 1 { + t.Fatal("ACK scopes retained deleted source") + } +} + +func readyDiscoveryProject(name string, uid types.UID) *resourcemanagerv1alpha1.Project { + return &resourcemanagerv1alpha1.Project{ObjectMeta: metav1.ObjectMeta{Name: name, UID: uid}, Status: resourcemanagerv1alpha1.ProjectStatus{Conditions: []metav1.Condition{{Type: resourcemanagerv1alpha1.ProjectReady, Status: metav1.ConditionTrue, Reason: "Ready", LastTransitionTime: metav1.Now()}}}} +} +func awaitDiscoveredSource(t *testing.T, starts <-chan types.UID, expected types.UID) { + t.Helper() + select { + case actual := <-starts: + if actual != expected { + t.Fatalf("started %s want %s", actual, expected) + } + case <-time.After(time.Second): + t.Fatal("new project source did not start") + } +} + +func TestAllNamespaceRoutingPreservesListsAndStatus(t *testing.T) { + scheme, err := newScheme() + if err != nil { + t.Fatal(err) + } + zone := &dnsv1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "same-name", Namespace: "internal-dns-platform"}} + other := &dnsv1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: zone.Name, Namespace: "other"}} + source := fake.NewClientBuilder().WithScheme(scheme).WithObjects(zone, other).WithStatusSubresource(zone, other).Build() + managed := fake.NewClientBuilder().WithScheme(scheme).Build() + routed := &RoutingClient{Client: managed, Source: source, PlatformNamespace: zone.Namespace} + var zones dnsv1.DNSZoneList + if err := routed.List(context.Background(), &zones); err != nil { + t.Fatal(err) + } + if len(zones.Items) != 2 { + t.Fatalf("source namespaces were dropped: %#v", zones.Items) + } + current := &zones.Items[0] + base := current.DeepCopy() + current.Status.Conditions = []metav1.Condition{{Type: "Published", Status: metav1.ConditionTrue, Reason: "Ready", LastTransitionTime: metav1.Now()}} + if err := routed.Status().Patch(context.Background(), current, client.MergeFrom(base)); err != nil { + t.Fatal(err) + } + if err := source.Get(context.Background(), client.ObjectKeyFromObject(current), current); err != nil || len(current.Status.Conditions) != 1 { + t.Fatalf("status did not reach source: %v", err) + } +} diff --git a/internal/internaldns/runtime/regional_transport.go b/internal/internaldns/runtime/regional_transport.go new file mode 100644 index 00000000..f0abd3e0 --- /dev/null +++ b/internal/internaldns/runtime/regional_transport.go @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package runtime + +import ( + "context" + "fmt" + "sort" + + "go.miloapis.com/dns-operator/internal/internaldns/model" + "go.miloapis.com/dns-operator/internal/internaldns/platform" + "go.miloapis.com/dns-operator/internal/internaldns/transport" +) + +type regionalRoute struct { + bus *transport.JetStream + publisher transport.ExportPublisher + bootstrap platform.BootstrapWriter + store *transport.SnapshotStore + stream transport.StreamConfig +} + +type regionalTransport struct{ routes map[string]regionalRoute } + +func connectRegional(ctx context.Context, cfg Config, regions []string) (*regionalTransport, error) { + unique := map[string]bool{} + for _, region := range regions { + if region == "" { + return nil, fmt.Errorf("transport region is required") + } + unique[region] = true + } + ordered := make([]string, 0, len(unique)) + for region := range unique { + ordered = append(ordered, region) + } + sort.Strings(ordered) + routes := ®ionalTransport{routes: map[string]regionalRoute{}} + if len(cfg.RegionalTransports) == 0 { + if len(ordered) != 1 { + return nil, fmt.Errorf("one region per regional transport worker is required") + } + natsConfig := cfg.NATS + natsConfig.Name += "-" + model.SafeToken(ordered[0]) + bus, err := connectWithRetry(ctx, natsConfig) + if err != nil { + return nil, err + } + if cfg.EnsureStream { + if err := bus.EnsureStream(ctx, cfg.Stream); err != nil { + _ = bus.Close() + return nil, err + } + } + store, err := bus.SnapshotStore(ctx, cfg.Stream.SnapshotsBucket) + if err != nil { + _ = bus.Close() + return nil, err + } + for _, region := range ordered { + routes.routes[region] = regionalRoute{bus: bus, publisher: bus, bootstrap: store, store: store, stream: cfg.Stream} + } + return routes, nil + } + for _, region := range ordered { + nats, stream, ensure, err := cfg.transportForRegion(region) + if err != nil { + routes.Close() + return nil, err + } + bus, err := connectWithRetry(ctx, nats) + if err != nil { + routes.Close() + return nil, err + } + if ensure { + if err := bus.EnsureStream(ctx, stream); err != nil { + _ = bus.Close() + routes.Close() + return nil, err + } + } + store, err := bus.SnapshotStore(ctx, stream.SnapshotsBucket) + if err != nil { + _ = bus.Close() + routes.Close() + return nil, err + } + routes.routes[region] = regionalRoute{bus: bus, publisher: bus, bootstrap: store, store: store, stream: stream} + } + return routes, nil +} + +func (r *regionalTransport) route(region string) (regionalRoute, error) { + v, ok := r.routes[region] + if !ok { + return regionalRoute{}, fmt.Errorf("no connected transport for region %q", region) + } + return v, nil +} +func (r *regionalTransport) Publish(ctx context.Context, subject string, env model.Envelope) (transport.PublishAck, error) { + v, err := r.route(env.Region) + if err != nil { + return transport.PublishAck{}, err + } + return v.publisher.Publish(ctx, subject, env) +} +func (r *regionalTransport) Put(ctx context.Context, key string, env model.Envelope) (uint64, error) { + v, err := r.route(env.Region) + if err != nil { + return 0, err + } + return v.bootstrap.Put(ctx, key, env) +} +func (r *regionalTransport) Close() { + seen := map[*transport.JetStream]bool{} + for _, v := range r.routes { + if !seen[v.bus] { + _ = v.bus.Close() + seen[v.bus] = true + } + } +} diff --git a/internal/internaldns/runtime/regional_transport_test.go b/internal/internaldns/runtime/regional_transport_test.go new file mode 100644 index 00000000..7e636b17 --- /dev/null +++ b/internal/internaldns/runtime/regional_transport_test.go @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package runtime + +import ( + "context" + "sort" + "testing" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" + "go.miloapis.com/dns-operator/internal/internaldns/platform" + "go.miloapis.com/dns-operator/internal/internaldns/transport" +) + +type capturePublisher struct{ regions []string } + +func (p *capturePublisher) Publish(_ context.Context, _ string, e model.Envelope) (transport.PublishAck, error) { + p.regions = append(p.regions, e.Region) + return transport.PublishAck{}, nil +} + +func TestRegionalWorkersStartIndependently(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + started := make(chan string, 2) + terminal := make(chan error, 2) + planners := []platform.PlannerConfig{{Region: "east", Shard: "one"}, {Region: "west", Shard: "two"}} + startRegionalWorkers(planners, func(region string, _ []platform.PlannerConfig) error { + started <- region + <-ctx.Done() + return nil + }, terminal) + var regions []string + deadline := time.After(time.Second) + for len(regions) < 2 { + select { + case region := <-started: + regions = append(regions, region) + case <-deadline: + t.Fatalf("a blocked region prevented another worker from starting: %v", regions) + } + } + sort.Strings(regions) + if regions[0] != "east" || regions[1] != "west" { + t.Fatalf("unexpected regional workers: %v", regions) + } +} + +type captureBootstrap struct{ regions []string } + +func (b *captureBootstrap) Put(_ context.Context, _ string, e model.Envelope) (uint64, error) { + b.regions = append(b.regions, e.Region) + return 1, nil +} + +func TestRegionalTransportRoutesWithoutCrossRegionFallback(t *testing.T) { + eastPub, westPub := &capturePublisher{}, &capturePublisher{} + eastStore, westStore := &captureBootstrap{}, &captureBootstrap{} + r := ®ionalTransport{routes: map[string]regionalRoute{"east": {publisher: eastPub, bootstrap: eastStore}, "west": {publisher: westPub, bootstrap: westStore}}} + for _, region := range []string{"east", "west"} { + env, err := model.NewEnvelope(model.KindServingSnapshot, "event-"+region, region, "one", "resource", 1, 1, time.Now(), map[string]string{"region": region}) + if err != nil { + t.Fatal(err) + } + if _, err := r.Publish(context.Background(), "subject", env); err != nil { + t.Fatal(err) + } + if _, err := r.Put(context.Background(), "key", env); err != nil { + t.Fatal(err) + } + } + if len(eastPub.regions) != 1 || eastPub.regions[0] != "east" || len(westPub.regions) != 1 || westPub.regions[0] != "west" { + t.Fatalf("publish crossed regions: east=%v west=%v", eastPub.regions, westPub.regions) + } + if len(eastStore.regions) != 1 || len(westStore.regions) != 1 { + t.Fatalf("bootstrap crossed regions: east=%v west=%v", eastStore.regions, westStore.regions) + } + unknown, _ := model.NewEnvelope(model.KindServingSnapshot, "event-unknown", "north", "one", "resource", 1, 1, time.Now(), map[string]string{"region": "north"}) + if _, err := r.Publish(context.Background(), "subject", unknown); err == nil { + t.Fatal("unknown region silently fell back to another transport") + } +} diff --git a/internal/internaldns/runtime/routing_client.go b/internal/internaldns/runtime/routing_client.go new file mode 100644 index 00000000..e1e2c53e --- /dev/null +++ b/internal/internaldns/runtime/routing_client.go @@ -0,0 +1,85 @@ +package runtime + +import ( + "context" + + dnsv1 "go.miloapis.com/dns-operator/api/v1alpha1" + + k8sruntime "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// RoutingClient keeps project resources in their source API server while +// storing serving artifacts in the durable platform control plane. It never +// mirrors or rewrites API-assigned UIDs and generations. +type RoutingClient struct { + client.Client + Source client.Client + PlatformNamespace string +} + +func (c *RoutingClient) target(obj k8sruntime.Object) client.Client { + switch obj.(type) { + case *dnsv1.DNSZone, *dnsv1.DNSZoneList, *dnsv1.DNSZoneAssociation, *dnsv1.DNSZoneAssociationList, + *dnsv1.DNSRecordSet, *dnsv1.DNSRecordSetList, *dnsv1.DNSResolverContext, *dnsv1.DNSResolverContextList, + *dnsv1.DNSResolverAccessBinding, *dnsv1.DNSResolverAccessBindingList, + *dnsv1.DNSRegistration, *dnsv1.DNSRegistrationList, *dnsv1.DNSNamingPolicy, *dnsv1.DNSNamingPolicyList, + *dnsv1.DNSRecordContribution, *dnsv1.DNSRecordContributionList: + return c.Source + default: + return c.Client + } +} +func (c *RoutingClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + return c.target(obj).Get(ctx, key, obj, opts...) +} +func (c *RoutingClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + return c.target(list).List(ctx, list, opts...) +} +func (c *RoutingClient) Apply(ctx context.Context, obj k8sruntime.ApplyConfiguration, opts ...client.ApplyOption) error { + return c.Client.Apply(ctx, obj, opts...) +} +func (c *RoutingClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + return c.target(obj).Create(ctx, obj, opts...) +} +func (c *RoutingClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + return c.target(obj).Delete(ctx, obj, opts...) +} +func (c *RoutingClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + return c.target(obj).Update(ctx, obj, opts...) +} +func (c *RoutingClient) Patch(ctx context.Context, obj client.Object, p client.Patch, opts ...client.PatchOption) error { + return c.target(obj).Patch(ctx, obj, p, opts...) +} +func (c *RoutingClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + return c.target(obj).DeleteAllOf(ctx, obj, opts...) +} +func (c *RoutingClient) Status() client.SubResourceWriter { + return &routingSubresource{parent: c, name: "status"} +} +func (c *RoutingClient) SubResource(name string) client.SubResourceClient { + return &routingSubresource{parent: c, name: name} +} + +type routingSubresource struct { + parent *RoutingClient + name string +} + +func (s *routingSubresource) Get(ctx context.Context, obj client.Object, sub client.Object, opts ...client.SubResourceGetOption) error { + return s.parent.target(obj).SubResource(s.name).Get(ctx, obj, sub, opts...) +} +func (s *routingSubresource) Create(ctx context.Context, obj client.Object, sub client.Object, opts ...client.SubResourceCreateOption) error { + return s.parent.target(obj).SubResource(s.name).Create(ctx, obj, sub, opts...) +} +func (s *routingSubresource) Update(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error { + return s.parent.target(obj).SubResource(s.name).Update(ctx, obj, opts...) +} +func (s *routingSubresource) Patch(ctx context.Context, obj client.Object, p client.Patch, opts ...client.SubResourcePatchOption) error { + return s.parent.target(obj).SubResource(s.name).Patch(ctx, obj, p, opts...) +} +func (s *routingSubresource) Apply(ctx context.Context, obj k8sruntime.ApplyConfiguration, opts ...client.SubResourceApplyOption) error { + return s.parent.Client.SubResource(s.name).Apply(ctx, obj, opts...) +} + +var _ client.Client = (*RoutingClient)(nil) diff --git a/internal/internaldns/runtime/routing_client_test.go b/internal/internaldns/runtime/routing_client_test.go new file mode 100644 index 00000000..63defced --- /dev/null +++ b/internal/internaldns/runtime/routing_client_test.go @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package runtime + +import ( + "context" + "testing" + + dnsv1 "go.miloapis.com/dns-operator/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestRoutingClientKeepsSourceAndPlatformObjectsSeparate(t *testing.T) { + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := dnsv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + source := fake.NewClientBuilder().WithScheme(scheme).Build() + platform := fake.NewClientBuilder().WithScheme(scheme).Build() + routed := &RoutingClient{Client: platform, Source: source, PlatformNamespace: "internal-dns-system"} + ctx := context.Background() + projectObject := &dnsv1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "input", Namespace: "internal-dns-system"}} + platformObject := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "manifest", Namespace: "internal-dns-system"}} + if err := routed.Create(ctx, projectObject); err != nil { + t.Fatal(err) + } + if err := routed.Create(ctx, platformObject); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + cl client.Client + key client.ObjectKey + found bool + object client.Object + }{ + {"project in source", source, client.ObjectKeyFromObject(projectObject), true, &dnsv1.DNSZone{}}, + {"project absent from platform", platform, client.ObjectKeyFromObject(projectObject), false, &dnsv1.DNSZone{}}, + {"artifact in platform", platform, client.ObjectKeyFromObject(platformObject), true, &corev1.ConfigMap{}}, + {"artifact absent from source", source, client.ObjectKeyFromObject(platformObject), false, &corev1.ConfigMap{}}, + } { + t.Run(tc.name, func(t *testing.T) { + err := tc.cl.Get(ctx, tc.key, tc.object) + if (err == nil) != tc.found { + t.Fatalf("found=%v want=%v err=%v", err == nil, tc.found, err) + } + }) + } +} diff --git a/internal/internaldns/runtime/run.go b/internal/internaldns/runtime/run.go new file mode 100644 index 00000000..b24e687e --- /dev/null +++ b/internal/internaldns/runtime/run.go @@ -0,0 +1,616 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +// Package runtime wires the internal DNS control plane and fleet processes. +package runtime + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "net/http" + "os" + "strings" + "sync" + "time" + + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" + dnsadmission "go.miloapis.com/dns-operator/internal/internaldns/admission" + "go.miloapis.com/dns-operator/internal/internaldns/controlplane" + "go.miloapis.com/dns-operator/internal/internaldns/model" + "go.miloapis.com/dns-operator/internal/internaldns/platform" + "go.miloapis.com/dns-operator/internal/internaldns/serving" + "go.miloapis.com/dns-operator/internal/internaldns/transport" + resourcemanagerv1alpha1 "go.miloapis.com/milo/pkg/apis/resourcemanager/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/util/flowcontrol" + ctrl "sigs.k8s.io/controller-runtime" + + "sigs.k8s.io/controller-runtime/pkg/client" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" + kubeadmission "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +const parentNameExtra = "iam.miloapis.com/parent-name" + +// Run starts one of the deliberately separate production roles. Agent and +// readiness never load Kubernetes credentials. +func Run(ctx context.Context, role string, cfg Config, kubeconfig string) error { + if err := cfg.Validate(role); err != nil { + return err + } + switch role { + case roleControlPlane: + return runControlPlane(ctx, cfg, kubeconfig) + case roleAgent: + return runAgent(ctx, cfg) + case roleReadiness: + return serving.Readiness(ctx, cfg.Agent) + case rolePrepare: + return serving.PrepareDNSDist(cfg.Agent.Render) + case roleServing: + return serving.RunDaemons(ctx, cfg.Agent) + default: + return fmt.Errorf("unsupported role %q", role) + } +} + +type projectRuntime struct { + config ProjectConfig + direct client.Client + start func(context.Context) error +} + +func runControlPlane(ctx context.Context, cfg Config, kubeconfig string) error { + scheme, err := newScheme() + if err != nil { + return err + } + platformREST, err := restConfig(kubeconfig) + if err != nil { + return fmt.Errorf("load platform kubeconfig: %w", err) + } + applyKubeAPILimit(platformREST, cfg.PlatformKubeAPI) + platformClient, err := client.New(platformREST, client.Options{Scheme: scheme}) + if err != nil { + return fmt.Errorf("create uncached platform client: %w", err) + } + platformClient = &deadlineClient{Client: platformClient, Timeout: cfg.PlatformKubeAPI.requestTimeout()} + allocator := &platform.Allocator{Client: platformClient, Namespace: cfg.PlatformNamespace, ClusterPrefix: cfg.ClusterPrefix} + planners := cfg.plannerConfigs() + regions := make([]controlplane.ServingRegion, 0, len(planners)) + for _, p := range planners { + regions = append(regions, controlplane.ServingRegion{Region: p.Region, Shard: p.Shard}) + } + + identity := holderIdentity() + var discoveryReader client.Reader + makeProject := func(p ProjectConfig, sourceREST *rest.Config) (projectRuntime, error) { + projectREST := rest.CopyConfig(sourceREST) + if p.Kubeconfig != "" { + loaded, loadErr := restConfig(p.Kubeconfig) + if loadErr != nil { + return projectRuntime{}, fmt.Errorf("load project %q kubeconfig: %w", p.Name, loadErr) + } + projectREST = loaded + } + applyKubeAPILimit(projectREST, p.KubeAPI) + direct, err := client.New(projectREST, client.Options{Scheme: scheme}) + if err != nil { + return projectRuntime{}, fmt.Errorf("create project %q direct client: %w", p.Name, err) + } + direct = &deadlineClient{Client: direct, Timeout: p.KubeAPI.requestTimeout()} + project := p + start := func(startCtx context.Context) error { + mgr, err := ctrl.NewManager(projectREST, ctrl.Options{ + Scheme: scheme, + Cache: projectCache(project.Namespace), + Metrics: metricsserver.Options{BindAddress: "0"}, + HealthProbeBindAddress: "0", + LeaderElection: false, + }) + if err != nil { + return fmt.Errorf("create manager: %w", err) + } + // Manager caches deliver watch events only. Every reconciliation read goes + // to the source API so a partition cannot refresh access mappings from stale + // contexts, access bindings, associations, registrations, or producer observations. + routed := &RoutingClient{Client: platformClient, Source: direct, PlatformNamespace: cfg.PlatformNamespace} + reconciler := &controlplane.Reconciler{Client: routed, APIReader: direct, Scheme: scheme, Options: controlplane.ReconcilerOptions{ + ProjectUID: project.ProjectUID, + VerifyProjectIdentity: func(ctx context.Context) error { + if discoveryReader == nil { + return nil + } + return verifyProjectIdentity(ctx, discoveryReader, project.Name, project.ProjectUID) + }, + ProjectNamespace: project.Namespace, PlatformNamespace: cfg.PlatformNamespace, + Region: cfg.Region, Shard: cfg.Shard, Identity: identity, + LeaseDuration: cfg.ownershipLease(), + AddressAllocator: allocator, ManagedDomainSuffix: cfg.ManagedDomainSuffix, + PrivateZoneClassName: cfg.PrivateZoneClassName, ServingRegions: regions, + }} + if err := reconciler.SetupWithManager(mgr); err != nil { + return fmt.Errorf("setup reconciler: %w", err) + } + return mgr.Start(startCtx) + } + return projectRuntime{config: project, direct: direct, start: start}, nil + } + projects := make([]projectRuntime, 0, len(cfg.Projects)) + for _, p := range cfg.Projects { + project, err := makeProject(p, platformREST) + if err != nil { + return err + } + projects = append(projects, project) + } + directory := newProjectDirectory(projects) + var discovery *projectDiscovery + if cfg.ProjectDiscovery != nil { + discoveryREST, loadErr := restConfig(cfg.ProjectDiscovery.Kubeconfig) + if loadErr != nil { + return fmt.Errorf("load project discovery kubeconfig: %w", loadErr) + } + applyKubeAPILimit(discoveryREST, cfg.ProjectDiscovery.KubeAPI) + discoveryClient, clientErr := client.New(discoveryREST, client.Options{Scheme: scheme}) + if clientErr != nil { + return fmt.Errorf("create project discovery client: %w", clientErr) + } + discoveryClient = &deadlineClient{Client: discoveryClient, Timeout: cfg.ProjectDiscovery.KubeAPI.requestTimeout()} + discoveryReader = discoveryClient + discovery = &projectDiscovery{client: discoveryClient, directory: directory, rest: discoveryREST, kubeAPI: cfg.ProjectDiscovery.KubeAPI, makeProject: makeProject} + } + + var admissionManager ctrl.Manager + if cfg.Admission.Enabled { + admissionManager, err = ctrl.NewManager(platformREST, ctrl.Options{ + Scheme: scheme, + Metrics: metricsserver.Options{BindAddress: "0"}, + HealthProbeBindAddress: "0", + LeaderElection: false, + WebhookServer: webhook.NewServer(webhook.Options{Port: cfg.Admission.Port, CertDir: cfg.Admission.CertDir}), + }) + if err != nil { + return fmt.Errorf("create admission manager: %w", err) + } + handler := &dnsadmission.Handler{ + ClientForRequest: directory.selectProject, + } + admissionManager.GetWebhookServer().Register(dnsadmission.Path, &kubeadmission.Webhook{Handler: handler}) + admissionManager.GetWebhookServer().Register("/readyz", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + } + + errCh := make(chan error, len(projects)+len(planners)*3+8) + startProjectSources(ctx, projects) + if discovery != nil { + go retryLoop(ctx, time.Duration(cfg.ProjectDiscovery.IntervalSeconds)*time.Second, "project discovery", discovery.step, errCh) + } + if admissionManager != nil { + go func() { errCh <- admissionManager.Start(ctx) }() + } + + publicationMembers := regionalMembers(planners) + publicationRegions := map[string][]model.Member{} + for _, planner := range planners { + publicationRegions[planner.Region+"/"+planner.Shard] = append([]model.Member(nil), planner.Members...) + } + + sinks := map[string]*platform.AckSink{} + for _, plannerCfg := range planners { + pc := plannerCfg + planner := &platform.Planner{Client: platformClient, Allocator: allocator, Config: pc} + interval := pc.LeaseDuration / 3 + if interval < time.Second { + interval = time.Second + } + go retryLoop(ctx, interval, "planner "+pc.Region+"/"+pc.Shard, planner.Step, errCh) + sink := &platform.AckSink{Client: platformClient, Namespace: cfg.PlatformNamespace, ProjectScopes: directory.scopes, + PublicationMembers: publicationMembers, PublicationRegions: publicationRegions, Region: pc.Region, Shard: pc.Shard, Members: pc.Members} + sinks[pc.Region+"/"+pc.Shard] = sink + go retryLoop(ctx, 2*time.Second, "ack refresh "+pc.Region+"/"+pc.Shard, sink.Refresh, errCh) + } + startRegionalWorkers(planners, func(region string, regionalPlanners []platform.PlannerConfig) error { + return runControlPlaneRegion(ctx, cfg, region, regionalPlanners, platformClient, sinks) + }, errCh) + + select { + case <-ctx.Done(): + return nil + case err := <-errCh: + if err == nil || errors.Is(err, context.Canceled) { + return nil + } + return err + } +} + +func startProjectSources(ctx context.Context, projects []projectRuntime) { + for _, project := range projects { + go runProjectSource(ctx, project) + } +} + +func runProjectSource(ctx context.Context, project projectRuntime) { + name := project.config.Name + if name == "" { + name = string(project.config.ProjectUID) + } + log := ctrl.LoggerFrom(ctx).WithName("internal-dns-project-source").WithValues("project", name) + for ctx.Err() == nil { + err := project.start(ctx) + if ctx.Err() != nil { + return + } + if err == nil { + err = errors.New("source manager stopped unexpectedly") + } + log.Error(err, "source manager unavailable; rebuilding and retrying") + if !waitForRetry(ctx) { + return + } + } +} + +// startRegionalWorkers starts every transport domain independently. In +// particular, it does not wait for one region's initial NATS connection before +// starting source controllers, planners, ACK expiry, or another region. +func startRegionalWorkers(planners []platform.PlannerConfig, run func(string, []platform.PlannerConfig) error, terminal chan<- error) { + byRegion := map[string][]platform.PlannerConfig{} + for _, planner := range planners { + byRegion[planner.Region] = append(byRegion[planner.Region], planner) + } + for region, regionalPlanners := range byRegion { + go func() { terminal <- run(region, regionalPlanners) }() + } +} + +func runControlPlaneRegion(ctx context.Context, cfg Config, region string, planners []platform.PlannerConfig, platformClient client.Client, sinks map[string]*platform.AckSink) error { + log := ctrl.LoggerFrom(ctx).WithName("internal-dns-region").WithValues("region", region) + for ctx.Err() == nil { + regional, err := connectRegional(ctx, cfg, []string{region}) + if err != nil { + if ctx.Err() != nil { + return nil + } + log.Error(err, "regional transport setup failed; retrying") + if !waitForRetry(ctx) { + return nil + } + continue + } + route, err := regional.route(region) + if err != nil { + regional.Close() + return err + } + consumers := make(map[string]*transport.Consumer, len(planners)) + setupFailed := false + for _, planner := range planners { + consumer, consumerErr := route.bus.Consumer(ctx, transport.ConsumerConfig{ + Stream: route.stream.Name, + Durable: durable(roleControlPlane, planner.Region, planner.Shard, "acks"), + FilterSubjects: []string{ackFilter(planner.Region, planner.Shard)}, + AckWait: 30 * time.Second, MaxAckPending: 128, + }) + if consumerErr != nil { + log.Error(consumerErr, "regional ACK consumer setup failed; retrying", "shard", planner.Shard) + setupFailed = true + break + } + consumers[planner.Region+"/"+planner.Shard] = consumer + } + if setupFailed { + regional.Close() + if !waitForRetry(ctx) { + return nil + } + continue + } + workerCtx, cancel := context.WithCancel(ctx) + localDone := make(chan error, len(consumers)+1) + outbox := &platform.Outbox{Client: platformClient, Namespace: cfg.PlatformNamespace, Region: region, Publisher: route.publisher, Bootstrap: route.bootstrap, PendingOnly: true} + go retryLoop(workerCtx, 500*time.Millisecond, "outbox "+region, outbox.Step, localDone) + for key, consumer := range consumers { + go consumeAcks(workerCtx, consumer, sinks[key], localDone) + } + <-ctx.Done() + cancel() + regional.Close() + return nil + } + return nil +} + +func waitForRetry(ctx context.Context) bool { + timer := time.NewTimer(time.Second) + defer timer.Stop() + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true + } +} + +func runAgent(ctx context.Context, cfg Config) error { + log := ctrl.LoggerFrom(ctx).WithName("internal-dns-agent") + publisher := &agentPublisher{} + agent, err := serving.NewAgent(cfg.Agent, serving.Dependencies{AckPublisher: publisher, OnError: func(err error) { log.Error(err, "serving agent operation failed; retrying") }}) + if err != nil { + return err + } + workerCtx, cancel := context.WithCancel(ctx) + defer cancel() + done := make(chan error, 1) + go func() { done <- agent.Start(workerCtx) }() + go func() { + for workerCtx.Err() == nil { + err := runAgentTransport(workerCtx, cfg, agent, publisher) + if workerCtx.Err() != nil { + return + } + if err != nil { + log.Error(err, "DNS state delivery unavailable; retrying") + } + if !waitForRetry(workerCtx) { + return + } + } + }() + select { + case <-ctx.Done(): + return <-done + case err := <-done: + return err + } +} + +// agentPublisher does not block local activation while the transport connects. +type agentPublisher struct { + mu sync.RWMutex + delegate transport.ExportPublisher +} + +func (p *agentPublisher) set(delegate transport.ExportPublisher) { + p.mu.Lock() + defer p.mu.Unlock() + p.delegate = delegate +} +func (p *agentPublisher) Publish(ctx context.Context, subject string, env model.Envelope) (transport.PublishAck, error) { + p.mu.RLock() + delegate := p.delegate + p.mu.RUnlock() + if delegate == nil { + return transport.PublishAck{}, errors.New("DNS state transport unavailable") + } + return delegate.Publish(ctx, subject, env) +} +func runAgentTransport(ctx context.Context, cfg Config, agent *serving.Agent, publisher *agentPublisher) error { + regional, err := connectRegional(ctx, cfg, []string{cfg.Agent.Region}) + if err != nil { + return err + } + defer regional.Close() + route, _ := regional.route(cfg.Agent.Region) + consumer := func(suffix string, filters ...string) (*transport.Consumer, error) { + return route.bus.Consumer(ctx, transport.ConsumerConfig{Stream: route.stream.Name, Durable: durable(cfg.Agent.MemberID, suffix), FilterSubjects: filters, AckWait: 30 * time.Second, MaxAckPending: 64}) + } + var sources []serving.EventSource + for _, feed := range []struct{ suffix, filter string }{{"serving", model.ServingSubject(cfg.Agent.Region, cfg.Agent.Shard)}, {"records", model.RecordConsumerFilter(cfg.Agent.Region, cfg.Agent.Shard)}, {"acks", ackFilter(cfg.Agent.Region, cfg.Agent.Shard)}} { + source, err := consumer(feed.suffix, feed.filter) + if err != nil { + return err + } + sources = append(sources, source) + } + publisher.set(route.bus) + defer publisher.set(nil) + return agent.RunTransport(ctx, serving.SnapshotBootstrap{Store: route.store, Region: cfg.Agent.Region, Shard: cfg.Agent.Shard}, sources...) +} + +func newScheme() (*runtime.Scheme, error) { + scheme := runtime.NewScheme() + if err := clientgoscheme.AddToScheme(scheme); err != nil { + return nil, err + } + if err := resourcemanagerv1alpha1.AddToScheme(scheme); err != nil { + return nil, err + } + if err := corev1.AddToScheme(scheme); err != nil { + return nil, err + } + if err := dnsv1alpha1.AddToScheme(scheme); err != nil { + return nil, err + } + return scheme, nil +} + +func restConfig(path string) (*rest.Config, error) { + if path == "" { + return ctrl.GetConfig() + } + if !strings.HasPrefix(path, "/") { + return nil, fmt.Errorf("kubeconfig path must be absolute: %s", path) + } + return clientcmd.BuildConfigFromFlags("", path) +} + +// applyKubeAPILimit installs one token bucket on the REST config before that +// config is shared by a manager and its uncached client. client-go otherwise +// creates a separate default 5 QPS limiter for every constructed client. +func applyKubeAPILimit(c *rest.Config, limits KubeAPIClientConfig) { + c.QPS = limits.QPS + c.Burst = limits.Burst + c.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(limits.QPS, limits.Burst) +} + +func connectWithRetry(ctx context.Context, cfg transport.Config) (*transport.JetStream, error) { + log := ctrl.LoggerFrom(ctx).WithName("nats") + for { + js, err := transport.Connect(cfg) + if err == nil { + return js, nil + } + log.Error(err, "NATS unavailable; retrying") + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(time.Second): + } + } +} + +func retryLoop(ctx context.Context, interval time.Duration, name string, step func(context.Context) error, terminal chan<- error) { + log := ctrl.LoggerFrom(ctx).WithName("internal-dns").WithValues("loop", name) + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + if err := step(ctx); err != nil && !errors.Is(err, context.Canceled) { + log.Error(err, "operation failed; retrying") + } + select { + case <-ctx.Done(): + terminal <- nil + return + case <-ticker.C: + } + } +} + +func consumeAcks(ctx context.Context, source *transport.Consumer, sink *platform.AckSink, terminal chan<- error) { + log := ctrl.LoggerFrom(ctx).WithName("internal-dns-ack-consumer").WithValues("region", sink.Region, "shard", sink.Shard) + for { + message, err := source.Next(ctx) + if err != nil { + if ctx.Err() != nil { + terminal <- nil + return + } + log.Error(err, "read acknowledgement failed; retrying") + time.Sleep(time.Second) + continue + } + if err := sink.Apply(ctx, message.Subject(), message.Envelope()); err != nil { + log.Error(err, "apply acknowledgement failed; retrying") + _ = message.Retry(time.Second) + continue + } + if err := message.Ack(ctx); err != nil { + log.Error(err, "acknowledgement commit was ambiguous; awaiting redelivery") + } + } +} + +func projectSelector(projects []projectRuntime) func(context.Context, kubeadmission.Request) (client.Client, error) { + return func(_ context.Context, req kubeadmission.Request) (client.Client, error) { + parents := req.UserInfo.Extra[parentNameExtra] + if len(parents) == 0 && len(projects) == 1 { + p := projects[0] + if req.Namespace != p.config.Namespace { + return nil, fmt.Errorf("request namespace is outside the configured project") + } + return p.direct, nil + } + if len(parents) != 1 { + return nil, fmt.Errorf("authenticated project parent is missing or ambiguous") + } + for _, p := range projects { + if p.config.Name == parents[0] && req.Namespace == p.config.Namespace { + return p.direct, nil + } + } + return nil, fmt.Errorf("authenticated project parent is not configured for this namespace") + } +} + +func (c Config) plannerConfigs() []platform.PlannerConfig { + configs := append([]platform.PlannerConfig(nil), c.Shards...) + if len(configs) == 0 { + configs = []platform.PlannerConfig{{Region: c.Region, Shard: c.Shard}} + } + for i := range configs { + p := &configs[i] + if p.Namespace == "" { + p.Namespace = c.PlatformNamespace + } + if p.Identity == "" { + p.Identity = holderIdentity() + "-" + model.SafeToken(p.Region) + "-" + model.SafeToken(p.Shard) + } + if p.LeaseDuration <= 0 { + p.LeaseDuration = c.ownershipLease() + } + if len(p.NodeBackends) == 0 { + p.NodeBackends = append([]model.Backend(nil), c.NodeBackends...) + } + if len(p.ClusterBackends) == 0 { + p.ClusterBackends = append([]model.Backend(nil), c.ClusterBackends...) + } + + if len(p.Members) == 0 { + p.Members = append([]model.Member(nil), c.Members...) + } + } + return configs +} + +func regionalMembers(configs []platform.PlannerConfig) []model.Member { + var result []model.Member + for _, p := range configs { + for _, m := range p.Members { + if m.Role == memberRoleRegional || m.Role == memberRoleCluster { + result = append(result, m) + } + } + } + return result +} + +func ackFilter(region, shard string) string { + return "dns.private.acks." + model.SafeToken(region) + "." + model.SafeToken(shard) + ".>" +} + +func durable(parts ...string) string { + for i := range parts { + parts[i] = model.SafeToken(parts[i]) + } + return strings.Join(parts, "-") +} + +var ( + processIdentityOnce sync.Once + processIdentity string +) + +func holderIdentity() string { + processIdentityOnce.Do(func() { + processIdentity = newProcessIdentity() + }) + return processIdentity +} + +// newProcessIdentity distinguishes concurrently running control-plane +// processes even when they share a host. holderIdentity caches one value for +// the process lifetime so every project reconciler and shard planner in this +// process uses the same stable lease identity. +func newProcessIdentity() string { + host, _ := os.Hostname() + if host == "" { + host = "internal-dns" + } + random := make([]byte, 16) + if _, err := rand.Read(random); err == nil { + return host + "-" + hex.EncodeToString(random) + } + // crypto/rand failure is exceptional. PID and startup time still prevent + // same-host processes from silently sharing a lease identity. + return fmt.Sprintf("%s-%d-%d", host, os.Getpid(), time.Now().UTC().UnixNano()) +} diff --git a/internal/internaldns/serving/activation.go b/internal/internaldns/serving/activation.go index 74a2c6dd..d9f8d1f4 100644 --- a/internal/internaldns/serving/activation.go +++ b/internal/internaldns/serving/activation.go @@ -28,7 +28,7 @@ func (t *FileTransaction) beginActivation() error { return err } if t.activationLock == nil { - lock, err := os.OpenFile(t.MarkerPath+".lock", os.O_CREATE|os.O_RDWR, 0o600) + lock, err := os.OpenFile(t.MarkerPath+".lock", os.O_CREATE|os.O_RDWR, 0o640) if err != nil { return err } @@ -98,7 +98,7 @@ func (t *FileTransaction) abandonActivation() error { // installation cannot race with a process loading its configuration files. func daemonPermission(stateDir string) (start, retain bool, release func()) { marker := activationMarkerPath(stateDir) - lock, err := os.OpenFile(marker+".lock", os.O_CREATE|os.O_RDWR, 0o600) + lock, err := os.Open(marker + ".lock") if err != nil { return false, false, func() {} } diff --git a/internal/internaldns/serving/activation_test.go b/internal/internaldns/serving/activation_test.go index 6ee5421f..b0384830 100644 --- a/internal/internaldns/serving/activation_test.go +++ b/internal/internaldns/serving/activation_test.go @@ -132,3 +132,39 @@ func TestRestoreActiveUsesInstalledConfigurationHash(t *testing.T) { t.Fatal("approved installed hash was not checkpointed") } } + +func TestServingStateUsesReadOnlyGroupAccess(t *testing.T) { + dir := t.TempDir() + checkpoint := dir + "/checkpoint.json" + if err := os.WriteFile(checkpoint, []byte("{}"), 0600); err != nil { + t.Fatal(err) + } + if err := prepareServingState(dir); err != nil { + t.Fatal(err) + } + for _, path := range []string{checkpoint, activationMarkerPath(dir) + ".lock"} { + info, err := os.Stat(path) + if err != nil || info.Mode().Perm() != 0640 { + t.Fatalf("serving group permissions on %s: %v %v", path, info, err) + } + } + start, retain, release := daemonPermission(dir) + release() + if !start || !retain { + t.Fatal("read-only descriptor cannot check accepted activation") + } +} + +func TestServingCheckpointReplacesInterruptedWritePermissions(t *testing.T) { + path := t.TempDir() + "/checkpoint.json.tmp" + if err := os.WriteFile(path, []byte("partial"), 0600); err != nil { + t.Fatal(err) + } + if err := writeDurable(path, []byte("complete"), 0640); err != nil { + t.Fatal(err) + } + info, err := os.Stat(path) + if err != nil || info.Mode().Perm() != 0640 { + t.Fatalf("retained temporary file prevents serving group access: %v %v", info, err) + } +} diff --git a/internal/internaldns/serving/agent.go b/internal/internaldns/serving/agent.go index 24f8b7bc..f257dab8 100644 --- a/internal/internaldns/serving/agent.go +++ b/internal/internaldns/serving/agent.go @@ -229,6 +229,9 @@ func NewAgent(c Config, d Dependencies) (*Agent, error) { } } store := stateStore{path: filepath.Join(c.StateDir, "checkpoint.json")} + if err := prepareServingState(c.StateDir); err != nil { + return nil, err + } state, err := store.Load() if err != nil { gateErr := (FileTransaction{Config: c.Transaction, Runner: d.Runner}).FailClosed(context.Background()) diff --git a/internal/internaldns/serving/coredns_test.go b/internal/internaldns/serving/coredns_test.go index 52649fff..a7e5bd7e 100644 --- a/internal/internaldns/serving/coredns_test.go +++ b/internal/internaldns/serving/coredns_test.go @@ -307,7 +307,7 @@ func TestRegionalPoolRequiresOneMemberWithEveryCurrentZone(t *testing.T) { t.Fatal(err) } conf := string(rendered.Files[c.ClusterDNSDist.Path]) - if strings.Count(conf, "server:addPool(") != 1 { + if strings.Count(conf, "]:addPool(") != 1 { t.Fatalf("stale live member remains in context pool:\n%s", conf) } // A newer ACK without the matching publication plan gates even the formerly diff --git a/internal/internaldns/serving/daemons.go b/internal/internaldns/serving/daemons.go index a11db457..a20830ac 100644 --- a/internal/internaldns/serving/daemons.go +++ b/internal/internaldns/serving/daemons.go @@ -63,20 +63,14 @@ func RunDaemons(ctx context.Context, config Config) error { addBind := func(bind CoreDNSRenderConfig) { processes = append(processes, &daemonProcess{name: "coredns", config: bind.Path, args: []string{"-config", bind.Path}}) } - addRouter := func(router DNSDistRenderConfig) { - processes = append(processes, &daemonProcess{name: "dnsdist", config: router.Path, args: []string{"--supervised", "-C", router.Path}}) - } + switch config.Render.Role { case RenderNode: addBind(config.Render.NodeCoreDNS) - addRouter(config.Render.NodeDNSDist) case RenderRegionalCoreDNS: addBind(config.Render.ClusterCoreDNS) - case RenderRegionalDNSDist: - addRouter(config.Render.ClusterDNSDist) case RenderCluster: addBind(config.Render.ClusterCoreDNS) - addRouter(config.Render.ClusterDNSDist) default: return fmt.Errorf("unsupported serving process role %q", config.Render.Role) } diff --git a/internal/internaldns/serving/dnsdist.go b/internal/internaldns/serving/dnsdist.go new file mode 100644 index 00000000..071ff5f9 --- /dev/null +++ b/internal/internaldns/serving/dnsdist.go @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package serving + +import ( + "crypto/rand" + "encoding/base64" + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +const DNSDistKeyPath = "/run/internal-dns/dnsdist.key" +const DNSDistImage = "powerdns/dnsdist-20:2.0.10@sha256:24ac720af997aff6b98cb5b50d748754f815428e07b5368dfb0eccd5aa8d07e8" + +func dnsdistConsole(path string) string { + if strings.Contains(filepath.Base(path), "node") { + return "127.0.0.1:5199" + } + return "127.0.0.1:5299" +} + +func PrepareDNSDist(config RenderConfig) error { + if err := validateRenderConfig(config); err != nil { + return err + } + var routers []DNSDistRenderConfig + switch config.Role { + case RenderNode: + routers = append(routers, config.NodeDNSDist) + case RenderRegionalDNSDist: + routers = append(routers, config.ClusterDNSDist) + case RenderRegionalCoreDNS: + return nil + default: + return fmt.Errorf("unsupported Kubernetes fleet role %q", config.Role) + } + if err := prepareDNSDistKey(DNSDistKeyPath); err != nil { + return err + } + for _, router := range routers { + if err := prepareDNSDistRouter(router); err != nil { + return err + } + } + return nil +} + +func prepareDNSDistRouter(router DNSDistRenderConfig) error { + if len(router.Backends) == 0 { + return errors.New("dnsdist startup requires fixed shared backends") + } + if err := os.MkdirAll(filepath.Dir(router.Path), 0750); err != nil { + return err + } + if _, err := os.Stat(router.Path); err == nil { + return nil + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + temporary := router.Path + ".bootstrap" + if err := writeDurable(temporary, renderDNSDist(router, router.Backends, nil, nil, false, nil, nil), 0644); err != nil { + return err + } + if err := os.Rename(temporary, router.Path); err != nil { + return err + } + if err := syncDirs([]string{router.Path}); err != nil { + return err + } + return nil +} + +func prepareDNSDistKey(path string) error { + if err := os.MkdirAll(filepath.Dir(path), 0750); err != nil { + return err + } + data, err := os.ReadFile(path) + if err == nil { + key, decodeErr := base64.StdEncoding.DecodeString(strings.TrimSpace(string(data))) + if decodeErr != nil || len(key) != 32 { + return errors.New("invalid dnsdist control key") + } + return os.Chmod(path, 0640) + } + if !errors.Is(err, os.ErrNotExist) { + return err + } + key := make([]byte, 32) + if _, err := rand.Read(key); err != nil { + return err + } + temporary := path + ".tmp" + if err := writeDurable(temporary, []byte(base64.StdEncoding.EncodeToString(key)+"\n"), 0640); err != nil { + return err + } + return os.Rename(temporary, path) +} diff --git a/internal/internaldns/serving/dnsdist_test.go b/internal/internaldns/serving/dnsdist_test.go new file mode 100644 index 00000000..485311ba --- /dev/null +++ b/internal/internaldns/serving/dnsdist_test.go @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package serving + +import ( + "encoding/base64" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestDNSDistKeyPersistsAcrossAgentRestarts(t *testing.T) { + path := filepath.Join(t.TempDir(), "runtime", "dnsdist.key") + if err := prepareDNSDistKey(path); err != nil { + t.Fatal(err) + } + first, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if err := prepareDNSDistKey(path); err != nil { + t.Fatal(err) + } + second, err := os.ReadFile(path) + if err != nil || string(first) != string(second) { + t.Fatal("control identity changed", err) + } + decoded, err := base64.StdEncoding.DecodeString(strings.TrimSpace(string(first))) + if err != nil || len(decoded) != 32 { + t.Fatal("invalid control identity", err) + } + info, err := os.Stat(path) + if err != nil || info.Mode().Perm() != 0640 { + t.Fatal("control key must be readable only by its owner and serving group", err) + } + if err := os.WriteFile(path, []byte("invalid"), 0600); err != nil { + t.Fatal(err) + } + if err := prepareDNSDistKey(path); err == nil { + t.Fatal("invalid retained key was accepted") + } +} + +func TestDNSDistBootstrapPublishesOnlyCompleteConfiguration(t *testing.T) { + config, _ := renderFixture(t) + router := config.NodeDNSDist + router.Path = filepath.Join(t.TempDir(), "node-dnsdist.conf") + if err := os.Mkdir(router.Path+".bootstrap", 0700); err != nil { + t.Fatal(err) + } + if err := prepareDNSDistRouter(router); err == nil { + t.Fatal("failed staging was accepted") + } + if _, err := os.Stat(router.Path); !os.IsNotExist(err) { + t.Fatal("failed staging exposed a startup file", err) + } + if err := os.Remove(router.Path + ".bootstrap"); err != nil { + t.Fatal(err) + } + if err := prepareDNSDistRouter(router); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(router.Path) + if err != nil || !strings.Contains(string(data), "DNSAction.Refused") || !strings.Contains(string(data), "setRules(rules)") { + t.Fatal("bootstrap must refuse unknown contexts", err) + } + if err := os.WriteFile(router.Path, []byte("accepted snapshot"), 0644); err != nil { + t.Fatal(err) + } + if err := prepareDNSDistRouter(router); err != nil { + t.Fatal(err) + } + retained, err := os.ReadFile(router.Path) + if err != nil || string(retained) != "accepted snapshot" { + t.Fatal("bootstrap replaced accepted routing", err) + } +} diff --git a/internal/internaldns/serving/render.go b/internal/internaldns/serving/render.go index 28115c47..6756d9a2 100644 --- a/internal/internaldns/serving/render.go +++ b/internal/internaldns/serving/render.go @@ -10,6 +10,7 @@ import ( "path/filepath" "sort" "strconv" + "strings" contextdns "go.miloapis.com/dns-operator/internal/internaldns/coredns" "go.miloapis.com/dns-operator/internal/internaldns/model" @@ -200,28 +201,54 @@ func collectBackends(bindings []model.Binding, get func(model.Binding) []model.B func renderDNSDist(c DNSDistRenderConfig, backends []model.Backend, bindings []model.Binding, active map[string]bool, cluster bool, eligible map[string]map[string]bool, zones map[string]map[string]map[string]bool) []byte { var out bytes.Buffer - fmt.Fprintf(&out, "setLocal(%q)\n", c.ListenAddress) - fmt.Fprint(&out, "setACL({") + endpoint := dnsdistConsole(c.Path) + fmt.Fprint(&out, "local client = inClientStartup()\nif client or not internalDNSInitialized then\n") + fmt.Fprintf(&out, " local key\n if os.getenv(\"INTERNAL_DNS_VALIDATE\") == \"1\" then key = string.rep(\"A\",43)..\"=\" else\n local handle = assert(io.open(%q,\"r\"))\n key = handle:read(\"*l\"); handle:close()\n end\n setKey(key)\n controlSocket(%q)\n setConsoleACL({\"127.0.0.1/32\"})\nend\nif not client then\n", DNSDistKeyPath, endpoint) + topology, _ := json.Marshal(struct { + Listen string + ACLs []string + Backends []model.Backend + }{c.ListenAddress, c.ACLs, backends}) + fingerprint := model.OpaqueToken(string(topology)) + fmt.Fprint(&out, "if not internalDNSInitialized then\n") + fmt.Fprintf(&out, "setLocal(%q)\nsetACL({", c.ListenAddress) for i, acl := range c.ACLs { if i > 0 { fmt.Fprint(&out, ",") } fmt.Fprintf(&out, "%q", acl) } - fmt.Fprint(&out, "})\nsetSecurityPollSuffix(\"\")\n\n") - for _, b := range backends { - fmt.Fprintf(&out, "local server = newServer({address=%q, name=%q, pool=%q, useProxyProtocol=true, tcpOnly=true, checkTCP=true, checkName=\".\", checkType=\"SOA\", mustResolve=false})\n", net.JoinHostPort(b.Address, strconv.Itoa(int(b.Port))), "b_"+model.OpaqueToken(b.MemberID), c.PoolName) - if cluster { - for _, binding := range bindings { - for _, zone := range binding.Zones { - if zones != nil && active[binding.BindingUID] && zones[binding.BindingUID][zone.ZoneUID][b.MemberID] { - fmt.Fprintf(&out, "server:addPool(%q)\n", binding.ViewName()+"_"+model.OpaqueToken(zone.ZoneUID)) - } + fmt.Fprint(&out, "})\nsetSecurityPollSuffix(\"\")\ninternalDNSServers = {}\ninternalDNSPools = {}\n") + for _, backend := range backends { + fmt.Fprintf(&out, "internalDNSServers[%q] = newServer({address=%q,name=%q,pool=%q,useProxyProtocol=true,tcpOnly=true,checkTCP=true,checkName=\".\",checkType=\"SOA\",mustResolve=false})\n", backend.MemberID, net.JoinHostPort(backend.Address, strconv.Itoa(int(backend.Port))), "b_"+model.OpaqueToken(backend.MemberID), c.PoolName) + } + fmt.Fprintf(&out, "internalDNSTopology = %q\ninternalDNSInitialized = true\nend\nassert(internalDNSTopology == %q,\"dnsdist topology requires a replacement member\")\n", fingerprint, fingerprint) + fmt.Fprint(&out, "local rules = {}\nlocal function appendRule(rule,action) rules[#rules+1] = newRuleAction(rule,action) end\n") + if cluster { + pools := map[string]map[string]bool{} + for _, binding := range bindings { + if eligible != nil { + pools[eligiblePool(backends, eligible[binding.BindingUID])] = eligible[binding.BindingUID] + } + for _, zone := range binding.Zones { + if zones != nil { + pools[eligiblePool(backends, zones[binding.BindingUID][zone.ZoneUID])] = zones[binding.BindingUID][zone.ZoneUID] } - if !binding.ConfigurationPending && active[binding.BindingUID] && (eligible == nil || eligible[binding.BindingUID][b.MemberID]) { - fmt.Fprintf(&out, "server:addPool(%q)\n", binding.ViewName()) + } + } + names := make([]string, 0, len(pools)) + for name := range pools { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + fmt.Fprintf(&out, "if not internalDNSPools[%q] then\ngetPool(%q)\n", name, name) + for _, backend := range backends { + if pools[name][backend.MemberID] { + fmt.Fprintf(&out, "internalDNSServers[%q]:addPool(%q)\n", backend.MemberID, name) } } + fmt.Fprintf(&out, "internalDNSPools[%q] = true\nend\n", name) } } fmt.Fprint(&out, "\nlocal bindings = {\n") @@ -232,33 +259,49 @@ func renderDNSDist(c DNSDistRenderConfig, backends []model.Backend, bindings []m } fmt.Fprintf(&out, " [%q] = {context=%q, ready=%t},\n", addr, b.ViewName(), !b.ConfigurationPending && active[b.BindingUID]) } - fmt.Fprint(&out, "}\n\nfunction classifyContext(dq)\n local binding = bindings[dq.localaddr:toString()]\n if binding == nil then\n return DNSAction.Refused, \"\"\n end\n dq:setTag(\"dns-context\", binding.context)\n if not binding.ready then dq:setTag(\"context-unready\", \"true\") end\n return DNSAction.None, \"\"\nend\n\n") - fmt.Fprint(&out, "addAction(AllRule(), LuaAction(classifyContext))\n") - fmt.Fprint(&out, "addAction(TagRule(\"context-unready\", \"true\"), RCodeAction(DNSRCode.SERVFAIL))\n") + fmt.Fprint(&out, "}\n\nlocal function classifyContext(dq)\n local binding = bindings[dq.localaddr:toString()]\n if binding == nil then\n return DNSAction.Refused, \"\"\n end\n dq:setTag(\"dns-context\", binding.context)\n if not binding.ready then dq:setTag(\"context-unready\", \"true\") end\n return DNSAction.None, \"\"\nend\n\n") + fmt.Fprint(&out, "appendRule(AllRule(), LuaAction(classifyContext))\n") + fmt.Fprint(&out, "appendRule(TagRule(\"context-unready\", \"true\"), RCodeAction(DNSRCode.SERVFAIL))\n") for _, binding := range bindings { attached := append([]model.ZoneAttachment(nil), binding.Zones...) sort.Slice(attached, func(i, j int) bool { return len(attached[i].Apex) > len(attached[j].Apex) }) for _, zone := range attached { rule := fmt.Sprintf("AndRule({TagRule(\"dns-context\",%q),QNameSuffixRule(%q)})", binding.ViewName(), model.AbsoluteName(zone.Apex)) if zone.Pending || zones != nil && len(zones[binding.BindingUID][zone.ZoneUID]) == 0 { - fmt.Fprintf(&out, "addAction(%s,RCodeAction(DNSRCode.SERVFAIL))\n", rule) + fmt.Fprintf(&out, "appendRule(%s,RCodeAction(DNSRCode.SERVFAIL))\n", rule) } else if cluster && zones != nil { - pool := binding.ViewName() + "_" + model.OpaqueToken(zone.ZoneUID) - fmt.Fprintf(&out, "getPool(%q)\naddAction(AndRule({%s,NotRule(PoolAvailableRule(%q))}),RCodeAction(DNSRCode.SERVFAIL))\naddAction(%s,PoolAction(%q))\n", pool, rule, pool, rule, pool) + pool := eligiblePool(backends, zones[binding.BindingUID][zone.ZoneUID]) + fmt.Fprintf(&out, "getPool(%q)\nappendRule(AndRule({%s,NotRule(PoolAvailableRule(%q))}),RCodeAction(DNSRCode.SERVFAIL))\nappendRule(%s,PoolAction(%q))\n", pool, rule, pool, rule, pool) } } } if cluster { for _, binding := range bindings { - fmt.Fprintf(&out, "getPool(%q)\naddAction(AndRule({TagRule(\"dns-context\",%q),NotRule(PoolAvailableRule(%q))}), RCodeAction(DNSRCode.SERVFAIL))\naddAction(TagRule(\"dns-context\",%q),PoolAction(%q))\n", binding.ViewName(), binding.ViewName(), binding.ViewName(), binding.ViewName(), binding.ViewName()) + pool := eligiblePool(backends, eligible[binding.BindingUID]) + if eligible == nil { + pool = c.PoolName + } + fmt.Fprintf(&out, "getPool(%q)\nappendRule(AndRule({TagRule(\"dns-context\",%q),NotRule(PoolAvailableRule(%q))}), RCodeAction(DNSRCode.SERVFAIL))\nappendRule(TagRule(\"dns-context\",%q),PoolAction(%q))\n", pool, binding.ViewName(), pool, binding.ViewName(), pool) } - fmt.Fprint(&out, "addAction(AllRule(),RCodeAction(DNSRCode.SERVFAIL))\n") + fmt.Fprint(&out, "appendRule(AllRule(),RCodeAction(DNSRCode.SERVFAIL))\n") } else { - fmt.Fprintf(&out, "addAction(NotRule(PoolAvailableRule(%q)), RCodeAction(DNSRCode.SERVFAIL))\naddAction(AllRule(),PoolAction(%q))\n", c.PoolName, c.PoolName) + fmt.Fprintf(&out, "appendRule(NotRule(PoolAvailableRule(%q)), RCodeAction(DNSRCode.SERVFAIL))\nappendRule(AllRule(),PoolAction(%q))\n", c.PoolName, c.PoolName) } + fmt.Fprint(&out, "setRules(rules)\nend\n") return out.Bytes() } +func eligiblePool(backends []model.Backend, allowed map[string]bool) string { + members := make([]string, 0, len(allowed)) + for _, backend := range backends { + if allowed[backend.MemberID] { + members = append(members, backend.MemberID) + } + } + sort.Strings(members) + return "ready_" + model.OpaqueToken(strings.Join(members, "\x00")) +} + func renderNodeCoreDNS(c CoreDNSRenderConfig, bindings []model.Binding) []byte { cfg := coreDNSConfig(c) for _, b := range bindings { diff --git a/internal/internaldns/serving/state.go b/internal/internaldns/serving/state.go index ebc122bc..f5036ae8 100644 --- a/internal/internaldns/serving/state.go +++ b/internal/internaldns/serving/state.go @@ -126,7 +126,7 @@ func (s stateStore) Save(state checkpoint) error { return err } temp := s.path + ".tmp" - if err := writeDurable(temp, b, 0o600); err != nil { + if err := writeDurable(temp, b, 0o640); err != nil { return err } if err := os.Rename(temp, s.path); err != nil { @@ -141,3 +141,24 @@ func (s stateStore) Save(state checkpoint) error { } func chunkKey(index int) string { return fmt.Sprint(index) } + +func prepareServingState(dir string) error { + if err := os.MkdirAll(dir, 0750); err != nil { + return err + } + lock, err := os.OpenFile(activationMarkerPath(dir)+".lock", os.O_CREATE|os.O_RDWR, 0640) + if err != nil { + return err + } + if err := lock.Chmod(0640); err != nil { + _ = lock.Close() + return err + } + if err := lock.Close(); err != nil { + return err + } + if err := os.Chmod(filepath.Join(dir, "checkpoint.json"), 0640); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return nil +} diff --git a/internal/internaldns/serving/transaction.go b/internal/internaldns/serving/transaction.go index bd8f756a..cb1e69c8 100644 --- a/internal/internaldns/serving/transaction.go +++ b/internal/internaldns/serving/transaction.go @@ -287,6 +287,10 @@ func writeDurable(path string, data []byte, mode os.FileMode) error { if err != nil { return err } + if err := f.Chmod(mode); err != nil { + _ = f.Close() + return err + } if _, err = f.Write(data); err != nil { _ = f.Close() return err @@ -344,5 +348,5 @@ func restoreFiles(paths []string, old map[string]oldFile) error { } func routingConfigurationChanged(path string, before, after []byte) bool { - return strings.HasSuffix(path, ".conf") && !bytes.Equal(before, after) + return strings.HasSuffix(path, ".conf") && !strings.HasSuffix(path, "-dnsdist.conf") && !bytes.Equal(before, after) }