From 89c5008eb577e9c00c815cd056bf969e6fab4e78 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 12:09:22 -0500 Subject: [PATCH 01/23] feat: Package the internal DNS control plane and serving agents --- cmd/internal-dns/main.go | 41 ++ config/internal-dns/Dockerfile | 24 + config/internal-dns/README.md | 140 +++++ config/internal-dns/admission.example.yaml | 67 +++ .../internal-dns/control-plane.example.yaml | 120 ++++ config/internal-dns/fleet.example.yaml | 123 ++++ config/internal-dns/kustomization.yaml | 8 + config/internal-dns/namespace.yaml | 6 + config/internal-dns/nats-server.conf.example | 149 +++++ config/internal-dns/rbac.yaml | 128 ++++ internal/internaldns/runtime/config.go | 350 +++++++++++ internal/internaldns/runtime/config_test.go | 154 +++++ .../internaldns/runtime/deadline_client.go | 134 +++++ .../runtime/deadline_client_test.go | 31 + .../runtime/deployment_examples_test.go | 89 +++ .../runtime/deployment_permissions_test.go | 275 +++++++++ internal/internaldns/runtime/identity_test.go | 42 ++ .../internaldns/runtime/regional_transport.go | 123 ++++ .../runtime/regional_transport_test.go | 83 +++ .../internaldns/runtime/routing_client.go | 85 +++ .../runtime/routing_client_test.go | 51 ++ internal/internaldns/runtime/run.go | 546 ++++++++++++++++++ 22 files changed, 2769 insertions(+) create mode 100644 cmd/internal-dns/main.go create mode 100644 config/internal-dns/Dockerfile create mode 100644 config/internal-dns/README.md create mode 100644 config/internal-dns/admission.example.yaml create mode 100644 config/internal-dns/control-plane.example.yaml create mode 100644 config/internal-dns/fleet.example.yaml create mode 100644 config/internal-dns/kustomization.yaml create mode 100644 config/internal-dns/namespace.yaml create mode 100644 config/internal-dns/nats-server.conf.example create mode 100644 config/internal-dns/rbac.yaml create mode 100644 internal/internaldns/runtime/config.go create mode 100644 internal/internaldns/runtime/config_test.go create mode 100644 internal/internaldns/runtime/deadline_client.go create mode 100644 internal/internaldns/runtime/deadline_client_test.go create mode 100644 internal/internaldns/runtime/deployment_examples_test.go create mode 100644 internal/internaldns/runtime/deployment_permissions_test.go create mode 100644 internal/internaldns/runtime/identity_test.go create mode 100644 internal/internaldns/runtime/regional_transport.go create mode 100644 internal/internaldns/runtime/regional_transport_test.go create mode 100644 internal/internaldns/runtime/routing_client.go create mode 100644 internal/internaldns/runtime/routing_client_test.go create mode 100644 internal/internaldns/runtime/run.go diff --git a/cmd/internal-dns/main.go b/cmd/internal-dns/main.go new file mode 100644 index 00000000..3c8d7c74 --- /dev/null +++ b/cmd/internal-dns/main.go @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package main + +import ( + "flag" + "fmt" + "os" + + internruntime "go.miloapis.com/dns-operator/internal/internaldns/runtime" + _ "k8s.io/client-go/plugin/pkg/client/auth" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +func main() { + var role, configPath, kubeconfig string + flag.StringVar(&role, "role", "", "Process role: control-plane, agent, or watchdog") + flag.StringVar(&configPath, "config", "", "Absolute path to the internal DNS JSON configuration") + // controller-runtime/client-go may install this standard flag from an init + // hook. Reuse it when present so the command keeps one stable contract. + if flag.Lookup("kubeconfig") == nil { + flag.StringVar(&kubeconfig, "kubeconfig", "", "Control-plane Kubernetes configuration; ignored by agent and watchdog") + } + logOptions := zap.Options{Development: false} + logOptions.BindFlags(flag.CommandLine) + flag.Parse() + if existing := flag.Lookup("kubeconfig"); existing != nil { + kubeconfig = existing.Value.String() + } + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&logOptions))) + + cfg, err := internruntime.LoadConfig(configPath) + if err == nil { + err = internruntime.Run(ctrl.SetupSignalHandler(), role, cfg, kubeconfig) + } + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "internal-dns: %v\n", err) + os.Exit(1) + } +} diff --git a/config/internal-dns/Dockerfile b/config/internal-dns/Dockerfile new file mode 100644 index 00000000..01cca4e7 --- /dev/null +++ b/config/internal-dns/Dockerfile @@ -0,0 +1,24 @@ +# syntax=docker/dockerfile:1.7 + +FROM --platform=$BUILDPLATFORM golang:1.26 AS build +ARG TARGETOS=linux +ARG TARGETARCH +WORKDIR /src +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/pkg/mod go mod download +COPY cmd/internal-dns ./cmd/internal-dns +COPY api ./api +COPY internal ./internal +RUN --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH \ + go build -trimpath -tags netgo,osusergo -ldflags='-s -w' \ + -o /out/internal-dns ./cmd/internal-dns + +# This is the control-plane image and the binary artifact used by serving +# images. Node, regional-frontend, and regional-BIND images must also contain +# the validation, reload, cache-flush, and fail-closed commands declared in +# their serving configuration. +FROM scratch +COPY --from=build /out/internal-dns /internal-dns +USER 65532:65532 +ENTRYPOINT ["/internal-dns"] diff --git a/config/internal-dns/README.md b/config/internal-dns/README.md new file mode 100644 index 00000000..1be96293 --- /dev/null +++ b/config/internal-dns/README.md @@ -0,0 +1,140 @@ +# Internal DNS regional fleet deployment + +These manifests show the process boundaries for one shared regional shard. +Replace the example images, addresses, certificates, storage classes, admission +CA injection, and site command wrappers before deployment. + +The fixed fleet has three roles: + +- A `node` resolver agent owns the shared node dnsdist/BIND pair. +- A `regional-dnsdist` resolver agent owns the shared regional frontend and its + pool of regional BIND members. +- Two or more `regional-bind` combined agents independently render the complete + private primary-zone set for their member, validate it with + `named-checkconf -z`, activate it in isolated BIND context views, prove the + exact version over UDP and TCP, and publish a leased ACK. + +Contexts, access bindings, and zones add destinations, views, and zone files to +these processes. They never create workloads per VPC or zone. The legacy +`cluster` and `both` render roles remain development fallbacks and should not be +used for the multi-member production layout. + +Each member needs a distinct member ID, NATS identity and durable consumers, +checkpoint PVC, configuration PVC, and independent watchdog lease. Treat that +set as a single-writer unit. A rolling update must allocate a new member +identity and storage; do not run two pods against one checkpoint. + +## Control-plane and API boundaries + +Control-plane replicas can concurrently watch the platform API and source +projects. Publication and shard ownership use resource-version checks and +increasing writer epochs on takeover. Each source project has an authenticated +client and optional request limits. An unavailable source must not prevent ACK +projection or publication for healthy sources. + +DNS always consumes `DNSResolverContext` and `DNSResolverAccessBinding` from +the project API. It does not watch a networking API. A trusted integration +principal in `admission.integrationSubjects` creates a context and writes +short-lived regional access intent. Product publishers cannot authorize access. +The context UID, access UID, authorization writer epoch, sequence, and original +deadline are carried into the internal `DNSResolverBinding`. + +The control plane allocates a managed namespace when the context requests one. +Compute discovers the context status and publishes through +`DNSRegistration` and `DNSRecordContribution` under a separately issued +`DNSContributionGrant`; it does +not choose a zone for the managed-name path and receives no NATS credentials. + +## Serving identity and BIND views + +Consumer and regional listener prefixes must be local routes in their dnsdist +network namespaces. Galactic authorizes and translates the consumer path to a +service-side destination. dnsdist preserves that destination in a trusted +PROXYv2 header. Node BIND selects a context view and forwards to its regional +destination. Regional dnsdist selects the same context and load-balances the +healthy BIND members. Every regional BIND member directly hosts the context's +private primary zones. + +An owned name without eligible answers carries service-only `TYPE65280` +metadata at that exact name. This private-use record preserves NODATA without +adding a child name or conflicting with a recovered CNAME. Tenant declarations +cannot use that type. Dynamic registrations require TTLs from 1 to 30 seconds; +client caches can retain withdrawn answers for at most that TTL. Static records +retain their explicit RRset TTL, or the default of 300 seconds. Entries in one +static RRset must use the same TTL. + +A context without private zones proves recursion with the root SOA over UDP and +TCP. `verifier.publicName` can override this probe. BIND members need a writable +`/tmp` runtime volume even when the image root filesystem is read-only. + +Private packet caching stays disabled in dnsdist. Each BIND view has its own +positive and negative cache. `max-ncache-ttl` defaults to five seconds and is +bounded at 60 seconds. Public recursion retains DNSSEC validation; each view +adds validation exceptions only for its private apexes. Unknown or expired +destinations are rejected before cache lookup. + +Regional publication proof connects directly to that member's BIND listener, +supplies the exact context destination in a protected PROXYv2 header over both +UDP and TCP, and verifies the installed SOA serial and fingerprint. Restrict +`proxyPeers` to regional dnsdist and the member's local publication agent +network. There are no PowerDNS views, network variants, or per-context source +markers. + +## Transactions, health, and expiry + +The agent writes a complete candidate set into its stage directory. Regional +zone files are siblings of `cluster-bind.conf`; validation rewrites file +references to the staged paths and runs `named-checkconf -z`. Installation is +atomic only after every candidate validates. Reload wrappers must wait for the +listener and backend pool before returning so the immediate DNS proof observes +the activated state. + +Publication ACKs are leases. A regional frontend and node may keep a context +available while at least one assigned regional member has a fresh current ACK; +the project-facing Ready condition can still report degraded membership. If no +member remains, the frontend gates the context. A regional member that cannot +apply or prove a publication fails closed and stops renewing its watchdog. + +Agents preserve original record and access deadlines in their checkpoints. +They remove expired records, rewrite regional zone files, and flush affected +BIND names locally even while NATS or a source API is unavailable. Replay and +restart cannot extend a deadline or regress writer/configuration fences. + +The watchdog lease is at most five seconds. It is renewed only after local +expiry enforcement, a complete validated transaction, and required DNS proofs. +The gate must stop or firewall the local private listener without depending on +Kubernetes or NATS. Validation, reload, readiness wait, and proof must fit +inside the watchdog budget; reduce shard size or improve local operations when +they do not. + +Mount the transaction stage, active BIND configuration, zone files, checkpoint, +and watchdog lease on low-latency storage local to the serving host. Do not put +the hot transaction or lease path on a desktop file-sharing mount or a remote +filesystem whose fsync latency can consume the watchdog budget. Replication is +provided by the durable publication stream and independent regional members; +each member still has its own crash-safe checkpoint volume. + +The regional BIND reload helper must reload existing primary zones, for +example with `rndc reload`. `rndc reconfig` only guarantees configuration and +new-zone loading, so it is insufficient when an existing private apex moves to +a newly rendered immutable zone file. The node-tier helper may use the +deployment's normal configuration reload because that BIND tier is recursive. + +## Capacity and transport + +Set `maxBindings` from measured BIND view memory, validation/reload time, +dnsdist rule cost, NATS payload size, and watchdog timing. The example caps a +shard at 200 contexts and budgets 2 MiB per view. Add another fixed shard before +raising those bounds. + +NATS uses TLS with a verified server name, client certificate, and one scoped +identity per process. Control-plane credentials publish serving/publication +subjects and consume ACKs. Each serving member can consume its assigned durable +subjects and publish only its own hashed ACK subject. Broker partitions do not +extend record or authorization deadlines. + +`fleet.example.yaml` provides node, regional frontend, and regional BIND JSON +configurations plus one Deployment template. Instantiate the template for each +member and create a second regional BIND member with distinct addresses and +storage. `control-plane.example.yaml` lists both regional members in +`clusterBackends` and `members`. diff --git a/config/internal-dns/admission.example.yaml b/config/internal-dns/admission.example.yaml new file mode 100644 index 00000000..be700bbe --- /dev/null +++ b/config/internal-dns/admission.example.yaml @@ -0,0 +1,67 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: internal-dns-authorization + annotations: + cert-manager.io/inject-ca-from: internal-dns-system/internal-dns-admission-tls +webhooks: +- name: authorization.internal-dns.miloapis.com + admissionReviewVersions: [v1] + sideEffects: None + failurePolicy: Fail + matchPolicy: Equivalent + timeoutSeconds: 5 + clientConfig: + service: + namespace: internal-dns-system + name: internal-dns-admission + path: /validate-internal-dns + port: 443 + rules: + # Product declarations and the private-visibility guard for DNSZone. The + # legacy DNSRecordSet API is intentionally outside this webhook. + - apiGroups: [dns.networking.miloapis.com] + apiVersions: [v1alpha1] + operations: [CREATE, UPDATE, DELETE] + resources: + - dnszones + - dnszoneassociations + - dnsregistrations + - dnscontributiongrants + - dnsrecordcontributions + - dnsnamingpolicies + - dnsresolvercontexts + - dnsresolveraccessbindings + scope: Namespaced + # A product can report only its own contribution freshness. No + # dnsrecordsets/status rule is installed, preserving the public operator. + - apiGroups: [dns.networking.miloapis.com] + apiVersions: [v1alpha1] + operations: [UPDATE] + resources: [dnsrecordcontributions/status] + scope: Namespaced + - apiGroups: [dns.networking.miloapis.com] + apiVersions: [v1alpha1] + operations: [UPDATE] + resources: [dnsresolvercontexts/status, dnsresolveraccessbindings/status] + scope: Namespaced + # Platform-owned serving and publication state, including their status. + - apiGroups: [dns.networking.miloapis.com] + apiVersions: [v1alpha1] + operations: [CREATE, UPDATE, DELETE] + resources: + - dnsresolverbindings + - dnsresolverbindings/status + - dnspublicationmanifests + - dnspublicationmanifests/status + - dnspublicationchunks + - dnstransportoutboxes + - dnstransportoutboxes/status + - dnspublicationownerships + - dnspublicationownerships/status + - dnsmanagednamespaces + - dnsmanagednamespaces/status + - dnsregistrations/status + - dnscontributiongrants/status + - dnszones/status + scope: Namespaced diff --git a/config/internal-dns/control-plane.example.yaml b/config/internal-dns/control-plane.example.yaml new file mode 100644 index 00000000..e9efb835 --- /dev/null +++ b/config/internal-dns/control-plane.example.yaml @@ -0,0 +1,120 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: internal-dns-control-plane-config + namespace: internal-dns-system +data: + config.json: | + { + "region": "us-central1", + "shard": "shared-0", + "platformNamespace": "internal-dns-system", + "platformKubeAPI": {"qps":50,"burst":100,"requestTimeoutSeconds":10}, + "projects": [ + { + "name": "example-project", + "projectUID": "00000000-0000-0000-0000-000000000001", + "sourceClusterUID": "00000000-0000-0000-0000-000000000010", + "namespace": "example-project", + "kubeAPI": {"qps":20,"burst":40,"requestTimeoutSeconds":10} + } + ], + "nodeBackends": [ + {"memberID":"node-bind-0","address":"fd70:10::10","port":5300} + ], + "clusterBackends": [ + {"memberID":"regional-bind-us-central1-0","address":"fd70:20::10","port":5300}, + {"memberID":"regional-bind-us-central1-1","address":"fd70:20::11","port":5300} + ], + "members": [ + {"memberID":"node-us-central1-0","replicaID":"node-us-central1-0","role":"node"}, + {"memberID":"regional-front-us-central1-0","replicaID":"regional-front-us-central1-0","role":"resolver"}, + {"memberID":"regional-bind-us-central1-0","replicaID":"regional-bind-us-central1-0","role":"regional"}, + {"memberID":"regional-bind-us-central1-1","replicaID":"regional-bind-us-central1-1","role":"regional"} + ], + "consumerPrefix": "fd70:100::/48", + "clusterPrefix": "fd70:200::/48", + "managedDomainSuffix": "internal.example.net", + "privateZoneClassName": "private-bind", + "ownershipLeaseSeconds": 30, + "shards": [ + {"region":"us-central1","shard":"shared-0","maxBindings":200,"maxSnapshotBytes":524288} + ], + "nats": { + "url": "tls://nats.internal-dns-system.svc:4222", + "name": "internal-dns-control-plane", + "username": "internal-dns-control-plane", + "passwordFile": "/var/run/internal-dns/nats/password", + "tls": { + "caFile": "/var/run/internal-dns/nats/ca.crt", + "certificateFile": "/var/run/internal-dns/nats/tls.crt", + "keyFile": "/var/run/internal-dns/nats/tls.key", + "serverName": "nats.internal-dns-system.svc" + } + }, + "stream": { + "name": "DNS_PRIVATE", + "subjects": ["dns.private.serving.>","dns.private.records.>","dns.private.acks.>"], + "maxAge": 86400000000000, + "maxBytes": 10737418240, + "replicas": 3, + "snapshotsBucket": "DNS_PRIVATE_SNAPSHOTS" + }, + "ensureStream": true, + "admission": { + "enabled": true, + "port": 9443, + "certDir": "/var/run/internal-dns/admission", + "platformSubjects": ["system:serviceaccount:internal-dns-system:internal-dns-control-plane"], + "integrationSubjects": ["system:serviceaccount:network-system:dns-vpc-integration"], + "maxContributionLeaseSeconds": 90, + "maxAccessLeaseSeconds": 300 + } + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: internal-dns-control-plane + namespace: internal-dns-system +spec: + replicas: 1 + selector: + matchLabels: {app.kubernetes.io/name: internal-dns-control-plane} + template: + metadata: + labels: {app.kubernetes.io/name: internal-dns-control-plane} + spec: + serviceAccountName: internal-dns-control-plane + containers: + - name: control-plane + image: example.invalid/dns-operator/internal-dns:VERSION + args: ["--role=control-plane","--config=/etc/internal-dns/config.json"] + ports: + - {name: admission, containerPort: 9443} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + capabilities: {drop: [ALL]} + volumeMounts: + - {name: config, mountPath: /etc/internal-dns, readOnly: true} + - {name: nats, mountPath: /var/run/internal-dns/nats, readOnly: true} + - {name: admission, mountPath: /var/run/internal-dns/admission, readOnly: true} + volumes: + - name: config + configMap: {name: internal-dns-control-plane-config} + - name: nats + secret: {secretName: internal-dns-control-plane-nats} + - name: admission + secret: {secretName: internal-dns-admission-tls} +--- +apiVersion: v1 +kind: Service +metadata: + name: internal-dns-admission + namespace: internal-dns-system +spec: + selector: {app.kubernetes.io/name: internal-dns-control-plane} + ports: + - {name: https, port: 443, targetPort: admission} diff --git a/config/internal-dns/fleet.example.yaml b/config/internal-dns/fleet.example.yaml new file mode 100644 index 00000000..19ef501f --- /dev/null +++ b/config/internal-dns/fleet.example.yaml @@ -0,0 +1,123 @@ +# Fixed shared fleet members for one regional shard. Deploy at least two copies +# of the regional-bind member with distinct IDs, addresses, state, config, NATS +# credentials, and watchdog leases. Contexts and zones never create workloads. +apiVersion: v1 +kind: ConfigMap +metadata: {name: internal-dns-node-0-config, namespace: internal-dns-system} +data: + config.json: | + { + "nats":{"url":"tls://nats.internal-dns-system.svc:4222","name":"node-us-central1-0","username":"node-us-central1-0","passwordFile":"/var/run/internal-dns/nats/password","tls":{"caFile":"/var/run/internal-dns/nats/ca.crt","certificateFile":"/var/run/internal-dns/nats/tls.crt","keyFile":"/var/run/internal-dns/nats/tls.key","serverName":"nats.internal-dns-system.svc"}}, + "stream":{"name":"DNS_PRIVATE","snapshotsBucket":"DNS_PRIVATE_SNAPSHOTS"}, + "agent":{ + "region":"us-central1","shard":"shared-0","memberID":"node-us-central1-0","replicaID":"node-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"resolver", + "render":{"role":"node","nodeDNSDist":{"path":"/etc/internal-dns/node-dnsdist.conf","listenAddress":"[::]:53","acls":["fd71:100::/64"],"poolName":"node_shared","backends":[{"memberID":"node-bind-0","address":"fd70:10::10","port":5300}]},"nodeBIND":{"path":"/etc/internal-dns/node-bind.conf","listenAddress":"fd70:10::10","port":5300,"proxyPeers":["fd70:10::20"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, + "transaction":{"stageDir":"/var/lib/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-dnsdist","args":["node","{stageDir}/node-dnsdist.conf"],"whenChanged":["/etc/internal-dns/node-dnsdist.conf"]},{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["node","{stageDir}/node-bind.conf"],"whenChanged":["/etc/internal-dns/node-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["node"],"whenChanged":["/etc/internal-dns/node-bind.conf"]},{"path":"/usr/local/sbin/internal-dns-reload-dnsdist","args":["node"],"whenChanged":["/etc/internal-dns/node-dnsdist.conf"]}]}, + "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["node","{name}"]}],"expiryInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"localServingLease":90000000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"resolverDNSProbe":{"timeout":3000000000},"resolverVerifyConcurrency":16 + }, + "watchdog":{"leasePath":"/run/internal-dns/lease.json","memberID":"node-us-central1-0","replicaID":"node-us-central1-0","maxLeaseSeconds":5,"startupGraceSeconds":0,"intervalMillis":250,"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["node"]}]} + } +--- +apiVersion: v1 +kind: ConfigMap +metadata: {name: internal-dns-regional-front-0-config, namespace: internal-dns-system} +data: + config.json: | + { + "nats":{"url":"tls://nats.internal-dns-system.svc:4222","name":"regional-front-us-central1-0","username":"regional-front-us-central1-0","passwordFile":"/var/run/internal-dns/nats/password","tls":{"caFile":"/var/run/internal-dns/nats/ca.crt","certificateFile":"/var/run/internal-dns/nats/tls.crt","keyFile":"/var/run/internal-dns/nats/tls.key","serverName":"nats.internal-dns-system.svc"}}, + "stream":{"name":"DNS_PRIVATE","snapshotsBucket":"DNS_PRIVATE_SNAPSHOTS"}, + "agent":{ + "region":"us-central1","shard":"shared-0","memberID":"regional-front-us-central1-0","replicaID":"regional-front-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"resolver", + "render":{"role":"regional-dnsdist","clusterDNSDist":{"path":"/etc/internal-dns/cluster-dnsdist.conf","listenAddress":"[::]:53","acls":["fd71:200::/64"],"poolName":"regional_shared","backends":[{"memberID":"regional-bind-us-central1-0","address":"fd70:20::10","port":5300},{"memberID":"regional-bind-us-central1-1","address":"fd70:20::11","port":5300}]}}, + "transaction":{"stageDir":"/var/lib/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-dnsdist","args":["regional","{stageDir}/cluster-dnsdist.conf"],"whenChanged":["/etc/internal-dns/cluster-dnsdist.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-dnsdist","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-dnsdist.conf"]}]}, + "expiryInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"localServingLease":90000000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"resolverDNSProbe":{"timeout":3000000000},"resolverVerifyConcurrency":16 + }, + "watchdog":{"leasePath":"/run/internal-dns/lease.json","memberID":"regional-front-us-central1-0","replicaID":"regional-front-us-central1-0","maxLeaseSeconds":5,"startupGraceSeconds":0,"intervalMillis":250,"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["regional-front"]}]} + } +--- +apiVersion: v1 +kind: ConfigMap +metadata: {name: internal-dns-regional-bind-0-config, namespace: internal-dns-system} +data: + config.json: | + { + "nats":{"url":"tls://nats.internal-dns-system.svc:4222","name":"regional-bind-us-central1-0","username":"regional-bind-us-central1-0","passwordFile":"/var/run/internal-dns/nats/password","tls":{"caFile":"/var/run/internal-dns/nats/ca.crt","certificateFile":"/var/run/internal-dns/nats/tls.crt","keyFile":"/var/run/internal-dns/nats/tls.key","serverName":"nats.internal-dns-system.svc"}}, + "stream":{"name":"DNS_PRIVATE","snapshotsBucket":"DNS_PRIVATE_SNAPSHOTS"}, + "agent":{ + "region":"us-central1","shard":"shared-0","memberID":"regional-bind-us-central1-0","replicaID":"regional-bind-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"combined", + "render":{"role":"regional-bind","clusterBIND":{"path":"/etc/internal-dns/cluster-bind.conf","listenAddress":"fd70:20::10","port":5300,"proxyPeers":["fd70:20::20","fd70:20::30"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, + "transaction":{"stageDir":"/var/lib/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["regional","-z","{stageDir}/cluster-bind.conf"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}]}, + "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["regional","{name}"]}],"expiryInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"publicationDNSProbe":{"server":"[fd70:20::10]:5300","timeout":3000000000} + }, + "watchdog":{"leasePath":"/run/internal-dns/lease.json","memberID":"regional-bind-us-central1-0","replicaID":"regional-bind-us-central1-0","maxLeaseSeconds":5,"startupGraceSeconds":0,"intervalMillis":250,"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["regional-bind"]}]} + } +--- +# The second regional member has independent identity, listener, state, +# checkpoint, and watchdog storage. Its backend memberID exactly matches the +# second clusterBackends entry in the control-plane configuration. +apiVersion: v1 +kind: ConfigMap +metadata: {name: internal-dns-regional-bind-1-config, namespace: internal-dns-system} +data: + config.json: | + { + "nats":{"url":"tls://nats.internal-dns-system.svc:4222","name":"regional-bind-us-central1-1","username":"regional-bind-us-central1-1","passwordFile":"/var/run/internal-dns/nats/password","tls":{"caFile":"/var/run/internal-dns/nats/ca.crt","certificateFile":"/var/run/internal-dns/nats/tls.crt","keyFile":"/var/run/internal-dns/nats/tls.key","serverName":"nats.internal-dns-system.svc"}}, + "stream":{"name":"DNS_PRIVATE","snapshotsBucket":"DNS_PRIVATE_SNAPSHOTS"}, + "agent":{ + "region":"us-central1","shard":"shared-0","memberID":"regional-bind-us-central1-1","replicaID":"regional-bind-us-central1-1","stateDir":"/var/lib/internal-dns","mode":"combined", + "render":{"role":"regional-bind","clusterBIND":{"path":"/etc/internal-dns/cluster-bind.conf","listenAddress":"fd70:20::11","port":5300,"proxyPeers":["fd70:20::20","fd70:20::31"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, + "transaction":{"stageDir":"/var/lib/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["regional","-z","{stageDir}/cluster-bind.conf"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}]}, + "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["regional","{name}"]}],"expiryInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"publicationDNSProbe":{"server":"[fd70:20::11]:5300","timeout":3000000000} + }, + "watchdog":{"leasePath":"/run/internal-dns/lease.json","memberID":"regional-bind-us-central1-1","replicaID":"regional-bind-us-central1-1","maxLeaseSeconds":5,"startupGraceSeconds":0,"intervalMillis":250,"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["regional-bind"]}]} + } +--- +# This Deployment template is instantiated once for each ConfigMap above and a +# second time for regional-bind-1. The serving image contains the local dnsdist +# and/or BIND process selected by MEMBER_ROLE plus the validation/reload/gate +# wrappers referenced by that member's config. +apiVersion: apps/v1 +kind: Deployment +metadata: {name: internal-dns-regional-bind-0, namespace: internal-dns-system} +spec: + replicas: 1 + strategy: {type: Recreate} + selector: {matchLabels: {app.kubernetes.io/name: internal-dns-regional-bind-0}} + template: + metadata: {labels: {app.kubernetes.io/name: internal-dns-regional-bind-0, internal-dns.datum.net/tier: regional}} + spec: + serviceAccountName: internal-dns-fleet + automountServiceAccountToken: false + shareProcessNamespace: true + securityContext: {fsGroup: 65532, fsGroupChangePolicy: OnRootMismatch} + containers: + - name: agent + image: example.invalid/dns-operator/internal-dns-serving:VERSION + args: ["--role=agent","--config=/config/config.json"] + securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, runAsNonRoot: true, capabilities: {drop: [ALL]}} + volumeMounts: &member-mounts + - {name: config, mountPath: /config, readOnly: true} + - {name: nats, mountPath: /var/run/internal-dns/nats, readOnly: true} + - {name: lease, mountPath: /run/internal-dns} + - {name: state, mountPath: /var/lib/internal-dns} + - {name: daemon-config, mountPath: /etc/internal-dns} + - name: watchdog + image: example.invalid/dns-operator/internal-dns-serving:VERSION + args: ["--role=watchdog","--config=/config/config.json"] + securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, runAsNonRoot: true, capabilities: {drop: [ALL]}} + volumeMounts: *member-mounts + - name: serving + image: example.invalid/dns-operator/internal-dns-bind:VERSION + args: ["regional-bind"] + securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, runAsUser: 65532, runAsGroup: 65532, capabilities: {drop: [ALL], add: [NET_BIND_SERVICE]}} + volumeMounts: + - {name: daemon-config, mountPath: /etc/internal-dns} + # BIND requires a writable working directory with a read-only root filesystem. + - {name: runtime, mountPath: /tmp} + volumes: + - {name: config, configMap: {name: internal-dns-regional-bind-0-config}} + - {name: nats, secret: {secretName: internal-dns-regional-bind-0-nats}} + - {name: lease, emptyDir: {medium: Memory}} + - {name: runtime, emptyDir: {}} + - {name: daemon-config, persistentVolumeClaim: {claimName: internal-dns-regional-bind-0-config}} + - {name: state, persistentVolumeClaim: {claimName: internal-dns-regional-bind-0-state}} diff --git a/config/internal-dns/kustomization.yaml b/config/internal-dns/kustomization.yaml new file mode 100644 index 00000000..32a75931 --- /dev/null +++ b/config/internal-dns/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- namespace.yaml +- rbac.yaml +- control-plane.example.yaml +- fleet.example.yaml +- admission.example.yaml diff --git a/config/internal-dns/namespace.yaml b/config/internal-dns/namespace.yaml new file mode 100644 index 00000000..13650a17 --- /dev/null +++ b/config/internal-dns/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: internal-dns-system + labels: + app.kubernetes.io/part-of: internal-dns diff --git a/config/internal-dns/nats-server.conf.example b/config/internal-dns/nats-server.conf.example new file mode 100644 index 00000000..c75850a0 --- /dev/null +++ b/config/internal-dns/nats-server.conf.example @@ -0,0 +1,149 @@ +# This file demonstrates application-subject isolation. Password variables are +# injected by the broker secret manager; use NKey/JWT credentials in a NATS +# operator deployment. No product workload receives a NATS identity. +server_name: internal-dns-1 +jetstream { + store_dir: "/var/lib/nats/jetstream" +} +tls { + cert_file: "/var/run/nats/tls/tls.crt" + key_file: "/var/run/nats/tls/tls.key" + ca_file: "/var/run/nats/tls/ca.crt" + verify: true + timeout: 2 +} +authorization { + users: [ + { + user: "internal-dns-control-plane" + password: $CONTROL_PLANE_PASSWORD + permissions: { + publish: { allow: [ + "dns.private.serving.>", "dns.private.records.>", + "$KV.DNS_PRIVATE_SNAPSHOTS.>", "$JS.API.>", "_INBOX.>" + ] } + subscribe: { allow: ["dns.private.acks.>", "_INBOX.>"] } + } + }, + { + user: "node-us-central1-0" + password: $NODE_RESOLVER_0_PASSWORD + permissions: { + publish: { allow: [ + "dns.private.acks.us-central1.shared-0.0195a8650ce9d5c927c91495", + "$JS.API.STREAM.INFO.DNS_PRIVATE", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.node-us-central1-0-serving", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.node-us-central1-0-records", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.node-us-central1-0-acks", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.node-us-central1-0-serving", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.node-us-central1-0-serving.>", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.node-us-central1-0-records", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.node-us-central1-0-records.>", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.node-us-central1-0-acks", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.node-us-central1-0-acks.>", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.node-us-central1-0-serving", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.node-us-central1-0-records", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.node-us-central1-0-acks", + "$JS.ACK.DNS_PRIVATE.node-us-central1-0-serving.>", + "$JS.ACK.DNS_PRIVATE.node-us-central1-0-records.>", + "$JS.ACK.DNS_PRIVATE.node-us-central1-0-acks.>", + "$JS.API.DIRECT.GET.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.STREAM.INFO.KV_DNS_PRIVATE_SNAPSHOTS", + "$JS.API.CONSUMER.CREATE.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.CONSUMER.INFO.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.CONSUMER.MSG.NEXT.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.CONSUMER.DELETE.KV_DNS_PRIVATE_SNAPSHOTS.>", + "_INBOX.>" + ] } + subscribe: { allow: ["_INBOX.>"] } + } + }, + { + user: "regional-front-us-central1-0" + password: $REGIONAL_FRONT_0_PASSWORD + permissions: { + publish: { allow: [ + "dns.private.acks.us-central1.shared-0.92106c62061ed0349bc66280", + "$JS.API.STREAM.INFO.DNS_PRIVATE", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-front-us-central1-0-serving", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-front-us-central1-0-records", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-front-us-central1-0-acks", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-front-us-central1-0-serving", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-front-us-central1-0-serving.>", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-front-us-central1-0-records", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-front-us-central1-0-records.>", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-front-us-central1-0-acks", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-front-us-central1-0-acks.>", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-front-us-central1-0-serving", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-front-us-central1-0-records", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-front-us-central1-0-acks", + "$JS.ACK.DNS_PRIVATE.regional-front-us-central1-0-serving.>", + "$JS.ACK.DNS_PRIVATE.regional-front-us-central1-0-records.>", + "$JS.ACK.DNS_PRIVATE.regional-front-us-central1-0-acks.>", + "$JS.API.DIRECT.GET.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.STREAM.INFO.KV_DNS_PRIVATE_SNAPSHOTS", + "$JS.API.CONSUMER.CREATE.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.CONSUMER.INFO.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.CONSUMER.MSG.NEXT.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.CONSUMER.DELETE.KV_DNS_PRIVATE_SNAPSHOTS.>", + "_INBOX.>" + ] } + subscribe: { allow: ["_INBOX.>"] } + } + }, + { + user: "regional-bind-us-central1-0" + password: $REGIONAL_BIND_0_PASSWORD + permissions: { + publish: { allow: [ + "dns.private.acks.us-central1.shared-0.b75afe155467f45fdab6d0a1", + "$JS.API.STREAM.INFO.DNS_PRIVATE", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-bind-us-central1-0-serving", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-bind-us-central1-0-records", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-bind-us-central1-0-acks", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-bind-us-central1-0-serving", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-bind-us-central1-0-serving.>", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-bind-us-central1-0-records", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-bind-us-central1-0-records.>", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-bind-us-central1-0-acks", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-bind-us-central1-0-acks.>", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-bind-us-central1-0-serving", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-bind-us-central1-0-records", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-bind-us-central1-0-acks", + "$JS.ACK.DNS_PRIVATE.regional-bind-us-central1-0-serving.>", + "$JS.ACK.DNS_PRIVATE.regional-bind-us-central1-0-records.>", + "$JS.ACK.DNS_PRIVATE.regional-bind-us-central1-0-acks.>", + "$JS.API.DIRECT.GET.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.STREAM.INFO.KV_DNS_PRIVATE_SNAPSHOTS", + "$JS.API.CONSUMER.CREATE.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.CONSUMER.INFO.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.CONSUMER.MSG.NEXT.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.CONSUMER.DELETE.KV_DNS_PRIVATE_SNAPSHOTS.>", + "_INBOX.>" + ] } + subscribe: { allow: ["_INBOX.>"] } + } + }, + { + user: "regional-bind-us-central1-1" + password: $REGIONAL_BIND_1_PASSWORD + permissions: { + publish: { allow: [ + "dns.private.acks.us-central1.shared-0.2600654ae9645c6e4439e8ac", + "$JS.API.STREAM.INFO.DNS_PRIVATE", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-bind-us-central1-1-serving", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-bind-us-central1-1-records", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-bind-us-central1-1-acks", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-bind-us-central1-1-serving", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-bind-us-central1-1-serving.>", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-bind-us-central1-1-records", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-bind-us-central1-1-records.>", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-bind-us-central1-1-acks", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-bind-us-central1-1-acks.>", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-bind-us-central1-1-serving", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-bind-us-central1-1-records", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-bind-us-central1-1-acks", + "$JS.ACK.DNS_PRIVATE.regional-bind-us-central1-1-serving.>", + "$JS.ACK.DNS_PRIVATE.regional-bind-us-central1-1-records.>", + "$JS.ACK.DNS_PRIVATE.regional-bind-us-central1-1-acks.>", + "$JS.API.DIRECT.GET.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.STREAM.INFO.KV_DNS_PRIVATE_SNAPSHOTS", + "$JS.API.CONSUMER.CREATE.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.CONSUMER.INFO.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.CONSUMER.MSG.NEXT.KV_DNS_PRIVATE_SNAPSHOTS.>", + "$JS.API.CONSUMER.DELETE.KV_DNS_PRIVATE_SNAPSHOTS.>", + "_INBOX.>" + ] } + subscribe: { allow: ["_INBOX.>"] } + } + } + ] +} diff --git a/config/internal-dns/rbac.yaml b/config/internal-dns/rbac.yaml new file mode 100644 index 00000000..140db19e --- /dev/null +++ b/config/internal-dns/rbac.yaml @@ -0,0 +1,128 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: internal-dns-control-plane + namespace: internal-dns-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: internal-dns-fleet + namespace: internal-dns-system +automountServiceAccountToken: false +--- +# Bind this Role only to the separately deployed VPC integration principal. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: internal-dns-vpc-integration + namespace: example-project +rules: +- apiGroups: [dns.networking.miloapis.com] + resources: [dnsresolvercontexts, dnsresolveraccessbindings] + verbs: [get, list, watch, create, update, patch, delete] +- apiGroups: [dns.networking.miloapis.com] + resources: [dnsresolvercontexts/status, dnsresolveraccessbindings/status, dnsmanagednamespaces, dnsmanagednamespaces/status] + verbs: [get, list, watch] +--- +# Project products write declarations and their own freshness observations. +# Platform artifacts and every other status subresource are deliberately absent. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: internal-dns-product-writer + namespace: example-project +rules: +- apiGroups: [dns.networking.miloapis.com] + resources: [dnsregistrations, dnsrecordcontributions] + verbs: [get, list, watch, create, update, patch, delete] +- apiGroups: [dns.networking.miloapis.com] + resources: [dnscontributiongrants, dnsresolvercontexts, dnsmanagednamespaces, dnsnamingpolicies] + verbs: [get, list, watch] +- apiGroups: [dns.networking.miloapis.com] + resources: [dnsrecordcontributions/status] + verbs: [get, update, patch] +--- +# Bind only to a trusted issuer that applies the product's registration policy. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: internal-dns-grant-issuer + namespace: example-project +rules: +- apiGroups: [dns.networking.miloapis.com] + resources: [dnsregistrations] + # Admission authorizes grant issuance against the exact registration's update permission. + verbs: [get, update] +- apiGroups: [dns.networking.miloapis.com] + resources: [dnscontributiongrants] + verbs: [get, list, watch, create, update, patch, delete] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: internal-dns-control-plane +rules: +- apiGroups: [dns.networking.miloapis.com] + resources: + - dnszones + - dnsrecordsets + - dnszoneassociations + - dnsregistrations + - dnscontributiongrants + - dnsrecordcontributions + - dnsnamingpolicies + - dnsresolvercontexts + - dnsresolveraccessbindings + verbs: [get, list, watch] +# Managed namespace creation and the private-zone publication finalizer. +- apiGroups: [dns.networking.miloapis.com] + resources: [dnszones, dnszoneassociations] + verbs: [create, patch] +- apiGroups: [dns.networking.miloapis.com] + resources: + - dnsresolverbindings + - dnspublicationmanifests + - dnspublicationchunks + - dnstransportoutboxes + - dnspublicationownerships + - dnsmanagednamespaces + verbs: [get, list, watch, create, update, patch, delete] +- apiGroups: [dns.networking.miloapis.com] + resources: + - dnsresolverbindings/status + - dnspublicationmanifests/status + - dnstransportoutboxes/status + - dnspublicationownerships/status + - dnsmanagednamespaces/status + - dnszones/status + - dnszoneassociations/status + - dnsnamingpolicies/status + - dnsregistrations/status + - dnscontributiongrants/status + - dnsrecordcontributions/status + - dnsresolvercontexts/status + - dnsresolveraccessbindings/status + verbs: [get, update, patch] +- apiGroups: [authorization.k8s.io] + resources: [subjectaccessreviews] + verbs: [create] +- apiGroups: [coordination.k8s.io] + resources: [leases] + verbs: [get, list, watch, create, update, patch] +- apiGroups: [""] + resources: [configmaps] + verbs: [get, list, watch, create, update, patch] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: internal-dns-control-plane +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: internal-dns-control-plane +subjects: +- kind: ServiceAccount + name: internal-dns-control-plane + namespace: internal-dns-system diff --git a/internal/internaldns/runtime/config.go b/internal/internaldns/runtime/config.go new file mode 100644 index 00000000..a381c093 --- /dev/null +++ b/internal/internaldns/runtime/config.go @@ -0,0 +1,350 @@ +package runtime + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" + "go.miloapis.com/dns-operator/internal/internaldns/platform" + "go.miloapis.com/dns-operator/internal/internaldns/serving" + "go.miloapis.com/dns-operator/internal/internaldns/transport" + "go.miloapis.com/dns-operator/internal/internaldns/watchdog" + "k8s.io/apimachinery/pkg/types" +) + +const ( + roleWatchdog = "watchdog" + roleControlPlane = "control-plane" + memberRoleRegional = "regional" + roleAgent = "agent" + memberRoleCluster = "cluster" +) + +type ProjectConfig struct { + Name string `json:"name,omitempty"` + ProjectUID types.UID `json:"projectUID"` + SourceClusterUID string `json:"sourceClusterUID"` + Namespace string `json:"namespace"` + Kubeconfig string `json:"kubeconfig,omitempty"` + KubeAPI KubeAPIClientConfig `json:"kubeAPI,omitempty"` +} + +type KubeAPIClientConfig struct { + QPS float32 `json:"qps,omitempty"` + Burst int `json:"burst,omitempty"` + RequestTimeoutSeconds int64 `json:"requestTimeoutSeconds,omitempty"` +} + +type AdmissionConfig struct { + Enabled bool `json:"enabled"` + Port int `json:"port,omitempty"` + CertDir string `json:"certDir,omitempty"` + PlatformSubjects []string `json:"platformSubjects,omitempty"` + MaxContributionLeaseSeconds int64 `json:"maxContributionLeaseSeconds,omitempty"` + IntegrationSubjects []string `json:"integrationSubjects,omitempty"` + MaxAccessLeaseSeconds int64 `json:"maxAccessLeaseSeconds,omitempty"` +} + +type RegionalTransportConfig struct { + Region string `json:"region"` + NATS transport.Config `json:"nats"` + Stream transport.StreamConfig `json:"stream"` + EnsureStream bool `json:"ensureStream,omitempty"` +} + +type Config struct { + Region string `json:"region"` + Shard string `json:"shard"` + PlatformNamespace string `json:"platformNamespace"` + PlatformKubeAPI KubeAPIClientConfig `json:"platformKubeAPI,omitempty"` + Projects []ProjectConfig `json:"projects"` + NodeBackends []model.Backend `json:"nodeBackends,omitempty"` + ClusterBackends []model.Backend `json:"clusterBackends,omitempty"` + Members []model.Member `json:"members,omitempty"` + ConsumerPrefix string `json:"consumerPrefix,omitempty"` + ClusterPrefix string `json:"clusterPrefix,omitempty"` + NATS transport.Config `json:"nats"` + Stream transport.StreamConfig `json:"stream"` + EnsureStream bool `json:"ensureStream,omitempty"` + RegionalTransports []RegionalTransportConfig `json:"regionalTransports,omitempty"` + Admission AdmissionConfig `json:"admission,omitempty"` + OwnershipLeaseSeconds int64 `json:"ownershipLeaseSeconds,omitempty"` + ManagedDomainSuffix string `json:"managedDomainSuffix,omitempty"` + PrivateZoneClassName string `json:"privateZoneClassName,omitempty"` + Agent serving.Config `json:"agent,omitempty"` + Watchdog watchdog.Config `json:"watchdog,omitempty"` + Shards []platform.PlannerConfig `json:"shards,omitempty"` +} + +func LoadConfig(path string) (Config, error) { + if path == "" { + return Config{}, errors.New("--config is required") + } + if !filepath.IsAbs(path) { + return Config{}, fmt.Errorf("config path must be absolute: %s", path) + } + b, err := os.ReadFile(path) + if err != nil { + return Config{}, err + } + var c Config + decoder := json.NewDecoder(bytes.NewReader(b)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&c); err != nil { + return Config{}, fmt.Errorf("decode config: %w", err) + } + if err := decoder.Decode(new(any)); err != io.EOF { + if err == nil { + return Config{}, errors.New("config must contain exactly one JSON object") + } + return Config{}, fmt.Errorf("decode trailing config data: %w", err) + } + return c, nil +} + +func (c *Config) defaults() { + if c.PlatformNamespace == "" { + c.PlatformNamespace = "internal-dns-system" + } + + if c.OwnershipLeaseSeconds <= 0 { + c.OwnershipLeaseSeconds = 30 + } + if c.Admission.Port == 0 { + c.Admission.Port = 9443 + } + if c.Admission.MaxContributionLeaseSeconds <= 0 { + c.Admission.MaxContributionLeaseSeconds = 90 + } + if c.Admission.MaxAccessLeaseSeconds <= 0 { + c.Admission.MaxAccessLeaseSeconds = 300 + } + + if c.NATS.Name == "" { + c.NATS.Name = "internal-dns" + } + + defaultKubeAPI(&c.PlatformKubeAPI, 50, 100) + for i := range c.Projects { + defaultKubeAPI(&c.Projects[i].KubeAPI, 20, 40) + } +} + +func defaultKubeAPI(c *KubeAPIClientConfig, qps float32, burst int) { + if c.QPS == 0 { + c.QPS = qps + } + if c.Burst == 0 { + c.Burst = burst + } + if c.RequestTimeoutSeconds == 0 { + c.RequestTimeoutSeconds = 10 + } +} + +func validateKubeAPI(name string, c KubeAPIClientConfig) error { + if c.QPS <= 0 || c.QPS > 1000 { + return fmt.Errorf("%s.qps must be greater than 0 and at most 1000", name) + } + if c.Burst <= 0 || c.Burst > 2000 { + return fmt.Errorf("%s.burst must be greater than 0 and at most 2000", name) + } + if c.RequestTimeoutSeconds <= 0 || c.RequestTimeoutSeconds > 120 { + return fmt.Errorf("%s.requestTimeoutSeconds must be greater than 0 and at most 120", name) + } + return nil +} + +func (c *Config) Validate(role string) error { + c.defaults() + switch role { + case roleControlPlane, roleAgent, roleWatchdog: + default: + return fmt.Errorf("unsupported role %q", role) + } + if role == roleWatchdog { + return c.Watchdog.Validate() + } + if len(c.RegionalTransports) == 0 { + if err := validateTransportConfig(c.NATS, c.Stream, c.EnsureStream); err != nil { + return err + } + } else { + seen := map[string]bool{} + for i := range c.RegionalTransports { + regional := &c.RegionalTransports[i] + if regional.NATS.Name == "" { + regional.NATS.Name = "internal-dns-" + model.SafeToken(regional.Region) + } + if regional.Region == "" || seen[regional.Region] { + return fmt.Errorf("regional transport %d has missing or duplicate region", i) + } + seen[regional.Region] = true + if err := validateTransportConfig(regional.NATS, regional.Stream, regional.EnsureStream); err != nil { + return fmt.Errorf("regional transport %s: %w", regional.Region, err) + } + } + } + if role == roleAgent { + if c.Agent.Region == "" { + c.Agent.Region = c.Region + } + if c.Agent.Shard == "" { + c.Agent.Shard = c.Shard + } + if _, _, _, err := c.transportForRegion(c.Agent.Region); err != nil { + return err + } + return c.Agent.Validate() + } + if c.PlatformNamespace == "" || len(c.Projects) == 0 { + return errors.New("control-plane requires platformNamespace and projects") + } + if err := validateKubeAPI("platformKubeAPI", c.PlatformKubeAPI); err != nil { + return err + } + names := map[string]bool{} + uids := map[types.UID]bool{} + namespaces := map[string]bool{} + for i, p := range c.Projects { + if p.ProjectUID == "" || p.SourceClusterUID == "" || p.Namespace == "" { + return fmt.Errorf("project %d requires projectUID, sourceClusterUID, and namespace", i) + } + if p.Namespace == c.PlatformNamespace { + return fmt.Errorf("project %d namespace collides with platformNamespace", i) + } + if err := validateKubeAPI(fmt.Sprintf("projects[%d].kubeAPI", i), p.KubeAPI); err != nil { + return err + } + if uids[p.ProjectUID] { + return fmt.Errorf("duplicate project UID %q", p.ProjectUID) + } + uids[p.ProjectUID] = true + if p.Name != "" { + if names[p.Name] { + return fmt.Errorf("duplicate project name %q", p.Name) + } + names[p.Name] = true + } + if p.Kubeconfig == "" && namespaces[p.Namespace] { + return fmt.Errorf("duplicate project namespace %q on the platform cluster", p.Namespace) + } + namespaces[p.Namespace] = true + } + if len(c.Projects) > 1 { + for _, p := range c.Projects { + if p.Name == "" { + return errors.New("every project needs name when several projects are configured") + } + } + } + if c.Admission.Enabled && !filepath.IsAbs(c.Admission.CertDir) { + return errors.New("admission.certDir must be absolute when admission is enabled") + } + if err := validateMembers(c); err != nil { + return err + } + shardPairs := map[string]bool{} + for i := range c.Shards { + if c.Shards[i].Region == "" || c.Shards[i].Shard == "" { + return fmt.Errorf("shard %d requires region and shard", i) + } + key := c.Shards[i].Region + "/" + c.Shards[i].Shard + if shardPairs[key] { + return fmt.Errorf("duplicate shard %q", key) + } + shardPairs[key] = true + } + for _, planner := range c.plannerConfigs() { + if _, _, _, err := c.transportForRegion(planner.Region); err != nil { + return err + } + count := 0 + for _, member := range planner.Members { + if member.Role == memberRoleRegional || member.Role == memberRoleCluster { + count++ + } + } + if count == 0 { + return fmt.Errorf("shard %s/%s has %d regional resolver members; at least one is required", planner.Region, planner.Shard, count) + } + } + return nil +} + +func (c AdmissionConfig) maxAccessLease() time.Duration { + return time.Duration(c.MaxAccessLeaseSeconds) * time.Second +} + +func validateTransportConfig(nats transport.Config, stream transport.StreamConfig, ensure bool) error { + if nats.URL == "" || stream.Name == "" { + return errors.New("nats.url and stream.name are required") + } + if ensure && len(stream.Subjects) == 0 { + return errors.New("stream.subjects are required when ensureStream is enabled") + } + if stream.SnapshotsBucket == "" { + return errors.New("stream.snapshotsBucket is required for restart bootstrap") + } + return nil +} + +func (c Config) transportForRegion(region string) (transport.Config, transport.StreamConfig, bool, error) { + if len(c.RegionalTransports) == 0 { + return c.NATS, c.Stream, c.EnsureStream, nil + } + for _, regional := range c.RegionalTransports { + if regional.Region == region { + return regional.NATS, regional.Stream, regional.EnsureStream, nil + } + } + return transport.Config{}, transport.StreamConfig{}, false, fmt.Errorf("no transport configured for region %q", region) +} + +func validateMembers(c *Config) error { + seen := map[string]bool{} + check := func(ms []model.Member) error { + for _, m := range ms { + if m.MemberID == "" || m.Role == "" { + return errors.New("member ID and role are required") + } + if m.Role != "node" && m.Role != memberRoleRegional && m.Role != memberRoleCluster && m.Role != "resolver" { + return fmt.Errorf("member %s has unsupported resolver role %s", m.MemberID, m.Role) + } + if seen[m.MemberID] { + return fmt.Errorf("member ID %q is not globally unique", m.MemberID) + } + seen[m.MemberID] = true + } + return nil + } + if len(c.Shards) == 0 { + return check(c.Members) + } + for _, s := range c.Shards { + members := s.Members + if len(members) == 0 { + members = c.Members + } + if err := check(members); err != nil { + return err + } + } + return nil +} + +func (c KubeAPIClientConfig) requestTimeout() time.Duration { + return time.Duration(c.RequestTimeoutSeconds) * time.Second +} +func (c Config) ownershipLease() time.Duration { + return time.Duration(c.OwnershipLeaseSeconds) * time.Second +} +func (c AdmissionConfig) maxLease() time.Duration { + return time.Duration(c.MaxContributionLeaseSeconds) * time.Second +} diff --git a/internal/internaldns/runtime/config_test.go b/internal/internaldns/runtime/config_test.go new file mode 100644 index 00000000..629e668e --- /dev/null +++ b/internal/internaldns/runtime/config_test.go @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package runtime + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" + "go.miloapis.com/dns-operator/internal/internaldns/serving" + "go.miloapis.com/dns-operator/internal/internaldns/transport" + "k8s.io/client-go/rest" +) + +func TestAgentUsesRootRegionAndDoesNotNeedKubernetes(t *testing.T) { + c := Config{ + Region: "us-central", Shard: "shared-1", + NATS: transport.Config{URL: "nats://fixture:4222", AllowInsecure: true}, + Stream: transport.StreamConfig{Name: "DNS", SnapshotsBucket: "DNS_BOOTSTRAP"}, + Agent: serving.Config{MemberID: "resolver-1", ReplicaID: "resolver-1", StateDir: t.TempDir(), Mode: serving.ModeResolver, + Render: serving.RenderConfig{Role: serving.RenderNode, + NodeDNSDist: serving.DNSDistRenderConfig{Path: t.TempDir() + "/dnsdist.conf", ListenAddress: "[::1]:53", PoolName: "node"}, + NodeBIND: serving.BINDRenderConfig{Path: t.TempDir() + "/named.conf", ListenAddress: "::1", Port: 5353}}, + Transaction: serving.TransactionConfig{StageDir: t.TempDir()}, UnsafeAllowNoFailClosed: true, UnsafeAllowNoDNSVerify: true}, + } + if err := c.Validate("agent"); err != nil { + t.Fatal(err) + } + if c.Agent.Region != c.Region || c.Agent.Shard != c.Shard { + t.Fatalf("agent did not inherit shard: %#v", c.Agent) + } +} + +func TestKubeAPIRateLimitDefaultsOverridesAndValidation(t *testing.T) { + base := Config{ + Region: "central", Shard: "one", + NATS: transport.Config{URL: "nats://fixture:4222", AllowInsecure: true}, + Stream: transport.StreamConfig{Name: "DNS", SnapshotsBucket: "DNS_BOOTSTRAP"}, + Projects: []ProjectConfig{{Name: "project", ProjectUID: "p1", SourceClusterUID: "c1", Namespace: "project", + KubeAPI: KubeAPIClientConfig{QPS: 30, Burst: 60}}}, + Members: []model.Member{{MemberID: "regional-1", Role: "regional"}}, + } + if err := base.Validate("control-plane"); err != nil { + t.Fatal(err) + } + if base.PlatformKubeAPI.QPS != 50 || base.PlatformKubeAPI.Burst != 100 { + t.Fatalf("unexpected platform defaults: %#v", base.PlatformKubeAPI) + } + if base.Projects[0].KubeAPI.QPS != 30 || base.Projects[0].KubeAPI.Burst != 60 { + t.Fatalf("source override was not preserved: %#v", base.Projects[0].KubeAPI) + } + restConfig := &rest.Config{} + applyKubeAPILimit(restConfig, base.Projects[0].KubeAPI) + if restConfig.QPS != 30 || restConfig.Burst != 60 || restConfig.RateLimiter == nil { + t.Fatalf("REST client did not receive shared rate limiter: %#v", restConfig) + } + invalid := base + invalid.PlatformKubeAPI.QPS = -1 + if err := invalid.Validate("control-plane"); err == nil { + t.Fatal("negative platform Kubernetes API QPS was accepted") + } + invalid = base + invalid.Projects = append([]ProjectConfig(nil), base.Projects...) + invalid.Projects[0].KubeAPI.Burst = 2001 + if err := invalid.Validate("control-plane"); err == nil { + t.Fatal("excessive source Kubernetes API burst was accepted") + } +} + +func TestProjectSourceFailureDoesNotStopHealthySource(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + failedStarted := make(chan struct{}, 1) + healthyStarted := make(chan struct{}, 1) + projects := []projectRuntime{ + {config: ProjectConfig{Name: "offline", ProjectUID: "p-offline"}, start: func(context.Context) error { + failedStarted <- struct{}{} + return errors.New("source API unavailable") + }}, + {config: ProjectConfig{Name: "healthy", ProjectUID: "p-healthy"}, start: func(ctx context.Context) error { + healthyStarted <- struct{}{} + <-ctx.Done() + return ctx.Err() + }}, + } + startProjectSources(ctx, projects) + for name, started := range map[string]<-chan struct{}{"offline": failedStarted, "healthy": healthyStarted} { + select { + case <-started: + case <-time.After(time.Second): + t.Fatalf("%s source did not start independently", name) + } + } +} + +func TestControlPlaneRejectsAmbiguousProjectsAndMemberIDs(t *testing.T) { + base := Config{ + NATS: transport.Config{URL: "nats://fixture:4222", AllowInsecure: true}, + Stream: transport.StreamConfig{Name: "DNS", SnapshotsBucket: "DNS_BOOTSTRAP"}, + Projects: []ProjectConfig{{ProjectUID: "p1", SourceClusterUID: "c1", Namespace: "n1"}, {ProjectUID: "p2", SourceClusterUID: "c2", Namespace: "n2"}}, + } + if err := base.Validate("control-plane"); err == nil { + t.Fatal("unnamed multi-project configuration was accepted") + } + base.Projects[0].Name, base.Projects[1].Name = "one", "two" + base.Members = []model.Member{{MemberID: "same", Role: "resolver"}, {MemberID: "same", Role: "regional"}} + if err := base.Validate("control-plane"); err == nil { + t.Fatal("duplicate fleet member identity was accepted") + } +} + +func TestConfigLoaderRejectsLegacyNetworkingIntegration(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.json") + if err := os.WriteFile(path, []byte(`{"networking":{"enabled":true}}`), 0600); err != nil { + t.Fatal(err) + } + if _, err := LoadConfig(path); err == nil { + t.Fatal("legacy DNS-owned networking integration was accepted") + } +} + +func TestConfigLoaderRejectsRemovedPrivateServingFields(t *testing.T) { + for _, field := range []string{"authoritativeBackends", "sourcePrefix", "minAuthoritativeReplicas", "resolverContexts", "cacheSize", "authorizationTTLSeconds"} { + t.Run(field, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.json") + if err := os.WriteFile(path, []byte("{\""+field+"\":null}"), 0600); err != nil { + t.Fatal(err) + } + _, err := LoadConfig(path) + if err == nil || !strings.Contains(err.Error(), "unknown field") { + t.Fatalf("removed field accepted: %v", err) + } + }) + } + path := filepath.Join(t.TempDir(), "config.json") + if err := os.WriteFile(path, []byte("{\"region\":\"central\",\"shard\":\"shared\"}"), 0600); err != nil { + t.Fatal(err) + } + loaded, err := LoadConfig(path) + if err != nil || loaded.Region != "central" { + t.Fatalf("current config rejected: %#v %v", loaded, err) + } + if err := os.WriteFile(path, []byte("{} {}"), 0600); err != nil { + t.Fatal(err) + } + if _, err := LoadConfig(path); err == nil { + t.Fatal("multiple config objects accepted") + } +} diff --git a/internal/internaldns/runtime/deadline_client.go b/internal/internaldns/runtime/deadline_client.go new file mode 100644 index 00000000..d3717798 --- /dev/null +++ b/internal/internaldns/runtime/deadline_client.go @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package runtime + +import ( + "context" + "time" + + k8sruntime "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// deadlineClient bounds every direct source API operation without imposing a +// short lifetime on the manager's long-running watch connections. +type deadlineClient struct { + client.Client + Timeout time.Duration +} + +func (c *deadlineClient) bounded(ctx context.Context) (context.Context, context.CancelFunc) { + if c.Timeout <= 0 { + return context.WithCancel(ctx) + } + if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) <= c.Timeout { + return context.WithCancel(ctx) + } + return context.WithTimeout(ctx, c.Timeout) +} + +func (c *deadlineClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + ctx, cancel := c.bounded(ctx) + defer cancel() + return c.Client.Get(ctx, key, obj, opts...) +} +func (c *deadlineClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + ctx, cancel := c.bounded(ctx) + defer cancel() + return c.Client.List(ctx, list, opts...) +} +func (c *deadlineClient) Apply(ctx context.Context, obj k8sruntime.ApplyConfiguration, opts ...client.ApplyOption) error { + ctx, cancel := c.bounded(ctx) + defer cancel() + return c.Client.Apply(ctx, obj, opts...) +} +func (c *deadlineClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + ctx, cancel := c.bounded(ctx) + defer cancel() + return c.Client.Create(ctx, obj, opts...) +} +func (c *deadlineClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + ctx, cancel := c.bounded(ctx) + defer cancel() + return c.Client.Delete(ctx, obj, opts...) +} +func (c *deadlineClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + ctx, cancel := c.bounded(ctx) + defer cancel() + return c.Client.Update(ctx, obj, opts...) +} +func (c *deadlineClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + ctx, cancel := c.bounded(ctx) + defer cancel() + return c.Client.Patch(ctx, obj, patch, opts...) +} +func (c *deadlineClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + ctx, cancel := c.bounded(ctx) + defer cancel() + return c.Client.DeleteAllOf(ctx, obj, opts...) +} +func (c *deadlineClient) Status() client.SubResourceWriter { + return &deadlineSubresourceWriter{client: c, subresource: c.Client.Status()} +} +func (c *deadlineClient) SubResource(name string) client.SubResourceClient { + return &deadlineSubresource{client: c, subresource: c.Client.SubResource(name)} +} + +type deadlineSubresource struct { + client *deadlineClient + subresource client.SubResourceClient +} + +type deadlineSubresourceWriter struct { + client *deadlineClient + subresource client.SubResourceWriter +} + +func (s *deadlineSubresourceWriter) Create(ctx context.Context, obj client.Object, sub client.Object, opts ...client.SubResourceCreateOption) error { + ctx, cancel := s.client.bounded(ctx) + defer cancel() + return s.subresource.Create(ctx, obj, sub, opts...) +} +func (s *deadlineSubresourceWriter) Update(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error { + ctx, cancel := s.client.bounded(ctx) + defer cancel() + return s.subresource.Update(ctx, obj, opts...) +} +func (s *deadlineSubresourceWriter) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + ctx, cancel := s.client.bounded(ctx) + defer cancel() + return s.subresource.Patch(ctx, obj, patch, opts...) +} +func (s *deadlineSubresourceWriter) Apply(ctx context.Context, obj k8sruntime.ApplyConfiguration, opts ...client.SubResourceApplyOption) error { + ctx, cancel := s.client.bounded(ctx) + defer cancel() + return s.subresource.Apply(ctx, obj, opts...) +} + +func (s *deadlineSubresource) Get(ctx context.Context, obj client.Object, sub client.Object, opts ...client.SubResourceGetOption) error { + ctx, cancel := s.client.bounded(ctx) + defer cancel() + return s.subresource.Get(ctx, obj, sub, opts...) +} +func (s *deadlineSubresource) Create(ctx context.Context, obj client.Object, sub client.Object, opts ...client.SubResourceCreateOption) error { + ctx, cancel := s.client.bounded(ctx) + defer cancel() + return s.subresource.Create(ctx, obj, sub, opts...) +} +func (s *deadlineSubresource) Update(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error { + ctx, cancel := s.client.bounded(ctx) + defer cancel() + return s.subresource.Update(ctx, obj, opts...) +} +func (s *deadlineSubresource) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + ctx, cancel := s.client.bounded(ctx) + defer cancel() + return s.subresource.Patch(ctx, obj, patch, opts...) +} +func (s *deadlineSubresource) Apply(ctx context.Context, obj k8sruntime.ApplyConfiguration, opts ...client.SubResourceApplyOption) error { + ctx, cancel := s.client.bounded(ctx) + defer cancel() + return s.subresource.Apply(ctx, obj, opts...) +} + +var _ client.Client = (*deadlineClient)(nil) diff --git a/internal/internaldns/runtime/deadline_client_test.go b/internal/internaldns/runtime/deadline_client_test.go new file mode 100644 index 00000000..7d307d99 --- /dev/null +++ b/internal/internaldns/runtime/deadline_client_test.go @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package runtime + +import ( + "context" + "errors" + "testing" + "time" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type blockedSourceClient struct{ client.Client } + +func (blockedSourceClient) List(ctx context.Context, _ client.ObjectList, _ ...client.ListOption) error { + <-ctx.Done() + return ctx.Err() +} + +func TestDeadlineClientBoundsPartitionedSourceRequests(t *testing.T) { + c := &deadlineClient{Client: blockedSourceClient{}, Timeout: 25 * time.Millisecond} + started := time.Now() + err := c.List(context.Background(), nil) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("partitioned source request returned %v", err) + } + if elapsed := time.Since(started); elapsed > 250*time.Millisecond { + t.Fatalf("partitioned source request exceeded its bound: %s", elapsed) + } +} diff --git a/internal/internaldns/runtime/deployment_examples_test.go b/internal/internaldns/runtime/deployment_examples_test.go new file mode 100644 index 00000000..94ea0eeb --- /dev/null +++ b/internal/internaldns/runtime/deployment_examples_test.go @@ -0,0 +1,89 @@ +package runtime + +import ( + "bytes" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "go.miloapis.com/dns-operator/internal/internaldns/model" + k8syaml "k8s.io/apimachinery/pkg/util/yaml" +) + +func TestDeploymentExampleConfigsValidate(t *testing.T) { + t.Parallel() + root := filepath.Join("..", "..", "..", "config", "internal-dns") + cases := []struct { + file string + namePrefix string + roles []string + }{ + {file: "control-plane.example.yaml", namePrefix: "internal-dns-control-plane-config", roles: []string{"control-plane"}}, + {file: "fleet.example.yaml", namePrefix: "internal-dns-", roles: []string{"agent", "watchdog"}}, + } + for _, tc := range cases { + t.Run(tc.file, func(t *testing.T) { + contents, err := os.ReadFile(filepath.Join(root, tc.file)) + if err != nil { + t.Fatal(err) + } + decoder := k8syaml.NewYAMLOrJSONDecoder(bytes.NewReader(contents), 4096) + found := 0 + for { + var object map[string]any + if err := decoder.Decode(&object); err == io.EOF { + break + } else if err != nil { + t.Fatal(err) + } + if object["kind"] != "ConfigMap" { + continue + } + metadata, _ := object["metadata"].(map[string]any) + name, _ := metadata["name"].(string) + if !strings.HasPrefix(name, tc.namePrefix) { + continue + } + data, _ := object["data"].(map[string]any) + raw, _ := data["config.json"].(string) + if raw == "" { + continue + } + found++ + for _, role := range tc.roles { + configPath := filepath.Join(t.TempDir(), "config.json") + if err := os.WriteFile(configPath, []byte(raw), 0600); err != nil { + t.Fatal(err) + } + config, err := LoadConfig(configPath) + if err != nil { + t.Fatalf("%s config JSON: %v", name, err) + } + if err := config.Validate(role); err != nil { + t.Errorf("%s role %s: %v", name, role, err) + } + } + } + if found == 0 { + t.Fatal("no example runtime ConfigMaps found") + } + }) + } +} + +func TestNATSExampleUsesExactMemberAckSubjects(t *testing.T) { + t.Parallel() + path := filepath.Join("..", "..", "..", "config", "internal-dns", "nats-server.conf.example") + contents, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + for _, memberID := range []string{"node-us-central1-0", "regional-front-us-central1-0", "regional-bind-us-central1-0", "regional-bind-us-central1-1"} { + subject := model.AckSubject("us-central1", "shared-0", memberID) + if !bytes.Contains(contents, []byte(`"`+subject+`"`)) { + t.Errorf("NATS policy is missing exact ACK subject %q", subject) + } + } +} diff --git a/internal/internaldns/runtime/deployment_permissions_test.go b/internal/internaldns/runtime/deployment_permissions_test.go new file mode 100644 index 00000000..6945ec14 --- /dev/null +++ b/internal/internaldns/runtime/deployment_permissions_test.go @@ -0,0 +1,275 @@ +package runtime + +import ( + "bytes" + "context" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" + authorizationv1 "k8s.io/api/authorization/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + k8syaml "k8s.io/apimachinery/pkg/util/yaml" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +func deploymentObjects(t *testing.T, name string) []*unstructured.Unstructured { + t.Helper() + data, err := os.ReadFile(filepath.Join("..", "..", "..", "config", "internal-dns", name)) + if err != nil { + t.Fatal(err) + } + decoder := k8syaml.NewYAMLOrJSONDecoder(bytes.NewReader(data), 4096) + var objects []*unstructured.Unstructured + for { + var object unstructured.Unstructured + err := decoder.Decode(&object.Object) + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + if object.GetKind() != "" { + objects = append(objects, &object) + } + } + return objects +} + +// Use the Kubernetes RBAC authorizer and the actual shipped Role/ClusterRole, +// rather than accepting reconciliation under an administrator kubeconfig. +func TestDeploymentRolesAuthorizeWorkerAndDiscoveryBoundaries(t *testing.T) { + assets := os.Getenv("KUBEBUILDER_ASSETS") + if assets == "" { + t.Skip("set KUBEBUILDER_ASSETS to verify packaged roles with the Kubernetes RBAC authorizer") + } + environment := &envtest.Environment{BinaryAssetsDirectory: assets, CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, ErrorIfCRDPathMissing: true} + environment.ControlPlane.GetAPIServer().Configure().Set("authorization-mode", "RBAC") + cfg, err := environment.Start() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := environment.Stop(); err != nil { + t.Error(err) + } + }) + scheme, err := newScheme() + if err != nil { + t.Fatal(err) + } + if err := authorizationv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + cl, err := client.New(cfg, client.Options{Scheme: scheme}) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + for _, namespace := range []string{"example-project", "internal-dns-system"} { + if err := cl.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}); err != nil { + t.Fatal(err) + } + } + verifyProjectReferenceSchemas(t, ctx, cl) + for _, object := range deploymentObjects(t, "rbac.yaml") { + if err := cl.Create(ctx, object); err != nil { + t.Fatal(err) + } + } + for _, identity := range []struct{ name, role string }{{"product", "internal-dns-product-writer"}, {"issuer", "internal-dns-grant-issuer"}} { + binding := &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: identity.name + "-test", Namespace: "example-project"}, Subjects: []rbacv1.Subject{{Kind: "ServiceAccount", Name: identity.name, Namespace: "example-project"}}, RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "Role", Name: identity.role}} + if err := cl.Create(ctx, binding); err != nil { + t.Fatal(err) + } + } + cases := []struct { + principal, resource, subresource, verb string + allowed bool + }{ + {"internal-dns-control-plane", "dnsrecordsets", "", "list", true}, + {"internal-dns-control-plane", "dnsrecordsets", "", "watch", true}, + {"internal-dns-control-plane", "dnszones", "", "create", true}, + {"internal-dns-control-plane", "dnszones", "", "patch", true}, + {"internal-dns-control-plane", "dnszoneassociations", "", "create", true}, + {"internal-dns-control-plane", "dnszones", "status", "patch", true}, + {"internal-dns-control-plane", "dnszoneassociations", "status", "patch", true}, + {"internal-dns-control-plane", "dnsnamingpolicies", "status", "patch", true}, + {"internal-dns-control-plane", "dnsregistrations", "status", "patch", true}, + {"internal-dns-control-plane", "dnscontributiongrants", "status", "patch", true}, + {"internal-dns-control-plane", "dnspublicationmanifests", "", "create", true}, + {"product", "dnsresolvercontexts", "", "get", true}, + {"product", "dnsresolvercontexts", "", "list", true}, + {"product", "dnsmanagednamespaces", "", "get", true}, + {"product", "dnsmanagednamespaces", "", "list", true}, + {"product", "dnsmanagednamespaces", "", "patch", false}, + {"product", "dnsmanagednamespaces", "status", "patch", false}, + {"product", "dnsnamingpolicies", "", "get", true}, + {"product", "dnsnamingpolicies", "", "list", true}, + {"product", "dnsnamingpolicies", "", "patch", false}, + {"product", "dnsnamingpolicies", "status", "patch", false}, + {"product", "dnsregistrations", "", "create", true}, + {"product", "dnsrecordcontributions", "status", "patch", true}, + {"product", "dnscontributiongrants", "", "get", true}, + {"product", "dnscontributiongrants", "", "create", false}, + {"product", "dnscontributiongrants", "status", "patch", false}, + {"product", "dnsresolvercontexts", "", "patch", false}, + {"product", "dnsresolveraccessbindings", "", "create", false}, + {"product", "dnspublicationmanifests", "", "create", false}, + {"issuer", "dnscontributiongrants", "", "create", true}, + {"issuer", "dnsregistrations", "", "update", true}, + {"issuer", "dnscontributiongrants", "status", "patch", false}, + {"issuer", "dnsrecordcontributions", "", "create", false}, + {"issuer", "dnsresolveraccessbindings", "", "create", false}, + } + for _, tc := range cases { + namespace := "example-project" + if tc.principal == "internal-dns-control-plane" { + namespace = "internal-dns-system" + } + user := "system:serviceaccount:" + namespace + ":" + tc.principal + deadline := time.Now().Add(5 * time.Second) + for { + sar := &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{User: user, Groups: []string{"system:authenticated", "system:serviceaccounts", "system:serviceaccounts:" + namespace}, ResourceAttributes: &authorizationv1.ResourceAttributes{Namespace: "example-project", Group: "dns.networking.miloapis.com", Resource: tc.resource, Subresource: tc.subresource, Verb: tc.verb}}} + if err := cl.Create(ctx, sar); err != nil { + t.Fatal(err) + } + if sar.Status.Allowed == tc.allowed { + break + } + if time.Now().After(deadline) { + t.Fatalf("%s %s %s/%s allowed=%v want=%v: %s", user, tc.verb, tc.resource, tc.subresource, sar.Status.Allowed, tc.allowed, sar.Status.Reason) + } + time.Sleep(20 * time.Millisecond) + } + } +} + +func TestEveryMemberCanCreateAllRuntimeConsumers(t *testing.T) { + data, err := os.ReadFile(filepath.Join("..", "..", "..", "config", "internal-dns", "nats-server.conf.example")) + if err != nil { + t.Fatal(err) + } + contents := string(data) + for _, member := range []string{"node-us-central1-0", "regional-front-us-central1-0", "regional-bind-us-central1-0", "regional-bind-us-central1-1"} { + start := strings.Index(contents, `user: "`+member+`"`) + if start < 0 { + t.Fatalf("missing member %s", member) + } + policy := strings.SplitN(contents[start:], "\n },", 2)[0] + for _, suffix := range []string{"serving", "records", "acks"} { + consumer := member + "-" + suffix + for _, subject := range []string{ + "$JS.API.CONSUMER.INFO.DNS_PRIVATE." + consumer, + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE." + consumer, + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE." + consumer + ".>", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE." + consumer, + "$JS.ACK.DNS_PRIVATE." + consumer + ".>", + } { + if !strings.Contains(policy, `"`+subject+`"`) { + t.Errorf("%s lacks startup/fetch permission %s", member, subject) + } + } + } + for _, broad := range []string{`"$JS.API.>"`, `"dns.private.records.>"`, `"dns.private.serving.>"`, `"dns.private.acks.>"`} { + if strings.Contains(policy, broad) { + t.Errorf("member %s has authority to write %s", member, broad) + } + } + } +} + +func TestReadOnlyServingImageHasWritableBINDWorkingDirectory(t *testing.T) { + found := false + for _, object := range deploymentObjects(t, "fleet.example.yaml") { + if object.GetKind() != "Deployment" { + continue + } + containers, _, err := unstructured.NestedSlice(object.Object, "spec", "template", "spec", "containers") + if err != nil { + t.Fatal(err) + } + for _, value := range containers { + container := value.(map[string]interface{}) + if container["name"] != "serving" { + continue + } + found = true + mounts, _, err := unstructured.NestedSlice(container, "volumeMounts") + if err != nil { + t.Fatal(err) + } + runtimeVolume := "" + for _, value := range mounts { + mount := value.(map[string]interface{}) + if mount["mountPath"] == "/tmp" && mount["readOnly"] != true { + runtimeVolume, _ = mount["name"].(string) + } + } + if runtimeVolume == "" { + t.Fatal("read-only serving image lacks writable BIND /tmp mount") + } + volumes, _, err := unstructured.NestedSlice(object.Object, "spec", "template", "spec", "volumes") + if err != nil { + t.Fatal(err) + } + writable := false + for _, value := range volumes { + volume := value.(map[string]interface{}) + if volume["name"] == runtimeVolume && volume["emptyDir"] != nil { + writable = true + } + } + if !writable { + t.Fatal("BIND working-directory mount is not a per-member writable runtime volume") + } + } + } + if !found { + t.Fatal("no serving container was checked") + } +} + +func verifyProjectReferenceSchemas(t *testing.T, ctx context.Context, cl client.Client) { + t.Helper() + zone := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "private", Namespace: "example-project"}, Spec: dnsv1alpha1.DNSZoneSpec{DomainName: "schema.internal", Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate, DNSZoneClassName: "private"}} + if err := cl.Create(ctx, zone); err != nil { + t.Fatal(err) + } + ref := dnsv1alpha1.DNSObjectReference{Name: zone.Name, UID: zone.UID} + contextRef := dnsv1alpha1.DNSObjectReference{Name: "context", UID: "context-uid"} + cases := []struct { + name string + object client.Object + valid bool + }{ + {"registration-valid", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: ref, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 30, PublicationPolicy: dnsv1alpha1.DNSPublicationPolicyEligibleContributions}}, true}, + {"registration-unpinned", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name}, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 30, PublicationPolicy: dnsv1alpha1.DNSPublicationPolicyEligibleContributions}}, false}, + {"registration-ttl31", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: ref, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 31, PublicationPolicy: dnsv1alpha1.DNSPublicationPolicyEligibleContributions}}, false}, + {"association-valid", &dnsv1alpha1.DNSZoneAssociation{Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: ref, ResolverContextRef: contextRef}}, true}, + {"association-unpinned", &dnsv1alpha1.DNSZoneAssociation{Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name}, ResolverContextRef: contextRef}}, false}, + {"naming-valid", &dnsv1alpha1.DNSNamingPolicy{Spec: dnsv1alpha1.DNSNamingPolicySpec{ResolverContextRef: contextRef, AdditionalNames: []dnsv1alpha1.DNSAdditionalNameRule{{RegistrationClass: dnsv1alpha1.DNSRegistrationClassServiceDiscovery, DNSZoneRef: ref, NamePrefix: "services"}}}}, true}, + {"naming-unpinned", &dnsv1alpha1.DNSNamingPolicy{Spec: dnsv1alpha1.DNSNamingPolicySpec{ResolverContextRef: contextRef, AdditionalNames: []dnsv1alpha1.DNSAdditionalNameRule{{RegistrationClass: dnsv1alpha1.DNSRegistrationClassServiceDiscovery, DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name}, NamePrefix: "services"}}}}, false}, + } + for _, tc := range cases { + tc.object.SetName(tc.name) + tc.object.SetNamespace("example-project") + err := cl.Create(ctx, tc.object) + if tc.valid && err != nil { + t.Errorf("valid %s rejected: %v", tc.name, err) + } + if !tc.valid && !apierrors.IsInvalid(err) { + t.Errorf("invalid %s should fail API schema validation: %v", tc.name, err) + } + } +} diff --git a/internal/internaldns/runtime/identity_test.go b/internal/internaldns/runtime/identity_test.go new file mode 100644 index 00000000..228b1043 --- /dev/null +++ b/internal/internaldns/runtime/identity_test.go @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package runtime + +import ( + "fmt" + "os" + "os/exec" + "strings" + "testing" +) + +func TestHolderIdentityHelper(t *testing.T) { + if os.Getenv("INTERNAL_DNS_IDENTITY_HELPER") != "1" { + return + } + fmt.Printf("%s\n%s\n", holderIdentity(), holderIdentity()) +} + +func TestHolderIdentityIsStablePerProcessAndUniqueAcrossSameHostProcesses(t *testing.T) { + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + run := func() string { + cmd := exec.Command(executable, "-test.run=^TestHolderIdentityHelper$") + cmd.Env = append(os.Environ(), "INTERNAL_DNS_IDENTITY_HELPER=1") + output, err := cmd.Output() + if err != nil { + t.Fatal(err) + } + identities := strings.Fields(string(output)) + if len(identities) < 2 || identities[0] != identities[1] { + t.Fatalf("holder identity changed within one process: %q", output) + } + return identities[0] + } + first, second := run(), run() + if first == second { + t.Fatalf("same-host processes shared holder identity %q", first) + } +} diff --git a/internal/internaldns/runtime/regional_transport.go b/internal/internaldns/runtime/regional_transport.go new file mode 100644 index 00000000..f0abd3e0 --- /dev/null +++ b/internal/internaldns/runtime/regional_transport.go @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package runtime + +import ( + "context" + "fmt" + "sort" + + "go.miloapis.com/dns-operator/internal/internaldns/model" + "go.miloapis.com/dns-operator/internal/internaldns/platform" + "go.miloapis.com/dns-operator/internal/internaldns/transport" +) + +type regionalRoute struct { + bus *transport.JetStream + publisher transport.ExportPublisher + bootstrap platform.BootstrapWriter + store *transport.SnapshotStore + stream transport.StreamConfig +} + +type regionalTransport struct{ routes map[string]regionalRoute } + +func connectRegional(ctx context.Context, cfg Config, regions []string) (*regionalTransport, error) { + unique := map[string]bool{} + for _, region := range regions { + if region == "" { + return nil, fmt.Errorf("transport region is required") + } + unique[region] = true + } + ordered := make([]string, 0, len(unique)) + for region := range unique { + ordered = append(ordered, region) + } + sort.Strings(ordered) + routes := ®ionalTransport{routes: map[string]regionalRoute{}} + if len(cfg.RegionalTransports) == 0 { + if len(ordered) != 1 { + return nil, fmt.Errorf("one region per regional transport worker is required") + } + natsConfig := cfg.NATS + natsConfig.Name += "-" + model.SafeToken(ordered[0]) + bus, err := connectWithRetry(ctx, natsConfig) + if err != nil { + return nil, err + } + if cfg.EnsureStream { + if err := bus.EnsureStream(ctx, cfg.Stream); err != nil { + _ = bus.Close() + return nil, err + } + } + store, err := bus.SnapshotStore(ctx, cfg.Stream.SnapshotsBucket) + if err != nil { + _ = bus.Close() + return nil, err + } + for _, region := range ordered { + routes.routes[region] = regionalRoute{bus: bus, publisher: bus, bootstrap: store, store: store, stream: cfg.Stream} + } + return routes, nil + } + for _, region := range ordered { + nats, stream, ensure, err := cfg.transportForRegion(region) + if err != nil { + routes.Close() + return nil, err + } + bus, err := connectWithRetry(ctx, nats) + if err != nil { + routes.Close() + return nil, err + } + if ensure { + if err := bus.EnsureStream(ctx, stream); err != nil { + _ = bus.Close() + routes.Close() + return nil, err + } + } + store, err := bus.SnapshotStore(ctx, stream.SnapshotsBucket) + if err != nil { + _ = bus.Close() + routes.Close() + return nil, err + } + routes.routes[region] = regionalRoute{bus: bus, publisher: bus, bootstrap: store, store: store, stream: stream} + } + return routes, nil +} + +func (r *regionalTransport) route(region string) (regionalRoute, error) { + v, ok := r.routes[region] + if !ok { + return regionalRoute{}, fmt.Errorf("no connected transport for region %q", region) + } + return v, nil +} +func (r *regionalTransport) Publish(ctx context.Context, subject string, env model.Envelope) (transport.PublishAck, error) { + v, err := r.route(env.Region) + if err != nil { + return transport.PublishAck{}, err + } + return v.publisher.Publish(ctx, subject, env) +} +func (r *regionalTransport) Put(ctx context.Context, key string, env model.Envelope) (uint64, error) { + v, err := r.route(env.Region) + if err != nil { + return 0, err + } + return v.bootstrap.Put(ctx, key, env) +} +func (r *regionalTransport) Close() { + seen := map[*transport.JetStream]bool{} + for _, v := range r.routes { + if !seen[v.bus] { + _ = v.bus.Close() + seen[v.bus] = true + } + } +} diff --git a/internal/internaldns/runtime/regional_transport_test.go b/internal/internaldns/runtime/regional_transport_test.go new file mode 100644 index 00000000..7e636b17 --- /dev/null +++ b/internal/internaldns/runtime/regional_transport_test.go @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package runtime + +import ( + "context" + "sort" + "testing" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" + "go.miloapis.com/dns-operator/internal/internaldns/platform" + "go.miloapis.com/dns-operator/internal/internaldns/transport" +) + +type capturePublisher struct{ regions []string } + +func (p *capturePublisher) Publish(_ context.Context, _ string, e model.Envelope) (transport.PublishAck, error) { + p.regions = append(p.regions, e.Region) + return transport.PublishAck{}, nil +} + +func TestRegionalWorkersStartIndependently(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + started := make(chan string, 2) + terminal := make(chan error, 2) + planners := []platform.PlannerConfig{{Region: "east", Shard: "one"}, {Region: "west", Shard: "two"}} + startRegionalWorkers(planners, func(region string, _ []platform.PlannerConfig) error { + started <- region + <-ctx.Done() + return nil + }, terminal) + var regions []string + deadline := time.After(time.Second) + for len(regions) < 2 { + select { + case region := <-started: + regions = append(regions, region) + case <-deadline: + t.Fatalf("a blocked region prevented another worker from starting: %v", regions) + } + } + sort.Strings(regions) + if regions[0] != "east" || regions[1] != "west" { + t.Fatalf("unexpected regional workers: %v", regions) + } +} + +type captureBootstrap struct{ regions []string } + +func (b *captureBootstrap) Put(_ context.Context, _ string, e model.Envelope) (uint64, error) { + b.regions = append(b.regions, e.Region) + return 1, nil +} + +func TestRegionalTransportRoutesWithoutCrossRegionFallback(t *testing.T) { + eastPub, westPub := &capturePublisher{}, &capturePublisher{} + eastStore, westStore := &captureBootstrap{}, &captureBootstrap{} + r := ®ionalTransport{routes: map[string]regionalRoute{"east": {publisher: eastPub, bootstrap: eastStore}, "west": {publisher: westPub, bootstrap: westStore}}} + for _, region := range []string{"east", "west"} { + env, err := model.NewEnvelope(model.KindServingSnapshot, "event-"+region, region, "one", "resource", 1, 1, time.Now(), map[string]string{"region": region}) + if err != nil { + t.Fatal(err) + } + if _, err := r.Publish(context.Background(), "subject", env); err != nil { + t.Fatal(err) + } + if _, err := r.Put(context.Background(), "key", env); err != nil { + t.Fatal(err) + } + } + if len(eastPub.regions) != 1 || eastPub.regions[0] != "east" || len(westPub.regions) != 1 || westPub.regions[0] != "west" { + t.Fatalf("publish crossed regions: east=%v west=%v", eastPub.regions, westPub.regions) + } + if len(eastStore.regions) != 1 || len(westStore.regions) != 1 { + t.Fatalf("bootstrap crossed regions: east=%v west=%v", eastStore.regions, westStore.regions) + } + unknown, _ := model.NewEnvelope(model.KindServingSnapshot, "event-unknown", "north", "one", "resource", 1, 1, time.Now(), map[string]string{"region": "north"}) + if _, err := r.Publish(context.Background(), "subject", unknown); err == nil { + t.Fatal("unknown region silently fell back to another transport") + } +} diff --git a/internal/internaldns/runtime/routing_client.go b/internal/internaldns/runtime/routing_client.go new file mode 100644 index 00000000..38211767 --- /dev/null +++ b/internal/internaldns/runtime/routing_client.go @@ -0,0 +1,85 @@ +package runtime + +import ( + "context" + + k8sruntime "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// RoutingClient keeps project resources in their source API server while +// storing serving artifacts in the durable platform control plane. It never +// mirrors or rewrites API-assigned UIDs and generations. +type RoutingClient struct { + client.Client + Source client.Client + PlatformNamespace string +} + +func (c *RoutingClient) target(obj client.Object) client.Client { + if obj != nil && obj.GetNamespace() == c.PlatformNamespace { + return c.Client + } + return c.Source +} +func (c *RoutingClient) targetNamespace(namespace string) client.Client { + if namespace == c.PlatformNamespace { + return c.Client + } + return c.Source +} +func (c *RoutingClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + return c.targetNamespace(key.Namespace).Get(ctx, key, obj, opts...) +} +func (c *RoutingClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + o := (&client.ListOptions{}).ApplyOptions(opts) + return c.targetNamespace(o.Namespace).List(ctx, list, opts...) +} +func (c *RoutingClient) Apply(ctx context.Context, obj k8sruntime.ApplyConfiguration, opts ...client.ApplyOption) error { + return c.Client.Apply(ctx, obj, opts...) +} +func (c *RoutingClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + return c.target(obj).Create(ctx, obj, opts...) +} +func (c *RoutingClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + return c.target(obj).Delete(ctx, obj, opts...) +} +func (c *RoutingClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + return c.target(obj).Update(ctx, obj, opts...) +} +func (c *RoutingClient) Patch(ctx context.Context, obj client.Object, p client.Patch, opts ...client.PatchOption) error { + return c.target(obj).Patch(ctx, obj, p, opts...) +} +func (c *RoutingClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + o := (&client.DeleteAllOfOptions{}).ApplyOptions(opts) + return c.targetNamespace(o.Namespace).DeleteAllOf(ctx, obj, opts...) +} +func (c *RoutingClient) Status() client.SubResourceWriter { + return &routingSubresource{parent: c, name: "status"} +} +func (c *RoutingClient) SubResource(name string) client.SubResourceClient { + return &routingSubresource{parent: c, name: name} +} + +type routingSubresource struct { + parent *RoutingClient + name string +} + +func (s *routingSubresource) Get(ctx context.Context, obj client.Object, sub client.Object, opts ...client.SubResourceGetOption) error { + return s.parent.target(obj).SubResource(s.name).Get(ctx, obj, sub, opts...) +} +func (s *routingSubresource) Create(ctx context.Context, obj client.Object, sub client.Object, opts ...client.SubResourceCreateOption) error { + return s.parent.target(obj).SubResource(s.name).Create(ctx, obj, sub, opts...) +} +func (s *routingSubresource) Update(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error { + return s.parent.target(obj).SubResource(s.name).Update(ctx, obj, opts...) +} +func (s *routingSubresource) Patch(ctx context.Context, obj client.Object, p client.Patch, opts ...client.SubResourcePatchOption) error { + return s.parent.target(obj).SubResource(s.name).Patch(ctx, obj, p, opts...) +} +func (s *routingSubresource) Apply(ctx context.Context, obj k8sruntime.ApplyConfiguration, opts ...client.SubResourceApplyOption) error { + return s.parent.Client.SubResource(s.name).Apply(ctx, obj, opts...) +} + +var _ client.Client = (*RoutingClient)(nil) diff --git a/internal/internaldns/runtime/routing_client_test.go b/internal/internaldns/runtime/routing_client_test.go new file mode 100644 index 00000000..a9363dd0 --- /dev/null +++ b/internal/internaldns/runtime/routing_client_test.go @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package runtime + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestRoutingClientKeepsSourceAndPlatformObjectsSeparate(t *testing.T) { + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + source := fake.NewClientBuilder().WithScheme(scheme).Build() + platform := fake.NewClientBuilder().WithScheme(scheme).Build() + routed := &RoutingClient{Client: platform, Source: source, PlatformNamespace: "internal-dns-system"} + ctx := context.Background() + projectObject := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "input", Namespace: "project-a"}} + platformObject := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "manifest", Namespace: "internal-dns-system"}} + if err := routed.Create(ctx, projectObject); err != nil { + t.Fatal(err) + } + if err := routed.Create(ctx, platformObject); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + cl client.Client + key client.ObjectKey + found bool + }{ + {"project in source", source, client.ObjectKeyFromObject(projectObject), true}, + {"project absent from platform", platform, client.ObjectKeyFromObject(projectObject), false}, + {"artifact in platform", platform, client.ObjectKeyFromObject(platformObject), true}, + {"artifact absent from source", source, client.ObjectKeyFromObject(platformObject), false}, + } { + t.Run(tc.name, func(t *testing.T) { + err := tc.cl.Get(ctx, tc.key, &corev1.ConfigMap{}) + if (err == nil) != tc.found { + t.Fatalf("found=%v want=%v err=%v", err == nil, tc.found, err) + } + }) + } +} diff --git a/internal/internaldns/runtime/run.go b/internal/internaldns/runtime/run.go new file mode 100644 index 00000000..6a7be693 --- /dev/null +++ b/internal/internaldns/runtime/run.go @@ -0,0 +1,546 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +// Package runtime wires the internal DNS control plane and fleet processes. +package runtime + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "os" + "strings" + "sync" + "time" + + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" + dnsadmission "go.miloapis.com/dns-operator/internal/internaldns/admission" + "go.miloapis.com/dns-operator/internal/internaldns/controlplane" + "go.miloapis.com/dns-operator/internal/internaldns/model" + "go.miloapis.com/dns-operator/internal/internaldns/platform" + "go.miloapis.com/dns-operator/internal/internaldns/serving" + "go.miloapis.com/dns-operator/internal/internaldns/transport" + "go.miloapis.com/dns-operator/internal/internaldns/watchdog" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/util/flowcontrol" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" + kubeadmission "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +const parentNameExtra = "iam.miloapis.com/parent-name" + +// Run starts one of the deliberately separate production roles. Agent and +// watchdog never load Kubernetes credentials. +func Run(ctx context.Context, role string, cfg Config, kubeconfig string) error { + if err := cfg.Validate(role); err != nil { + return err + } + switch role { + case roleControlPlane: + return runControlPlane(ctx, cfg, kubeconfig) + case roleAgent: + return runAgent(ctx, cfg) + case roleWatchdog: + return (&watchdog.Watchdog{Config: cfg.Watchdog}).Start(ctx) + default: + return fmt.Errorf("unsupported role %q", role) + } +} + +type projectRuntime struct { + config ProjectConfig + direct client.Client + start func(context.Context) error +} + +func runControlPlane(ctx context.Context, cfg Config, kubeconfig string) error { + scheme, err := newScheme() + if err != nil { + return err + } + platformREST, err := restConfig(kubeconfig) + if err != nil { + return fmt.Errorf("load platform kubeconfig: %w", err) + } + applyKubeAPILimit(platformREST, cfg.PlatformKubeAPI) + platformClient, err := client.New(platformREST, client.Options{Scheme: scheme}) + if err != nil { + return fmt.Errorf("create uncached platform client: %w", err) + } + platformClient = &deadlineClient{Client: platformClient, Timeout: cfg.PlatformKubeAPI.requestTimeout()} + allocator := &platform.Allocator{Client: platformClient, Namespace: cfg.PlatformNamespace, ConsumerPrefix: cfg.ConsumerPrefix, ClusterPrefix: cfg.ClusterPrefix} + planners := cfg.plannerConfigs() + regions := make([]controlplane.ServingRegion, 0, len(planners)) + for _, p := range planners { + regions = append(regions, controlplane.ServingRegion{Region: p.Region, Shard: p.Shard}) + } + + identity := holderIdentity() + projects := make([]projectRuntime, 0, len(cfg.Projects)) + for _, p := range cfg.Projects { + projectREST := rest.CopyConfig(platformREST) + if p.Kubeconfig != "" { + projectREST, err = restConfig(p.Kubeconfig) + if err != nil { + return fmt.Errorf("load project %q kubeconfig: %w", p.Name, err) + } + } + applyKubeAPILimit(projectREST, p.KubeAPI) + direct, err := client.New(projectREST, client.Options{Scheme: scheme}) + if err != nil { + return fmt.Errorf("create project %q direct client: %w", p.Name, err) + } + direct = &deadlineClient{Client: direct, Timeout: p.KubeAPI.requestTimeout()} + project := p + start := func(startCtx context.Context) error { + mgr, err := ctrl.NewManager(projectREST, ctrl.Options{ + Scheme: scheme, + Cache: cache.Options{DefaultNamespaces: map[string]cache.Config{project.Namespace: {}}}, + Metrics: metricsserver.Options{BindAddress: "0"}, + HealthProbeBindAddress: "0", + LeaderElection: false, + }) + if err != nil { + return fmt.Errorf("create manager: %w", err) + } + // Manager caches deliver watch events only. Every reconciliation read goes + // to the source API so a partition cannot renew authorization from stale + // contexts, access bindings, associations, registrations, or producer observations. + routed := &RoutingClient{Client: platformClient, Source: direct, PlatformNamespace: cfg.PlatformNamespace} + reconciler := &controlplane.Reconciler{Client: routed, Scheme: scheme, Options: controlplane.ReconcilerOptions{ + ProjectUID: project.ProjectUID, SourceClusterUID: project.SourceClusterUID, + ProjectNamespace: project.Namespace, PlatformNamespace: cfg.PlatformNamespace, + Region: cfg.Region, Shard: cfg.Shard, Identity: identity, + LeaseDuration: cfg.ownershipLease(), + AddressAllocator: allocator, ManagedDomainSuffix: cfg.ManagedDomainSuffix, + PrivateZoneClassName: cfg.PrivateZoneClassName, ServingRegions: regions, + MaxAccessLease: cfg.Admission.maxAccessLease(), + }} + if err := reconciler.SetupWithManager(mgr); err != nil { + return fmt.Errorf("setup reconciler: %w", err) + } + return mgr.Start(startCtx) + } + projects = append(projects, projectRuntime{config: project, direct: direct, start: start}) + } + + var admissionManager ctrl.Manager + if cfg.Admission.Enabled { + admissionManager, err = ctrl.NewManager(platformREST, ctrl.Options{ + Scheme: scheme, + Metrics: metricsserver.Options{BindAddress: "0"}, + HealthProbeBindAddress: "0", + LeaderElection: false, + WebhookServer: webhook.NewServer(webhook.Options{Port: cfg.Admission.Port, CertDir: cfg.Admission.CertDir}), + }) + if err != nil { + return fmt.Errorf("create admission manager: %w", err) + } + handler := &dnsadmission.Handler{ + PlatformSubjects: cfg.Admission.PlatformSubjects, + IntegrationSubjects: cfg.Admission.IntegrationSubjects, + MaxContributionLease: cfg.Admission.maxLease(), + MaxAccessLease: cfg.Admission.maxAccessLease(), + ResolverContextsEnabled: true, + ClientForRequest: projectSelector(projects), + } + admissionManager.GetWebhookServer().Register(dnsadmission.Path, &kubeadmission.Webhook{Handler: handler}) + } + + errCh := make(chan error, len(projects)+len(planners)*3+8) + startProjectSources(ctx, projects) + if admissionManager != nil { + go func() { errCh <- admissionManager.Start(ctx) }() + } + + publicationMembers := regionalMembers(planners) + publicationRegions := map[string][]model.Member{} + for _, planner := range planners { + publicationRegions[planner.Region+"/"+planner.Shard] = append([]model.Member(nil), planner.Members...) + } + projectScopes := make([]platform.ProjectScope, 0, len(projects)) + for _, p := range projects { + projectScopes = append(projectScopes, platform.ProjectScope{Client: p.direct, Namespace: p.config.Namespace, RequestTimeout: p.config.KubeAPI.requestTimeout()}) + } + sinks := map[string]*platform.AckSink{} + for _, plannerCfg := range planners { + pc := plannerCfg + planner := &platform.Planner{Client: platformClient, Allocator: allocator, Config: pc} + interval := pc.LeaseDuration / 3 + if interval < time.Second { + interval = time.Second + } + go retryLoop(ctx, interval, "planner "+pc.Region+"/"+pc.Shard, planner.Step, errCh) + sink := &platform.AckSink{Client: platformClient, Namespace: cfg.PlatformNamespace, Projects: projectScopes, + PublicationMembers: publicationMembers, PublicationRegions: publicationRegions, Region: pc.Region, Shard: pc.Shard, Members: pc.Members} + sinks[pc.Region+"/"+pc.Shard] = sink + go retryLoop(ctx, 2*time.Second, "ack refresh "+pc.Region+"/"+pc.Shard, sink.Refresh, errCh) + } + startRegionalWorkers(planners, func(region string, regionalPlanners []platform.PlannerConfig) error { + return runControlPlaneRegion(ctx, cfg, region, regionalPlanners, platformClient, sinks) + }, errCh) + + select { + case <-ctx.Done(): + return nil + case err := <-errCh: + if err == nil || errors.Is(err, context.Canceled) { + return nil + } + return err + } +} + +func startProjectSources(ctx context.Context, projects []projectRuntime) { + for _, project := range projects { + go runProjectSource(ctx, project) + } +} + +func runProjectSource(ctx context.Context, project projectRuntime) { + name := project.config.Name + if name == "" { + name = string(project.config.ProjectUID) + } + log := ctrl.LoggerFrom(ctx).WithName("internal-dns-project-source").WithValues("project", name) + for ctx.Err() == nil { + err := project.start(ctx) + if ctx.Err() != nil { + return + } + if err == nil { + err = errors.New("source manager stopped unexpectedly") + } + log.Error(err, "source manager unavailable; rebuilding and retrying") + if !waitForRetry(ctx, time.Second) { + return + } + } +} + +// startRegionalWorkers starts every transport domain independently. In +// particular, it does not wait for one region's initial NATS connection before +// starting source controllers, planners, ACK expiry, or another region. +func startRegionalWorkers(planners []platform.PlannerConfig, run func(string, []platform.PlannerConfig) error, terminal chan<- error) { + byRegion := map[string][]platform.PlannerConfig{} + for _, planner := range planners { + byRegion[planner.Region] = append(byRegion[planner.Region], planner) + } + for region, regionalPlanners := range byRegion { + go func() { terminal <- run(region, regionalPlanners) }() + } +} + +func runControlPlaneRegion(ctx context.Context, cfg Config, region string, planners []platform.PlannerConfig, platformClient client.Client, sinks map[string]*platform.AckSink) error { + log := ctrl.LoggerFrom(ctx).WithName("internal-dns-region").WithValues("region", region) + for ctx.Err() == nil { + regional, err := connectRegional(ctx, cfg, []string{region}) + if err != nil { + if ctx.Err() != nil { + return nil + } + log.Error(err, "regional transport setup failed; retrying") + if !waitForRetry(ctx, time.Second) { + return nil + } + continue + } + route, err := regional.route(region) + if err != nil { + regional.Close() + return err + } + consumers := make(map[string]*transport.Consumer, len(planners)) + setupFailed := false + for _, planner := range planners { + consumer, consumerErr := route.bus.Consumer(ctx, transport.ConsumerConfig{ + Stream: route.stream.Name, + Durable: durable(roleControlPlane, planner.Region, planner.Shard, "acks"), + FilterSubjects: []string{ackFilter(planner.Region, planner.Shard)}, + AckWait: 30 * time.Second, MaxAckPending: 128, + }) + if consumerErr != nil { + log.Error(consumerErr, "regional ACK consumer setup failed; retrying", "shard", planner.Shard) + setupFailed = true + break + } + consumers[planner.Region+"/"+planner.Shard] = consumer + } + if setupFailed { + regional.Close() + if !waitForRetry(ctx, time.Second) { + return nil + } + continue + } + workerCtx, cancel := context.WithCancel(ctx) + localDone := make(chan error, len(consumers)+1) + outbox := &platform.Outbox{Client: platformClient, Namespace: cfg.PlatformNamespace, Region: region, Publisher: route.publisher, Bootstrap: route.bootstrap, PendingOnly: true} + go retryLoop(workerCtx, 500*time.Millisecond, "outbox "+region, outbox.Step, localDone) + for key, consumer := range consumers { + go consumeAcks(workerCtx, consumer, sinks[key], localDone) + } + <-ctx.Done() + cancel() + regional.Close() + return nil + } + return nil +} + +func waitForRetry(ctx context.Context, delay time.Duration) bool { + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true + } +} + +func runAgent(ctx context.Context, cfg Config) error { + regional, err := connectRegional(ctx, cfg, []string{cfg.Agent.Region}) + if err != nil { + return err + } + defer regional.Close() + route, _ := regional.route(cfg.Agent.Region) + consumer := func(suffix string, filters ...string) (*transport.Consumer, error) { + return route.bus.Consumer(ctx, transport.ConsumerConfig{Stream: route.stream.Name, + Durable: durable(cfg.Agent.MemberID, suffix), FilterSubjects: filters, + AckWait: 30 * time.Second, MaxAckPending: 64}) + } + servingSource, err := consumer("serving", model.ServingSubject(cfg.Agent.Region, cfg.Agent.Shard)) + if err != nil { + return err + } + var recordSource, ackSource *transport.Consumer + if cfg.Agent.Mode == serving.ModeResolver || cfg.Agent.Mode == serving.ModeCombined { + recordSource, err = consumer("records", model.RecordConsumerFilter(cfg.Agent.Region, cfg.Agent.Shard)) + if err != nil { + return err + } + } + if cfg.Agent.Mode == serving.ModeResolver || cfg.Agent.Mode == serving.ModeCombined { + ackSource, err = consumer("acks", ackFilter(cfg.Agent.Region, cfg.Agent.Shard)) + if err != nil { + return err + } + } + log := ctrl.LoggerFrom(ctx).WithName("internal-dns-agent") + agent, err := serving.NewAgent(cfg.Agent, serving.Dependencies{ + ServingSource: servingSource, RecordSource: recordSource, AckSource: ackSource, + Bootstrap: serving.SnapshotBootstrap{Store: route.store, Region: cfg.Agent.Region, Shard: cfg.Agent.Shard}, + AckPublisher: route.bus, + OnError: func(err error) { log.Error(err, "serving agent operation failed; retrying") }, + }) + if err != nil { + return err + } + return agent.Start(ctx) +} + +func newScheme() (*runtime.Scheme, error) { + scheme := runtime.NewScheme() + if err := clientgoscheme.AddToScheme(scheme); err != nil { + return nil, err + } + if err := corev1.AddToScheme(scheme); err != nil { + return nil, err + } + if err := dnsv1alpha1.AddToScheme(scheme); err != nil { + return nil, err + } + return scheme, nil +} + +func restConfig(path string) (*rest.Config, error) { + if path == "" { + return ctrl.GetConfig() + } + if !strings.HasPrefix(path, "/") { + return nil, fmt.Errorf("kubeconfig path must be absolute: %s", path) + } + return clientcmd.BuildConfigFromFlags("", path) +} + +// applyKubeAPILimit installs one token bucket on the REST config before that +// config is shared by a manager and its uncached client. client-go otherwise +// creates a separate default 5 QPS limiter for every constructed client. +func applyKubeAPILimit(c *rest.Config, limits KubeAPIClientConfig) { + c.QPS = limits.QPS + c.Burst = limits.Burst + c.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(limits.QPS, limits.Burst) +} + +func connectWithRetry(ctx context.Context, cfg transport.Config) (*transport.JetStream, error) { + log := ctrl.LoggerFrom(ctx).WithName("nats") + for { + js, err := transport.Connect(cfg) + if err == nil { + return js, nil + } + log.Error(err, "NATS unavailable; retrying") + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(time.Second): + } + } +} + +func retryLoop(ctx context.Context, interval time.Duration, name string, step func(context.Context) error, terminal chan<- error) { + log := ctrl.LoggerFrom(ctx).WithName("internal-dns").WithValues("loop", name) + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + if err := step(ctx); err != nil && !errors.Is(err, context.Canceled) { + log.Error(err, "operation failed; retrying") + } + select { + case <-ctx.Done(): + terminal <- nil + return + case <-ticker.C: + } + } +} + +func consumeAcks(ctx context.Context, source *transport.Consumer, sink *platform.AckSink, terminal chan<- error) { + log := ctrl.LoggerFrom(ctx).WithName("internal-dns-ack-consumer").WithValues("region", sink.Region, "shard", sink.Shard) + for { + message, err := source.Next(ctx) + if err != nil { + if ctx.Err() != nil { + terminal <- nil + return + } + log.Error(err, "read acknowledgement failed; retrying") + time.Sleep(time.Second) + continue + } + if err := sink.Apply(ctx, message.Subject(), message.Envelope()); err != nil { + log.Error(err, "apply acknowledgement failed; retrying") + _ = message.Retry(time.Second) + continue + } + if err := message.Ack(ctx); err != nil { + log.Error(err, "acknowledgement commit was ambiguous; awaiting redelivery") + } + } +} + +func projectSelector(projects []projectRuntime) func(context.Context, kubeadmission.Request) (client.Client, string, error) { + return func(_ context.Context, req kubeadmission.Request) (client.Client, string, error) { + parents := req.UserInfo.Extra[parentNameExtra] + if len(parents) == 0 && len(projects) == 1 { + p := projects[0] + if req.Namespace != p.config.Namespace { + return nil, "", fmt.Errorf("request namespace is outside the configured project") + } + return p.direct, p.config.SourceClusterUID, nil + } + if len(parents) != 1 { + return nil, "", fmt.Errorf("authenticated project parent is missing or ambiguous") + } + for _, p := range projects { + if p.config.Name == parents[0] && req.Namespace == p.config.Namespace { + return p.direct, p.config.SourceClusterUID, nil + } + } + return nil, "", fmt.Errorf("authenticated project parent is not configured for this namespace") + } +} + +func (c Config) plannerConfigs() []platform.PlannerConfig { + configs := append([]platform.PlannerConfig(nil), c.Shards...) + if len(configs) == 0 { + configs = []platform.PlannerConfig{{Region: c.Region, Shard: c.Shard}} + } + for i := range configs { + p := &configs[i] + if p.Namespace == "" { + p.Namespace = c.PlatformNamespace + } + if p.Identity == "" { + p.Identity = holderIdentity() + "-" + model.SafeToken(p.Region) + "-" + model.SafeToken(p.Shard) + } + if p.LeaseDuration <= 0 { + p.LeaseDuration = c.ownershipLease() + } + if len(p.NodeBackends) == 0 { + p.NodeBackends = append([]model.Backend(nil), c.NodeBackends...) + } + if len(p.ClusterBackends) == 0 { + p.ClusterBackends = append([]model.Backend(nil), c.ClusterBackends...) + } + + if len(p.Members) == 0 { + p.Members = append([]model.Member(nil), c.Members...) + } + } + return configs +} + +func regionalMembers(configs []platform.PlannerConfig) []model.Member { + var result []model.Member + for _, p := range configs { + for _, m := range p.Members { + if m.Role == memberRoleRegional || m.Role == memberRoleCluster { + result = append(result, m) + } + } + } + return result +} + +func ackFilter(region, shard string) string { + return "dns.private.acks." + model.SafeToken(region) + "." + model.SafeToken(shard) + ".>" +} + +func durable(parts ...string) string { + for i := range parts { + parts[i] = model.SafeToken(parts[i]) + } + return strings.Join(parts, "-") +} + +var ( + processIdentityOnce sync.Once + processIdentity string +) + +func holderIdentity() string { + processIdentityOnce.Do(func() { + processIdentity = newProcessIdentity() + }) + return processIdentity +} + +// newProcessIdentity distinguishes concurrently running control-plane +// processes even when they share a host. holderIdentity caches one value for +// the process lifetime so every project reconciler and shard planner in this +// process uses the same stable lease identity. +func newProcessIdentity() string { + host, _ := os.Hostname() + if host == "" { + host = "internal-dns" + } + random := make([]byte, 16) + if _, err := rand.Read(random); err == nil { + return host + "-" + hex.EncodeToString(random) + } + // crypto/rand failure is exceptional. PID and startup time still prevent + // same-host processes from silently sharing a lease identity. + return fmt.Sprintf("%s-%d-%d", host, os.Getpid(), time.Now().UTC().UnixNano()) +} From 75ecde92a790ab024a555f5273af27af7cc89c76 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 16:35:50 -0500 Subject: [PATCH 02/23] Use scoped API roles for private DNS service access --- config/internal-dns/README.md | 17 +++++++++++++++-- config/internal-dns/control-plane.example.yaml | 2 -- config/internal-dns/rbac.yaml | 8 ++------ internal/internaldns/runtime/config.go | 12 +++++------- .../runtime/deployment_permissions_test.go | 10 ++++++++-- internal/internaldns/runtime/run.go | 2 -- 6 files changed, 30 insertions(+), 21 deletions(-) diff --git a/config/internal-dns/README.md b/config/internal-dns/README.md index 1be96293..307f97d3 100644 --- a/config/internal-dns/README.md +++ b/config/internal-dns/README.md @@ -34,8 +34,10 @@ projection or publication for healthy sources. DNS always consumes `DNSResolverContext` and `DNSResolverAccessBinding` from the project API. It does not watch a networking API. A trusted integration -principal in `admission.integrationSubjects` creates a context and writes -short-lived regional access intent. Product publishers cannot authorize access. +principal bound to `internal-dns-vpc-integration` creates a context and writes +short-lived regional access intent. RBAC separates integration, grant issuer, +product publisher, and DNS controller permissions. Product publishers cannot +issue grants or modify resolver authority. The context UID, access UID, authorization writer epoch, sequence, and original deadline are carried into the internal `DNSResolverBinding`. @@ -138,3 +140,14 @@ configurations plus one Deployment template. Instantiate the template for each member and create a second regional BIND member with distinct addresses and storage. `control-plane.example.yaml` lists both regional members in `clusterBackends` and `members`. + +## Resource access + +The project API server authorizes DNS resource and status operations. Bind the +roles in `rbac.yaml` to service principals in each project namespace. Customer +access uses the protected resources and DNS roles in `config/iam`. + +The webhook validates live references and lease limits. It also verifies that +changes to a publisher's contribution and freshness observation use that +publisher's current grant. Publication-result updates cannot renew a publisher's +observation. Resource permissions do not depend on webhook principal lists. diff --git a/config/internal-dns/control-plane.example.yaml b/config/internal-dns/control-plane.example.yaml index e9efb835..75e81ed3 100644 --- a/config/internal-dns/control-plane.example.yaml +++ b/config/internal-dns/control-plane.example.yaml @@ -65,8 +65,6 @@ data: "enabled": true, "port": 9443, "certDir": "/var/run/internal-dns/admission", - "platformSubjects": ["system:serviceaccount:internal-dns-system:internal-dns-control-plane"], - "integrationSubjects": ["system:serviceaccount:network-system:dns-vpc-integration"], "maxContributionLeaseSeconds": 90, "maxAccessLeaseSeconds": 300 } diff --git a/config/internal-dns/rbac.yaml b/config/internal-dns/rbac.yaml index 140db19e..4fee5534 100644 --- a/config/internal-dns/rbac.yaml +++ b/config/internal-dns/rbac.yaml @@ -26,7 +26,7 @@ rules: verbs: [get, list, watch] --- # Project products write declarations and their own freshness observations. -# Platform artifacts and every other status subresource are deliberately absent. +# Product writers cannot modify grants, resolver access, or publication artifacts. apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -52,8 +52,7 @@ metadata: rules: - apiGroups: [dns.networking.miloapis.com] resources: [dnsregistrations] - # Admission authorizes grant issuance against the exact registration's update permission. - verbs: [get, update] + verbs: [get] - apiGroups: [dns.networking.miloapis.com] resources: [dnscontributiongrants] verbs: [get, list, watch, create, update, patch, delete] @@ -104,9 +103,6 @@ rules: - dnsresolvercontexts/status - dnsresolveraccessbindings/status verbs: [get, update, patch] -- apiGroups: [authorization.k8s.io] - resources: [subjectaccessreviews] - verbs: [create] - apiGroups: [coordination.k8s.io] resources: [leases] verbs: [get, list, watch, create, update, patch] diff --git a/internal/internaldns/runtime/config.go b/internal/internaldns/runtime/config.go index a381c093..44cfd98d 100644 --- a/internal/internaldns/runtime/config.go +++ b/internal/internaldns/runtime/config.go @@ -42,13 +42,11 @@ type KubeAPIClientConfig struct { } type AdmissionConfig struct { - Enabled bool `json:"enabled"` - Port int `json:"port,omitempty"` - CertDir string `json:"certDir,omitempty"` - PlatformSubjects []string `json:"platformSubjects,omitempty"` - MaxContributionLeaseSeconds int64 `json:"maxContributionLeaseSeconds,omitempty"` - IntegrationSubjects []string `json:"integrationSubjects,omitempty"` - MaxAccessLeaseSeconds int64 `json:"maxAccessLeaseSeconds,omitempty"` + Enabled bool `json:"enabled"` + Port int `json:"port,omitempty"` + CertDir string `json:"certDir,omitempty"` + MaxContributionLeaseSeconds int64 `json:"maxContributionLeaseSeconds,omitempty"` + MaxAccessLeaseSeconds int64 `json:"maxAccessLeaseSeconds,omitempty"` } type RegionalTransportConfig struct { diff --git a/internal/internaldns/runtime/deployment_permissions_test.go b/internal/internaldns/runtime/deployment_permissions_test.go index 6945ec14..ec08846e 100644 --- a/internal/internaldns/runtime/deployment_permissions_test.go +++ b/internal/internaldns/runtime/deployment_permissions_test.go @@ -87,7 +87,7 @@ func TestDeploymentRolesAuthorizeWorkerAndDiscoveryBoundaries(t *testing.T) { t.Fatal(err) } } - for _, identity := range []struct{ name, role string }{{"product", "internal-dns-product-writer"}, {"issuer", "internal-dns-grant-issuer"}} { + for _, identity := range []struct{ name, role string }{{"product", "internal-dns-product-writer"}, {"issuer", "internal-dns-grant-issuer"}, {"integration", "internal-dns-vpc-integration"}} { binding := &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: identity.name + "-test", Namespace: "example-project"}, Subjects: []rbacv1.Subject{{Kind: "ServiceAccount", Name: identity.name, Namespace: "example-project"}}, RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "Role", Name: identity.role}} if err := cl.Create(ctx, binding); err != nil { t.Fatal(err) @@ -126,8 +126,14 @@ func TestDeploymentRolesAuthorizeWorkerAndDiscoveryBoundaries(t *testing.T) { {"product", "dnsresolvercontexts", "", "patch", false}, {"product", "dnsresolveraccessbindings", "", "create", false}, {"product", "dnspublicationmanifests", "", "create", false}, + {"integration", "dnsresolvercontexts", "", "create", true}, + {"integration", "dnsresolveraccessbindings", "", "create", true}, + {"integration", "dnsresolvercontexts", "status", "patch", false}, + {"integration", "dnscontributiongrants", "", "create", false}, + {"integration", "dnspublicationmanifests", "", "create", false}, {"issuer", "dnscontributiongrants", "", "create", true}, - {"issuer", "dnsregistrations", "", "update", true}, + + {"issuer", "dnsregistrations", "", "update", false}, {"issuer", "dnscontributiongrants", "status", "patch", false}, {"issuer", "dnsrecordcontributions", "", "create", false}, {"issuer", "dnsresolveraccessbindings", "", "create", false}, diff --git a/internal/internaldns/runtime/run.go b/internal/internaldns/runtime/run.go index 6a7be693..194d7818 100644 --- a/internal/internaldns/runtime/run.go +++ b/internal/internaldns/runtime/run.go @@ -146,8 +146,6 @@ func runControlPlane(ctx context.Context, cfg Config, kubeconfig string) error { return fmt.Errorf("create admission manager: %w", err) } handler := &dnsadmission.Handler{ - PlatformSubjects: cfg.Admission.PlatformSubjects, - IntegrationSubjects: cfg.Admission.IntegrationSubjects, MaxContributionLease: cfg.Admission.maxLease(), MaxAccessLease: cfg.Admission.maxAccessLease(), ResolverContextsEnabled: true, From 04aedab0744b8f28698752ce75629b40ffd50d52 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 16:37:04 -0500 Subject: [PATCH 03/23] Scope admission to DNS relationships and publisher freshness --- config/internal-dns/admission.example.yaml | 38 +++------------------- 1 file changed, 5 insertions(+), 33 deletions(-) diff --git a/config/internal-dns/admission.example.yaml b/config/internal-dns/admission.example.yaml index be700bbe..7c1ac20c 100644 --- a/config/internal-dns/admission.example.yaml +++ b/config/internal-dns/admission.example.yaml @@ -5,7 +5,7 @@ metadata: annotations: cert-manager.io/inject-ca-from: internal-dns-system/internal-dns-admission-tls webhooks: -- name: authorization.internal-dns.miloapis.com +- name: validation.internal-dns.miloapis.com admissionReviewVersions: [v1] sideEffects: None failurePolicy: Fail @@ -18,50 +18,22 @@ webhooks: path: /validate-internal-dns port: 443 rules: - # Product declarations and the private-visibility guard for DNSZone. The - # legacy DNSRecordSet API is intentionally outside this webhook. + # Live DNS references and bounded access leases require service validation. + # The API server authorizes resource and status operations. - apiGroups: [dns.networking.miloapis.com] apiVersions: [v1alpha1] - operations: [CREATE, UPDATE, DELETE] + operations: [CREATE, UPDATE] resources: - - dnszones - dnszoneassociations - dnsregistrations - dnscontributiongrants - dnsrecordcontributions - dnsnamingpolicies - - dnsresolvercontexts - dnsresolveraccessbindings scope: Namespaced - # A product can report only its own contribution freshness. No - # dnsrecordsets/status rule is installed, preserving the public operator. + # Freshness observations must match the authenticated publisher's live grant. - apiGroups: [dns.networking.miloapis.com] apiVersions: [v1alpha1] operations: [UPDATE] resources: [dnsrecordcontributions/status] scope: Namespaced - - apiGroups: [dns.networking.miloapis.com] - apiVersions: [v1alpha1] - operations: [UPDATE] - resources: [dnsresolvercontexts/status, dnsresolveraccessbindings/status] - scope: Namespaced - # Platform-owned serving and publication state, including their status. - - apiGroups: [dns.networking.miloapis.com] - apiVersions: [v1alpha1] - operations: [CREATE, UPDATE, DELETE] - resources: - - dnsresolverbindings - - dnsresolverbindings/status - - dnspublicationmanifests - - dnspublicationmanifests/status - - dnspublicationchunks - - dnstransportoutboxes - - dnstransportoutboxes/status - - dnspublicationownerships - - dnspublicationownerships/status - - dnsmanagednamespaces - - dnsmanagednamespaces/status - - dnsregistrations/status - - dnscontributiongrants/status - - dnszones/status - scope: Namespaced From d3c8d4f426eb369dfcb3463d0f10b2335e177ace Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 17:04:13 -0500 Subject: [PATCH 04/23] Scope private DNS customer configuration to project roles --- config/internal-dns/rbac.yaml | 14 ++++++++++++++ .../runtime/deployment_permissions_test.go | 13 ++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/config/internal-dns/rbac.yaml b/config/internal-dns/rbac.yaml index 4fee5534..eb4695c1 100644 --- a/config/internal-dns/rbac.yaml +++ b/config/internal-dns/rbac.yaml @@ -43,6 +43,20 @@ rules: resources: [dnsrecordcontributions/status] verbs: [get, update, patch] --- +# Customers configure private zones and naming within their project namespace. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: internal-dns-customer + namespace: example-project +rules: +- apiGroups: [dns.networking.miloapis.com] + resources: [dnszones, dnsrecordsets, dnszoneassociations, dnsnamingpolicies, dnsregistrations] + verbs: [get, list, watch, create, update, patch, delete] +- apiGroups: [dns.networking.miloapis.com] + resources: [dnsresolvercontexts, dnsmanagednamespaces] + verbs: [get, list, watch] +--- # Bind only to a trusted issuer that applies the product's registration policy. apiVersion: rbac.authorization.k8s.io/v1 kind: Role diff --git a/internal/internaldns/runtime/deployment_permissions_test.go b/internal/internaldns/runtime/deployment_permissions_test.go index ec08846e..093eee6c 100644 --- a/internal/internaldns/runtime/deployment_permissions_test.go +++ b/internal/internaldns/runtime/deployment_permissions_test.go @@ -87,7 +87,7 @@ func TestDeploymentRolesAuthorizeWorkerAndDiscoveryBoundaries(t *testing.T) { t.Fatal(err) } } - for _, identity := range []struct{ name, role string }{{"product", "internal-dns-product-writer"}, {"issuer", "internal-dns-grant-issuer"}, {"integration", "internal-dns-vpc-integration"}} { + for _, identity := range []struct{ name, role string }{{"product", "internal-dns-product-writer"}, {"issuer", "internal-dns-grant-issuer"}, {"integration", "internal-dns-vpc-integration"}, {"customer", "internal-dns-customer"}} { binding := &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: identity.name + "-test", Namespace: "example-project"}, Subjects: []rbacv1.Subject{{Kind: "ServiceAccount", Name: identity.name, Namespace: "example-project"}}, RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "Role", Name: identity.role}} if err := cl.Create(ctx, binding); err != nil { t.Fatal(err) @@ -126,6 +126,17 @@ func TestDeploymentRolesAuthorizeWorkerAndDiscoveryBoundaries(t *testing.T) { {"product", "dnsresolvercontexts", "", "patch", false}, {"product", "dnsresolveraccessbindings", "", "create", false}, {"product", "dnspublicationmanifests", "", "create", false}, + {"customer", "dnszones", "", "create", true}, + {"customer", "dnszoneassociations", "", "create", true}, + {"customer", "dnsnamingpolicies", "", "create", true}, + {"customer", "dnsresolvercontexts", "", "get", true}, + {"customer", "dnsmanagednamespaces", "", "get", true}, + {"customer", "dnsresolvercontexts", "", "patch", false}, + {"customer", "dnscontributiongrants", "", "create", false}, + {"customer", "dnscontributiongrants", "status", "patch", false}, + {"customer", "dnsresolveraccessbindings", "", "create", false}, + {"customer", "dnszoneassociations", "status", "patch", false}, + {"customer", "dnspublicationmanifests", "", "create", false}, {"integration", "dnsresolvercontexts", "", "create", true}, {"integration", "dnsresolveraccessbindings", "", "create", true}, {"integration", "dnsresolvercontexts", "status", "patch", false}, From 3422f2639eda912a286d462461bf52a5d852eaf2 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 15:24:57 -0500 Subject: [PATCH 05/23] fix: Include serving readiness dependencies Install jq for fleet readiness checks and verify that the serving image contains the required commands and executable helpers. Run these checks as the serving user during both architecture image builds. --- config/internal-dns/Dockerfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/config/internal-dns/Dockerfile b/config/internal-dns/Dockerfile index 92af8799..6416aa75 100644 --- a/config/internal-dns/Dockerfile +++ b/config/internal-dns/Dockerfile @@ -21,9 +21,15 @@ ENTRYPOINT ["/internal-dns"] FROM debian:trixie-slim AS serving RUN apt-get update && apt-get install --no-install-recommends -y \ - bind9 bind9-utils dnsdist iproute2 procps netcat-openbsd ca-certificates bash && \ + bind9 bind9-utils dnsdist iproute2 procps netcat-openbsd ca-certificates bash jq && \ rm -rf /var/lib/apt/lists/* COPY --from=build /out/internal-dns /internal-dns COPY config/internal-dns/commands/ /usr/local/sbin/ USER 65532:65532 +RUN for tool in bash jq date ip nc named named-checkconf named-checkzone dnsdist rndc; do \ + command -v "$tool" >/dev/null || exit 1; \ + done && jq --version && test -x /internal-dns +RUN for helper in /usr/local/sbin/internal-dns-*; do \ + test -x "$helper" && bash -n "$helper" || exit 1; \ + done ENTRYPOINT ["/internal-dns"] From 28df6aed4b61d9eb1842ca16ea5e52c9f4a6a4d7 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 15:58:49 -0500 Subject: [PATCH 06/23] feat: Discover internal DNS project sources Discover ready Milo projects and reconcile every source namespace without restarting the DNS service. Keep active sources during discovery outages and resolve admission requests against live project identity. Route serving artifacts by resource type and scope access withdrawal to its source namespace so shared namespace names stay isolated. --- config/internal-dns/README.md | 9 + .../internal-dns/control-plane.example.yaml | 25 ++- .../controlplane/context_reconciler.go | 23 ++- .../controlplane/context_reconciler_test.go | 49 +++++ .../internaldns/controlplane/reconciler.go | 2 +- internal/internaldns/platform/acks.go | 6 +- internal/internaldns/runtime/config.go | 33 +++- internal/internaldns/runtime/config_test.go | 20 ++ .../internaldns/runtime/project_discovery.go | 181 ++++++++++++++++++ .../runtime/project_discovery_test.go | 157 +++++++++++++++ .../internaldns/runtime/routing_client.go | 28 +-- .../runtime/routing_client_test.go | 25 ++- internal/internaldns/runtime/run.go | 59 ++++-- 13 files changed, 561 insertions(+), 56 deletions(-) create mode 100644 internal/internaldns/runtime/project_discovery.go create mode 100644 internal/internaldns/runtime/project_discovery_test.go diff --git a/config/internal-dns/README.md b/config/internal-dns/README.md index 8d9667e0..cd58e618 100644 --- a/config/internal-dns/README.md +++ b/config/internal-dns/README.md @@ -38,6 +38,15 @@ identity and storage; do not run two pods against one checkpoint. ## Control-plane and API boundaries +Control-plane replicas discover ready Milo projects through the core API and +watch every namespace in each project control plane. New projects are included +without restarting the service. `projectDiscovery.kubeconfig` authenticates +discovery and project API access; the process `--kubeconfig` authenticates the +DNS service project. Generated kubeconfig Secrets use the `value` key. +Unavailable discovery retains active sources and applied DNS configuration. +Admission resolves the authenticated project parent against its live identity, +and acknowledgement projection follows the discovered source clients. + Control-plane replicas can concurrently watch the platform API and source projects. Publication and shard ownership use resource-version checks and increasing writer epochs on takeover. Each source project has an authenticated diff --git a/config/internal-dns/control-plane.example.yaml b/config/internal-dns/control-plane.example.yaml index 313be5fd..171d9d3c 100644 --- a/config/internal-dns/control-plane.example.yaml +++ b/config/internal-dns/control-plane.example.yaml @@ -10,14 +10,11 @@ data: "shard": "shared-0", "platformNamespace": "internal-dns-system", "platformKubeAPI": {"qps":50,"burst":100,"requestTimeoutSeconds":10}, - "projects": [ - { - "name": "example-project", - "projectUID": "00000000-0000-0000-0000-000000000001", - "namespace": "example-project", - "kubeAPI": {"qps":20,"burst":40,"requestTimeoutSeconds":10} - } - ], + "projectDiscovery": { + "kubeconfig": "/var/run/internal-dns/discovery/kubeconfig", + "intervalSeconds": 10, + "kubeAPI": {"qps":20,"burst":40,"requestTimeoutSeconds":10} + }, "nodeBackends": [ {"memberID":"node-bind-0","address":"fd70:10::10","port":5300} ], @@ -83,7 +80,7 @@ spec: containers: - name: control-plane image: example.invalid/dns-operator/internal-dns:VERSION - args: ["--role=control-plane","--config=/etc/internal-dns/config.json"] + args: ["--role=control-plane","--config=/etc/internal-dns/config.json","--kubeconfig=/var/run/internal-dns/platform/kubeconfig"] ports: - {name: admission, containerPort: 9443} securityContext: @@ -92,10 +89,20 @@ spec: runAsNonRoot: true capabilities: {drop: [ALL]} volumeMounts: + - {name: discovery, mountPath: /var/run/internal-dns/discovery, readOnly: true} + - {name: platform, mountPath: /var/run/internal-dns/platform, readOnly: true} - {name: config, mountPath: /etc/internal-dns, readOnly: true} - {name: nats, mountPath: /var/run/internal-dns/nats, readOnly: true} - {name: admission, mountPath: /var/run/internal-dns/admission, readOnly: true} volumes: + - name: discovery + secret: + secretName: internal-dns-discovery-kubeconfig + items: [{key: value, path: kubeconfig}] + - name: platform + secret: + secretName: internal-dns-service-kubeconfig + items: [{key: value, path: kubeconfig}] - name: config configMap: {name: internal-dns-control-plane-config} - name: nats diff --git a/internal/internaldns/controlplane/context_reconciler.go b/internal/internaldns/controlplane/context_reconciler.go index c09efda6..826b2319 100644 --- a/internal/internaldns/controlplane/context_reconciler.go +++ b/internal/internaldns/controlplane/context_reconciler.go @@ -327,7 +327,7 @@ func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, as return err } } - binding := &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: r.Options.PlatformNamespace, Labels: map[string]string{"internal-dns.miloapis.com/context-uid": string(contextObject.UID), "internal-dns.miloapis.com/access-uid": string(a.UID), "internal-dns.miloapis.com/region": model.SafeToken(spec.Region), "internal-dns.miloapis.com/shard": model.SafeToken(r.shardForContextRegion(contextObject.UID, spec.Region, existing.Items))}}, Spec: dnsv1alpha1.DNSResolverBindingSpec{Source: dnsv1alpha1.DNSResolverBindingSource{ProjectUID: r.Options.ProjectUID, ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: contextObject.Name, UID: contextObject.UID}, AccessBindingRef: dnsv1alpha1.DNSObjectReference{Name: a.Name, UID: a.UID, Generation: a.Generation}}, Placement: dnsv1alpha1.DNSResolverBindingPlacement{Region: spec.Region, Shard: r.shardForContextRegion(contextObject.UID, spec.Region, existing.Items)}, Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Generation: generation, Revision: revision, Query: dnsv1alpha1.DNSResolverBindingQuery{NodeAddress: spec.DestinationAddress, RegionalAddress: cluster}, ZoneRefs: zoneReferences(zids, zones), ZoneApices: zoneApices(zids, zones)}, Version: version}} + binding := &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: r.Options.PlatformNamespace, Labels: map[string]string{"internal-dns.miloapis.com/source-namespace": ns, "internal-dns.miloapis.com/context-uid": string(contextObject.UID), "internal-dns.miloapis.com/access-uid": string(a.UID), "internal-dns.miloapis.com/region": model.SafeToken(spec.Region), "internal-dns.miloapis.com/shard": model.SafeToken(r.shardForContextRegion(contextObject.UID, spec.Region, existing.Items))}}, Spec: dnsv1alpha1.DNSResolverBindingSpec{Source: dnsv1alpha1.DNSResolverBindingSource{ProjectUID: r.Options.ProjectUID, ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: contextObject.Name, UID: contextObject.UID}, AccessBindingRef: dnsv1alpha1.DNSObjectReference{Name: a.Name, UID: a.UID, Generation: a.Generation}}, Placement: dnsv1alpha1.DNSResolverBindingPlacement{Region: spec.Region, Shard: r.shardForContextRegion(contextObject.UID, spec.Region, existing.Items)}, Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Generation: generation, Revision: revision, Query: dnsv1alpha1.DNSResolverBindingQuery{NodeAddress: spec.DestinationAddress, RegionalAddress: cluster}, ZoneRefs: zoneReferences(zids, zones), ZoneApices: zoneApices(zids, zones)}, Version: version}} if !exists { if err := r.Create(ctx, binding); err != nil { return err @@ -335,7 +335,7 @@ func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, as if err := r.Get(ctx, client.ObjectKeyFromObject(binding), binding); err != nil { return err } - } else if !bindingEqual(old.Spec, binding.Spec) { + } else if !bindingEqual(old.Spec, binding.Spec) || old.Labels["internal-dns.miloapis.com/source-namespace"] != ns { baseOld := old.DeepCopy() old.Spec = binding.Spec old.Labels = binding.Labels @@ -368,6 +368,25 @@ func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, as if b.Labels["internal-dns.miloapis.com/context-uid"] == "" { continue } + sourceNamespace := b.Labels["internal-dns.miloapis.com/source-namespace"] + if sourceNamespace != "" && sourceNamespace != ns { + continue + } + if sourceNamespace == "" && r.Options.ProjectNamespace == "" { + var sourceAccesses dnsv1alpha1.DNSResolverAccessBindingList + if err := r.sourceReader().List(ctx, &sourceAccesses); err != nil { + return err + } + for j := range sourceAccesses.Items { + if sourceAccesses.Items[j].UID == b.Spec.Source.AccessBindingRef.UID { + sourceNamespace = sourceAccesses.Items[j].Namespace + break + } + } + if sourceNamespace != "" && sourceNamespace != ns { + continue + } + } withdraw, err := r.confirmAccessWithdrawal(ctx, ns, b) if err != nil { return err diff --git a/internal/internaldns/controlplane/context_reconciler_test.go b/internal/internaldns/controlplane/context_reconciler_test.go index ceced401..98e196fd 100644 --- a/internal/internaldns/controlplane/context_reconciler_test.go +++ b/internal/internaldns/controlplane/context_reconciler_test.go @@ -14,6 +14,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -285,3 +286,51 @@ func TestStableOneShardPerRegionAssignment(t *testing.T) { t.Fatalf("adding a shard moved live VPC: before=%s after=%s", central.Shard, after[0].Shard) } } + +func TestNamespaceReconciliationCannotWithdrawAnotherNamespace(t *testing.T) { + for _, labeled := range []bool{false, true} { + t.Run(fmt.Sprintf("namespace-label-%t", labeled), func(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + if err := addReferenceStateToScheme(scheme); err != nil { + t.Fatal(err) + } + access := &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "access", Namespace: "other", UID: "other-access"}} + binding := &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{Name: "binding", Namespace: "platform", Labels: map[string]string{"internal-dns.miloapis.com/context-uid": "other-context"}}, Spec: dnsv1alpha1.DNSResolverBindingSpec{Source: dnsv1alpha1.DNSResolverBindingSource{ProjectUID: "project", AccessBindingRef: dnsv1alpha1.DNSObjectReference{Name: access.Name, UID: access.UID}, ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: "context", UID: "other-context"}}}} + if labeled { + binding.Labels["internal-dns.miloapis.com/source-namespace"] = access.Namespace + } + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(access, binding).Build() + reconciler := &Reconciler{Client: cl, APIReader: cl, Options: ReconcilerOptions{ProjectUID: "project", PlatformNamespace: "platform"}} + if err := reconciler.reconcileContextBindings(ctx, "different", nil, nil, time.Now()); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(binding), binding); err != nil { + t.Fatal(err) + } + if binding.Spec.Tombstone { + t.Fatal("another source namespace was withdrawn") + } + }) + } +} + +func TestAllNamespaceReconcileIncludesPlatformNamedSourceNamespace(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + if err := addReferenceStateToScheme(scheme); err != nil { + t.Fatal(err) + } + contextObject := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "context", Namespace: "platform", UID: "context", Generation: 1}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "network"}} + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(contextObject).WithStatusSubresource(contextObject).Build() + r := &Reconciler{Client: cl, APIReader: cl, Options: ReconcilerOptions{ProjectUID: "project", PlatformNamespace: "platform"}} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(contextObject)}); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(contextObject), contextObject); err != nil { + t.Fatal(err) + } + if !apimeta.IsStatusConditionTrue(contextObject.Status.Conditions, "Ready") { + t.Fatal("source namespace was excluded because it shares the managed namespace name") + } +} diff --git a/internal/internaldns/controlplane/reconciler.go b/internal/internaldns/controlplane/reconciler.go index a6b87a30..9ca5a66b 100644 --- a/internal/internaldns/controlplane/reconciler.go +++ b/internal/internaldns/controlplane/reconciler.go @@ -105,7 +105,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu if ns == "" { ns = req.Namespace } - if ns == "" || ns == r.Options.PlatformNamespace { + if ns == "" { return ctrl.Result{}, nil } return r.reconcileProject(ctx, ns) diff --git a/internal/internaldns/platform/acks.go b/internal/internaldns/platform/acks.go index bca7ea0a..d185bab0 100644 --- a/internal/internaldns/platform/acks.go +++ b/internal/internaldns/platform/acks.go @@ -36,6 +36,7 @@ type AckSink struct { Namespace string ProjectNamespaces []string Projects []ProjectScope + ProjectScopes func() []ProjectScope PublicationMembers []model.Member PublicationRegions map[string][]model.Member Region string @@ -182,6 +183,9 @@ func (s *AckSink) serving(ctx context.Context, a model.MemberAck, row dnsv1alpha func (s *AckSink) projectPublication(ctx context.Context, m *dnsv1alpha1.DNSPublicationManifest, complete bool, now time.Time) error { projects := append([]ProjectScope(nil), s.Projects...) + if s.ProjectScopes != nil { + projects = append(projects, s.ProjectScopes()...) + } for _, ns := range s.ProjectNamespaces { projects = append(projects, ProjectScope{Client: s.Client, Namespace: ns}) } @@ -230,7 +234,7 @@ func (s *AckSink) projectPublicationForScope(ctx context.Context, project Projec } } var regs dnsv1alpha1.DNSRegistrationList - if err := projectClient.List(ctx, ®s, client.InNamespace(ns)); err != nil { + if err := projectClient.List(ctx, ®s, client.InNamespace(z.Namespace)); err != nil { projectErrors = errors.Join(projectErrors, fmt.Errorf("project %s list registrations: %w", ns, err)) continue } diff --git a/internal/internaldns/runtime/config.go b/internal/internaldns/runtime/config.go index 393a3f4e..a5a4d024 100644 --- a/internal/internaldns/runtime/config.go +++ b/internal/internaldns/runtime/config.go @@ -35,6 +35,12 @@ type ProjectConfig struct { KubeAPI KubeAPIClientConfig `json:"kubeAPI,omitempty"` } +type ProjectDiscoveryConfig struct { + Kubeconfig string `json:"kubeconfig"` + IntervalSeconds int64 `json:"intervalSeconds,omitempty"` + KubeAPI KubeAPIClientConfig `json:"kubeAPI,omitempty"` +} + type KubeAPIClientConfig struct { QPS float32 `json:"qps,omitempty"` Burst int `json:"burst,omitempty"` @@ -59,7 +65,8 @@ type Config struct { Shard string `json:"shard"` PlatformNamespace string `json:"platformNamespace"` PlatformKubeAPI KubeAPIClientConfig `json:"platformKubeAPI,omitempty"` - Projects []ProjectConfig `json:"projects"` + Projects []ProjectConfig `json:"projects,omitempty"` + ProjectDiscovery *ProjectDiscoveryConfig `json:"projectDiscovery,omitempty"` NodeBackends []model.Backend `json:"nodeBackends,omitempty"` ClusterBackends []model.Backend `json:"clusterBackends,omitempty"` Members []model.Member `json:"members,omitempty"` @@ -119,6 +126,12 @@ func (c *Config) defaults() { } defaultKubeAPI(&c.PlatformKubeAPI, 50, 100) + if c.ProjectDiscovery != nil { + defaultKubeAPI(&c.ProjectDiscovery.KubeAPI, 20, 40) + if c.ProjectDiscovery.IntervalSeconds == 0 { + c.ProjectDiscovery.IntervalSeconds = 10 + } + } for i := range c.Projects { defaultKubeAPI(&c.Projects[i].KubeAPI, 20, 40) } @@ -191,12 +204,26 @@ func (c *Config) Validate(role string) error { } return c.Agent.Validate() } - if c.PlatformNamespace == "" || len(c.Projects) == 0 { - return errors.New("control-plane requires platformNamespace and projects") + if c.PlatformNamespace == "" || (len(c.Projects) == 0 && c.ProjectDiscovery == nil) { + return errors.New("control-plane requires platformNamespace and project discovery or projects") } if err := validateKubeAPI("platformKubeAPI", c.PlatformKubeAPI); err != nil { return err } + if c.ProjectDiscovery != nil { + if len(c.Projects) != 0 { + return errors.New("projectDiscovery and projects are mutually exclusive") + } + if !filepath.IsAbs(c.ProjectDiscovery.Kubeconfig) { + return errors.New("projectDiscovery.kubeconfig must be absolute") + } + if c.ProjectDiscovery.IntervalSeconds < 1 || c.ProjectDiscovery.IntervalSeconds > 300 { + return errors.New("projectDiscovery.intervalSeconds must be between 1 and 300") + } + if err := validateKubeAPI("projectDiscovery.kubeAPI", c.ProjectDiscovery.KubeAPI); err != nil { + return err + } + } names := map[string]bool{} uids := map[types.UID]bool{} namespaces := map[string]bool{} diff --git a/internal/internaldns/runtime/config_test.go b/internal/internaldns/runtime/config_test.go index 635824db..6ce6a902 100644 --- a/internal/internaldns/runtime/config_test.go +++ b/internal/internaldns/runtime/config_test.go @@ -205,3 +205,23 @@ func TestAgentRestoresCheckpointWhileBrokerUnavailable(t *testing.T) { t.Fatal("agent did not stop") } } + +func TestControlPlaneProjectDiscoveryValidation(t *testing.T) { + base := Config{Region: "central", Shard: "one", NATS: transport.Config{URL: "nats://fixture:4222", AllowInsecure: true}, Stream: transport.StreamConfig{Name: "DNS", SnapshotsBucket: "DNS_BOOTSTRAP"}, Members: []model.Member{{MemberID: "regional", Role: "regional"}}, ProjectDiscovery: &ProjectDiscoveryConfig{Kubeconfig: "/var/run/discovery/kubeconfig"}} + if err := base.Validate(roleControlPlane); err != nil { + t.Fatal(err) + } + if base.ProjectDiscovery.IntervalSeconds != 10 || base.ProjectDiscovery.KubeAPI.RequestTimeoutSeconds != 10 { + t.Fatal("discovery defaults missing") + } + mixed := base + mixed.Projects = []ProjectConfig{{ProjectUID: "project", Namespace: "project"}} + if err := mixed.Validate(roleControlPlane); err == nil { + t.Fatal("mixed discovery and static sources accepted") + } + relative := base + relative.ProjectDiscovery = &ProjectDiscoveryConfig{Kubeconfig: "relative"} + if err := relative.Validate(roleControlPlane); err == nil { + t.Fatal("relative discovery credential accepted") + } +} diff --git a/internal/internaldns/runtime/project_discovery.go b/internal/internaldns/runtime/project_discovery.go new file mode 100644 index 00000000..43d63e9e --- /dev/null +++ b/internal/internaldns/runtime/project_discovery.go @@ -0,0 +1,181 @@ +package runtime + +import ( + "context" + "errors" + "fmt" + "net/url" + "path" + "sort" + "sync" + + "go.miloapis.com/dns-operator/internal/internaldns/platform" + resourcemanagerv1alpha1 "go.miloapis.com/milo/pkg/apis/resourcemanager/v1alpha1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + kubeadmission "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +type directoryProject struct { + projectRuntime + cancel context.CancelFunc +} + +type projectDirectory struct { + mu sync.RWMutex + projects map[string]directoryProject + discover client.Reader +} + +func newProjectDirectory(projects []projectRuntime) *projectDirectory { + d := &projectDirectory{projects: map[string]directoryProject{}} + for _, p := range projects { + d.projects[p.config.Name] = directoryProject{projectRuntime: p} + } + return d +} + +func (d *projectDirectory) scopes() []platform.ProjectScope { + d.mu.RLock() + defer d.mu.RUnlock() + scopes := make([]platform.ProjectScope, 0, len(d.projects)) + names := make([]string, 0, len(d.projects)) + for name := range d.projects { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + p := d.projects[name] + scopes = append(scopes, platform.ProjectScope{Client: p.direct, Namespace: p.config.Namespace, RequestTimeout: p.config.KubeAPI.requestTimeout()}) + } + return scopes +} + +func (d *projectDirectory) selectProject(ctx context.Context, req kubeadmission.Request) (client.Client, error) { + parents := req.UserInfo.Extra[parentNameExtra] + d.mu.RLock() + if d.discover == nil { + projects := make([]projectRuntime, 0, len(d.projects)) + for _, p := range d.projects { + projects = append(projects, p.projectRuntime) + } + d.mu.RUnlock() + return projectSelector(projects)(ctx, req) + } + reader := d.discover + d.mu.RUnlock() + if len(parents) != 1 || req.Namespace == "" { + return nil, errors.New("authenticated project parent is missing or ambiguous") + } + var project resourcemanagerv1alpha1.Project + if err := reader.Get(ctx, client.ObjectKey{Name: parents[0]}, &project); err != nil { + return nil, fmt.Errorf("observe authenticated project: %w", err) + } + if !project.DeletionTimestamp.IsZero() || !apimeta.IsStatusConditionTrue(project.Status.Conditions, resourcemanagerv1alpha1.ProjectReady) { + return nil, errors.New("authenticated project is unavailable") + } + d.mu.RLock() + defer d.mu.RUnlock() + p, found := d.projects[parents[0]] + if !found || p.config.ProjectUID != project.UID { + return nil, errors.New("authenticated project source is not ready") + } + return p.direct, nil +} + +type projectDiscovery struct { + client client.Client + directory *projectDirectory + rest *rest.Config + kubeAPI KubeAPIClientConfig + makeProject func(ProjectConfig, *rest.Config) (projectRuntime, error) +} + +func (d *projectDiscovery) step(ctx context.Context) error { + d.directory.mu.Lock() + d.directory.discover = d.client + d.directory.mu.Unlock() + var projects resourcemanagerv1alpha1.ProjectList + if err := d.client.List(ctx, &projects); err != nil { + return err + } + seen := map[string]bool{} + var failures error + for i := range projects.Items { + p := &projects.Items[i] + seen[p.Name] = true + if p.UID == "" { + failures = errors.Join(failures, fmt.Errorf("project %s has no identity", p.Name)) + continue + } + d.directory.mu.RLock() + existing, exists := d.directory.projects[p.Name] + d.directory.mu.RUnlock() + ready := p.DeletionTimestamp.IsZero() && apimeta.IsStatusConditionTrue(p.Status.Conditions, resourcemanagerv1alpha1.ProjectReady) + if exists && (existing.config.ProjectUID != p.UID || !ready) { + d.remove(p.Name) + exists = false + } + if !ready || exists { + continue + } + projectREST, err := projectRESTConfig(d.rest, p.Name) + if err != nil { + failures = errors.Join(failures, err) + continue + } + worker, err := d.makeProject(ProjectConfig{Name: p.Name, ProjectUID: p.UID, KubeAPI: d.kubeAPI}, projectREST) + if err != nil { + failures = errors.Join(failures, err) + continue + } + sourceCtx, cancel := context.WithCancel(ctx) + d.directory.mu.Lock() + d.directory.projects[p.Name] = directoryProject{projectRuntime: worker, cancel: cancel} + d.directory.mu.Unlock() + go runProjectSource(sourceCtx, worker) + } + d.directory.mu.RLock() + names := make([]string, 0, len(d.directory.projects)) + for name := range d.directory.projects { + names = append(names, name) + } + d.directory.mu.RUnlock() + for _, name := range names { + if !seen[name] { + d.remove(name) + } + } + return failures +} + +func (d *projectDiscovery) remove(name string) { + d.directory.mu.Lock() + defer d.directory.mu.Unlock() + if p, found := d.directory.projects[name]; found { + if p.cancel != nil { + p.cancel() + } + delete(d.directory.projects, name) + } +} + +func projectRESTConfig(base *rest.Config, name string) (*rest.Config, error) { + endpoint, err := url.Parse(base.Host) + if err != nil || endpoint.Scheme == "" || endpoint.Host == "" { + return nil, fmt.Errorf("invalid project discovery API endpoint %q", base.Host) + } + endpoint.Path = path.Join(endpoint.Path, "apis/resourcemanager.miloapis.com/v1alpha1/projects", name, "control-plane") + config := rest.CopyConfig(base) + config.Host = endpoint.String() + return config, nil +} + +func projectCache(namespace string) cache.Options { + if namespace == "" { + return cache.Options{} + } + return cache.Options{DefaultNamespaces: map[string]cache.Config{namespace: {}}} +} diff --git a/internal/internaldns/runtime/project_discovery_test.go b/internal/internaldns/runtime/project_discovery_test.go new file mode 100644 index 00000000..6fa884e4 --- /dev/null +++ b/internal/internaldns/runtime/project_discovery_test.go @@ -0,0 +1,157 @@ +package runtime + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + dnsv1 "go.miloapis.com/dns-operator/api/v1alpha1" + resourcemanagerv1alpha1 "go.miloapis.com/milo/pkg/apis/resourcemanager/v1alpha1" + admissionv1 "k8s.io/api/admission/v1" + authenticationv1 "k8s.io/api/authentication/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + kubeadmission "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +type discoveryFixtureClient struct { + client.Client + unavailable bool +} + +func (c *discoveryFixtureClient) List(ctx context.Context, obj client.ObjectList, opts ...client.ListOption) error { + if c.unavailable { + return errors.New("discovery unavailable") + } + return c.Client.List(ctx, obj, opts...) +} + +func TestProjectDiscoveryAddsSourcesAndPreservesThemDuringOutage(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + scheme, err := newScheme() + if err != nil { + t.Fatal(err) + } + first := readyDiscoveryProject("first", "first-uid") + core := &discoveryFixtureClient{Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(first).Build()} + directory := newProjectDirectory(nil) + starts := make(chan types.UID, 4) + d := &projectDiscovery{client: core, directory: directory, rest: &rest.Config{Host: "https://milo.example:6443"}, makeProject: func(config ProjectConfig, sourceREST *rest.Config) (projectRuntime, error) { + if config.Namespace != "" || !strings.HasSuffix(sourceREST.Host, "/projects/"+config.Name+"/control-plane") { + t.Errorf("unexpected discovery scope: %#v %s", config, sourceREST.Host) + } + direct := fake.NewClientBuilder().WithScheme(scheme).Build() + return projectRuntime{config: config, direct: direct, start: func(workerCtx context.Context) error { + starts <- config.ProjectUID + <-workerCtx.Done() + return workerCtx.Err() + }}, nil + }} + if err := d.step(ctx); err != nil { + t.Fatal(err) + } + awaitDiscoveredSource(t, starts, first.UID) + second := readyDiscoveryProject("second", "second-uid") + if err := core.Create(ctx, second); err != nil { + t.Fatal(err) + } + if err := d.step(ctx); err != nil { + t.Fatal(err) + } + awaitDiscoveredSource(t, starts, second.UID) + scopes := directory.scopes() + if len(scopes) != 2 || scopes[0].Namespace != "" || scopes[1].Namespace != "" { + t.Fatalf("ACK scopes did not follow discovery: %#v", scopes) + } + core.unavailable = true + if err := d.step(ctx); err == nil { + t.Fatal("expected discovery error") + } + if len(directory.scopes()) != 2 { + t.Fatal("discovery outage discarded existing sources") + } + core.unavailable = false + request := kubeadmission.Request{AdmissionRequest: admissionv1.AdmissionRequest{Namespace: "internal-dns-platform", UserInfo: authenticationv1.UserInfo{Extra: map[string]authenticationv1.ExtraValue{parentNameExtra: {second.Name}}}}} + selected, err := directory.selectProject(ctx, request) + if err != nil || selected != scopes[1].Client { + t.Fatalf("dynamic admission did not map all namespaces: %v", err) + } + if err := core.Delete(ctx, second); err != nil { + t.Fatal(err) + } + if _, err := directory.selectProject(ctx, request); err == nil { + t.Fatal("deleted project passed admission") + } + recreated := readyDiscoveryProject(second.Name, "replacement-uid") + if err := core.Create(ctx, recreated); err != nil { + t.Fatal(err) + } + if _, err := directory.selectProject(ctx, request); err == nil { + t.Fatal("recreated project used previous source") + } + if err := d.step(ctx); err != nil { + t.Fatal(err) + } + awaitDiscoveredSource(t, starts, recreated.UID) + if _, err := directory.selectProject(ctx, request); err != nil { + t.Fatal(err) + } + if err := core.Delete(ctx, recreated); err != nil { + t.Fatal(err) + } + if err := d.step(ctx); err != nil { + t.Fatal(err) + } + if len(directory.scopes()) != 1 { + t.Fatal("ACK scopes retained deleted source") + } +} + +func readyDiscoveryProject(name string, uid types.UID) *resourcemanagerv1alpha1.Project { + return &resourcemanagerv1alpha1.Project{ObjectMeta: metav1.ObjectMeta{Name: name, UID: uid}, Status: resourcemanagerv1alpha1.ProjectStatus{Conditions: []metav1.Condition{{Type: resourcemanagerv1alpha1.ProjectReady, Status: metav1.ConditionTrue, Reason: "Ready", LastTransitionTime: metav1.Now()}}}} +} +func awaitDiscoveredSource(t *testing.T, starts <-chan types.UID, expected types.UID) { + t.Helper() + select { + case actual := <-starts: + if actual != expected { + t.Fatalf("started %s want %s", actual, expected) + } + case <-time.After(time.Second): + t.Fatal("new project source did not start") + } +} + +func TestAllNamespaceRoutingPreservesListsAndStatus(t *testing.T) { + scheme, err := newScheme() + if err != nil { + t.Fatal(err) + } + zone := &dnsv1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "same-name", Namespace: "internal-dns-platform"}} + other := &dnsv1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: zone.Name, Namespace: "other"}} + source := fake.NewClientBuilder().WithScheme(scheme).WithObjects(zone, other).WithStatusSubresource(zone, other).Build() + managed := fake.NewClientBuilder().WithScheme(scheme).Build() + routed := &RoutingClient{Client: managed, Source: source, PlatformNamespace: zone.Namespace} + var zones dnsv1.DNSZoneList + if err := routed.List(context.Background(), &zones); err != nil { + t.Fatal(err) + } + if len(zones.Items) != 2 { + t.Fatalf("source namespaces were dropped: %#v", zones.Items) + } + current := &zones.Items[0] + base := current.DeepCopy() + current.Status.Conditions = []metav1.Condition{{Type: "Published", Status: metav1.ConditionTrue, Reason: "Ready", LastTransitionTime: metav1.Now()}} + if err := routed.Status().Patch(context.Background(), current, client.MergeFrom(base)); err != nil { + t.Fatal(err) + } + if err := source.Get(context.Background(), client.ObjectKeyFromObject(current), current); err != nil || len(current.Status.Conditions) != 1 { + t.Fatalf("status did not reach source: %v", err) + } +} diff --git a/internal/internaldns/runtime/routing_client.go b/internal/internaldns/runtime/routing_client.go index 38211767..e1e2c53e 100644 --- a/internal/internaldns/runtime/routing_client.go +++ b/internal/internaldns/runtime/routing_client.go @@ -3,6 +3,8 @@ package runtime import ( "context" + dnsv1 "go.miloapis.com/dns-operator/api/v1alpha1" + k8sruntime "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -16,24 +18,23 @@ type RoutingClient struct { PlatformNamespace string } -func (c *RoutingClient) target(obj client.Object) client.Client { - if obj != nil && obj.GetNamespace() == c.PlatformNamespace { - return c.Client - } - return c.Source -} -func (c *RoutingClient) targetNamespace(namespace string) client.Client { - if namespace == c.PlatformNamespace { +func (c *RoutingClient) target(obj k8sruntime.Object) client.Client { + switch obj.(type) { + case *dnsv1.DNSZone, *dnsv1.DNSZoneList, *dnsv1.DNSZoneAssociation, *dnsv1.DNSZoneAssociationList, + *dnsv1.DNSRecordSet, *dnsv1.DNSRecordSetList, *dnsv1.DNSResolverContext, *dnsv1.DNSResolverContextList, + *dnsv1.DNSResolverAccessBinding, *dnsv1.DNSResolverAccessBindingList, + *dnsv1.DNSRegistration, *dnsv1.DNSRegistrationList, *dnsv1.DNSNamingPolicy, *dnsv1.DNSNamingPolicyList, + *dnsv1.DNSRecordContribution, *dnsv1.DNSRecordContributionList: + return c.Source + default: return c.Client } - return c.Source } func (c *RoutingClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { - return c.targetNamespace(key.Namespace).Get(ctx, key, obj, opts...) + return c.target(obj).Get(ctx, key, obj, opts...) } func (c *RoutingClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { - o := (&client.ListOptions{}).ApplyOptions(opts) - return c.targetNamespace(o.Namespace).List(ctx, list, opts...) + return c.target(list).List(ctx, list, opts...) } func (c *RoutingClient) Apply(ctx context.Context, obj k8sruntime.ApplyConfiguration, opts ...client.ApplyOption) error { return c.Client.Apply(ctx, obj, opts...) @@ -51,8 +52,7 @@ func (c *RoutingClient) Patch(ctx context.Context, obj client.Object, p client.P return c.target(obj).Patch(ctx, obj, p, opts...) } func (c *RoutingClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { - o := (&client.DeleteAllOfOptions{}).ApplyOptions(opts) - return c.targetNamespace(o.Namespace).DeleteAllOf(ctx, obj, opts...) + return c.target(obj).DeleteAllOf(ctx, obj, opts...) } func (c *RoutingClient) Status() client.SubResourceWriter { return &routingSubresource{parent: c, name: "status"} diff --git a/internal/internaldns/runtime/routing_client_test.go b/internal/internaldns/runtime/routing_client_test.go index a9363dd0..63defced 100644 --- a/internal/internaldns/runtime/routing_client_test.go +++ b/internal/internaldns/runtime/routing_client_test.go @@ -6,6 +6,7 @@ import ( "context" "testing" + dnsv1 "go.miloapis.com/dns-operator/api/v1alpha1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -18,11 +19,14 @@ func TestRoutingClientKeepsSourceAndPlatformObjectsSeparate(t *testing.T) { if err := corev1.AddToScheme(scheme); err != nil { t.Fatal(err) } + if err := dnsv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } source := fake.NewClientBuilder().WithScheme(scheme).Build() platform := fake.NewClientBuilder().WithScheme(scheme).Build() routed := &RoutingClient{Client: platform, Source: source, PlatformNamespace: "internal-dns-system"} ctx := context.Background() - projectObject := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "input", Namespace: "project-a"}} + projectObject := &dnsv1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "input", Namespace: "internal-dns-system"}} platformObject := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "manifest", Namespace: "internal-dns-system"}} if err := routed.Create(ctx, projectObject); err != nil { t.Fatal(err) @@ -31,18 +35,19 @@ func TestRoutingClientKeepsSourceAndPlatformObjectsSeparate(t *testing.T) { t.Fatal(err) } for _, tc := range []struct { - name string - cl client.Client - key client.ObjectKey - found bool + name string + cl client.Client + key client.ObjectKey + found bool + object client.Object }{ - {"project in source", source, client.ObjectKeyFromObject(projectObject), true}, - {"project absent from platform", platform, client.ObjectKeyFromObject(projectObject), false}, - {"artifact in platform", platform, client.ObjectKeyFromObject(platformObject), true}, - {"artifact absent from source", source, client.ObjectKeyFromObject(platformObject), false}, + {"project in source", source, client.ObjectKeyFromObject(projectObject), true, &dnsv1.DNSZone{}}, + {"project absent from platform", platform, client.ObjectKeyFromObject(projectObject), false, &dnsv1.DNSZone{}}, + {"artifact in platform", platform, client.ObjectKeyFromObject(platformObject), true, &corev1.ConfigMap{}}, + {"artifact absent from source", source, client.ObjectKeyFromObject(platformObject), false, &corev1.ConfigMap{}}, } { t.Run(tc.name, func(t *testing.T) { - err := tc.cl.Get(ctx, tc.key, &corev1.ConfigMap{}) + err := tc.cl.Get(ctx, tc.key, tc.object) if (err == nil) != tc.found { t.Fatalf("found=%v want=%v err=%v", err == nil, tc.found, err) } diff --git a/internal/internaldns/runtime/run.go b/internal/internaldns/runtime/run.go index 2cdea5b4..586c56e0 100644 --- a/internal/internaldns/runtime/run.go +++ b/internal/internaldns/runtime/run.go @@ -22,6 +22,7 @@ import ( "go.miloapis.com/dns-operator/internal/internaldns/serving" "go.miloapis.com/dns-operator/internal/internaldns/transport" "go.miloapis.com/dns-operator/internal/internaldns/watchdog" + resourcemanagerv1alpha1 "go.miloapis.com/milo/pkg/apis/resourcemanager/v1alpha1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -29,7 +30,7 @@ import ( "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/util/flowcontrol" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -85,26 +86,26 @@ func runControlPlane(ctx context.Context, cfg Config, kubeconfig string) error { } identity := holderIdentity() - projects := make([]projectRuntime, 0, len(cfg.Projects)) - for _, p := range cfg.Projects { - projectREST := rest.CopyConfig(platformREST) + makeProject := func(p ProjectConfig, sourceREST *rest.Config) (projectRuntime, error) { + projectREST := rest.CopyConfig(sourceREST) if p.Kubeconfig != "" { - projectREST, err = restConfig(p.Kubeconfig) - if err != nil { - return fmt.Errorf("load project %q kubeconfig: %w", p.Name, err) + loaded, loadErr := restConfig(p.Kubeconfig) + if loadErr != nil { + return projectRuntime{}, fmt.Errorf("load project %q kubeconfig: %w", p.Name, loadErr) } + projectREST = loaded } applyKubeAPILimit(projectREST, p.KubeAPI) direct, err := client.New(projectREST, client.Options{Scheme: scheme}) if err != nil { - return fmt.Errorf("create project %q direct client: %w", p.Name, err) + return projectRuntime{}, fmt.Errorf("create project %q direct client: %w", p.Name, err) } direct = &deadlineClient{Client: direct, Timeout: p.KubeAPI.requestTimeout()} project := p start := func(startCtx context.Context) error { mgr, err := ctrl.NewManager(projectREST, ctrl.Options{ Scheme: scheme, - Cache: cache.Options{DefaultNamespaces: map[string]cache.Config{project.Namespace: {}}}, + Cache: projectCache(project.Namespace), Metrics: metricsserver.Options{BindAddress: "0"}, HealthProbeBindAddress: "0", LeaderElection: false, @@ -129,7 +130,30 @@ func runControlPlane(ctx context.Context, cfg Config, kubeconfig string) error { } return mgr.Start(startCtx) } - projects = append(projects, projectRuntime{config: project, direct: direct, start: start}) + return projectRuntime{config: project, direct: direct, start: start}, nil + } + projects := make([]projectRuntime, 0, len(cfg.Projects)) + for _, p := range cfg.Projects { + project, err := makeProject(p, platformREST) + if err != nil { + return err + } + projects = append(projects, project) + } + directory := newProjectDirectory(projects) + var discovery *projectDiscovery + if cfg.ProjectDiscovery != nil { + discoveryREST, loadErr := restConfig(cfg.ProjectDiscovery.Kubeconfig) + if loadErr != nil { + return fmt.Errorf("load project discovery kubeconfig: %w", loadErr) + } + applyKubeAPILimit(discoveryREST, cfg.ProjectDiscovery.KubeAPI) + discoveryClient, clientErr := client.New(discoveryREST, client.Options{Scheme: scheme}) + if clientErr != nil { + return fmt.Errorf("create project discovery client: %w", clientErr) + } + discoveryClient = &deadlineClient{Client: discoveryClient, Timeout: cfg.ProjectDiscovery.KubeAPI.requestTimeout()} + discovery = &projectDiscovery{client: discoveryClient, directory: directory, rest: discoveryREST, kubeAPI: cfg.ProjectDiscovery.KubeAPI, makeProject: makeProject} } var admissionManager ctrl.Manager @@ -145,13 +169,16 @@ func runControlPlane(ctx context.Context, cfg Config, kubeconfig string) error { return fmt.Errorf("create admission manager: %w", err) } handler := &dnsadmission.Handler{ - ClientForRequest: projectSelector(projects), + ClientForRequest: directory.selectProject, } admissionManager.GetWebhookServer().Register(dnsadmission.Path, &kubeadmission.Webhook{Handler: handler}) } errCh := make(chan error, len(projects)+len(planners)*3+8) startProjectSources(ctx, projects) + if discovery != nil { + go retryLoop(ctx, time.Duration(cfg.ProjectDiscovery.IntervalSeconds)*time.Second, "project discovery", discovery.step, errCh) + } if admissionManager != nil { go func() { errCh <- admissionManager.Start(ctx) }() } @@ -161,10 +188,7 @@ func runControlPlane(ctx context.Context, cfg Config, kubeconfig string) error { for _, planner := range planners { publicationRegions[planner.Region+"/"+planner.Shard] = append([]model.Member(nil), planner.Members...) } - projectScopes := make([]platform.ProjectScope, 0, len(projects)) - for _, p := range projects { - projectScopes = append(projectScopes, platform.ProjectScope{Client: p.direct, Namespace: p.config.Namespace, RequestTimeout: p.config.KubeAPI.requestTimeout()}) - } + sinks := map[string]*platform.AckSink{} for _, plannerCfg := range planners { pc := plannerCfg @@ -174,7 +198,7 @@ func runControlPlane(ctx context.Context, cfg Config, kubeconfig string) error { interval = time.Second } go retryLoop(ctx, interval, "planner "+pc.Region+"/"+pc.Shard, planner.Step, errCh) - sink := &platform.AckSink{Client: platformClient, Namespace: cfg.PlatformNamespace, Projects: projectScopes, + sink := &platform.AckSink{Client: platformClient, Namespace: cfg.PlatformNamespace, ProjectScopes: directory.scopes, PublicationMembers: publicationMembers, PublicationRegions: publicationRegions, Region: pc.Region, Shard: pc.Shard, Members: pc.Members} sinks[pc.Region+"/"+pc.Shard] = sink go retryLoop(ctx, 2*time.Second, "ack refresh "+pc.Region+"/"+pc.Shard, sink.Refresh, errCh) @@ -383,6 +407,9 @@ func newScheme() (*runtime.Scheme, error) { if err := clientgoscheme.AddToScheme(scheme); err != nil { return nil, err } + if err := resourcemanagerv1alpha1.AddToScheme(scheme); err != nil { + return nil, err + } if err := corev1.AddToScheme(scheme); err != nil { return nil, err } From b5b8575f07d7a1dc2e4b1a44551f0d19c2405f73 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 15:59:15 -0500 Subject: [PATCH 07/23] test: Keep discovery fixtures self-contained --- .../internaldns/controlplane/context_reconciler_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/internaldns/controlplane/context_reconciler_test.go b/internal/internaldns/controlplane/context_reconciler_test.go index 98e196fd..0ef59496 100644 --- a/internal/internaldns/controlplane/context_reconciler_test.go +++ b/internal/internaldns/controlplane/context_reconciler_test.go @@ -10,6 +10,7 @@ import ( dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" "go.miloapis.com/dns-operator/internal/internaldns/model" + corev1 "k8s.io/api/core/v1" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -292,7 +293,7 @@ func TestNamespaceReconciliationCannotWithdrawAnotherNamespace(t *testing.T) { t.Run(fmt.Sprintf("namespace-label-%t", labeled), func(t *testing.T) { ctx := context.Background() scheme := runtime.NewScheme() - if err := addReferenceStateToScheme(scheme); err != nil { + if err := dnsv1alpha1.AddToScheme(scheme); err != nil { t.Fatal(err) } access := &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "access", Namespace: "other", UID: "other-access"}} @@ -318,12 +319,15 @@ func TestNamespaceReconciliationCannotWithdrawAnotherNamespace(t *testing.T) { func TestAllNamespaceReconcileIncludesPlatformNamedSourceNamespace(t *testing.T) { ctx := context.Background() scheme := runtime.NewScheme() - if err := addReferenceStateToScheme(scheme); err != nil { + if err := dnsv1alpha1.AddToScheme(scheme); err != nil { t.Fatal(err) } contextObject := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "context", Namespace: "platform", UID: "context", Generation: 1}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "network"}} cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(contextObject).WithStatusSubresource(contextObject).Build() r := &Reconciler{Client: cl, APIReader: cl, Options: ReconcilerOptions{ProjectUID: "project", PlatformNamespace: "platform"}} + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(contextObject)}); err != nil { t.Fatal(err) } From 9241d114e748856851c82c383f0a980f6564e1ab Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 16:02:26 -0500 Subject: [PATCH 08/23] fix: Retain project sources through cleanup --- internal/internaldns/runtime/project_discovery.go | 4 ++-- .../internaldns/runtime/project_discovery_test.go | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/internal/internaldns/runtime/project_discovery.go b/internal/internaldns/runtime/project_discovery.go index 43d63e9e..81d5e35e 100644 --- a/internal/internaldns/runtime/project_discovery.go +++ b/internal/internaldns/runtime/project_discovery.go @@ -113,8 +113,8 @@ func (d *projectDiscovery) step(ctx context.Context) error { d.directory.mu.RLock() existing, exists := d.directory.projects[p.Name] d.directory.mu.RUnlock() - ready := p.DeletionTimestamp.IsZero() && apimeta.IsStatusConditionTrue(p.Status.Conditions, resourcemanagerv1alpha1.ProjectReady) - if exists && (existing.config.ProjectUID != p.UID || !ready) { + ready := apimeta.IsStatusConditionTrue(p.Status.Conditions, resourcemanagerv1alpha1.ProjectReady) + if exists && existing.config.ProjectUID != p.UID { d.remove(p.Name) exists = false } diff --git a/internal/internaldns/runtime/project_discovery_test.go b/internal/internaldns/runtime/project_discovery_test.go index 6fa884e4..4343247b 100644 --- a/internal/internaldns/runtime/project_discovery_test.go +++ b/internal/internaldns/runtime/project_discovery_test.go @@ -77,6 +77,19 @@ func TestProjectDiscoveryAddsSourcesAndPreservesThemDuringOutage(t *testing.T) { t.Fatal("discovery outage discarded existing sources") } core.unavailable = false + if err := core.Get(ctx, client.ObjectKeyFromObject(first), first); err != nil { + t.Fatal(err) + } + first.Status.Conditions[0].Status = metav1.ConditionFalse + if err := core.Update(ctx, first); err != nil { + t.Fatal(err) + } + if err := d.step(ctx); err != nil { + t.Fatal(err) + } + if len(directory.scopes()) != 2 { + t.Fatal("unavailable project discarded its existing source") + } request := kubeadmission.Request{AdmissionRequest: admissionv1.AdmissionRequest{Namespace: "internal-dns-platform", UserInfo: authenticationv1.UserInfo{Extra: map[string]authenticationv1.ExtraValue{parentNameExtra: {second.Name}}}}} selected, err := directory.selectProject(ctx, request) if err != nil || selected != scopes[1].Client { From 7752b35cb4f227f790e55f26e3aa9f8be52f6588 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 16:09:45 -0500 Subject: [PATCH 09/23] fix: Observe project lifetime before compilation --- config/internal-dns/README.md | 1 + .../controlplane/context_reconciler_test.go | 22 +++++++++++++ .../internaldns/controlplane/reconciler.go | 32 +++++++++++-------- .../internaldns/runtime/project_discovery.go | 12 +++++++ .../runtime/project_discovery_test.go | 16 ++++++++++ internal/internaldns/runtime/run.go | 10 +++++- 6 files changed, 79 insertions(+), 14 deletions(-) diff --git a/config/internal-dns/README.md b/config/internal-dns/README.md index cd58e618..9dcc5f01 100644 --- a/config/internal-dns/README.md +++ b/config/internal-dns/README.md @@ -44,6 +44,7 @@ without restarting the service. `projectDiscovery.kubeconfig` authenticates discovery and project API access; the process `--kubeconfig` authenticates the DNS service project. Generated kubeconfig Secrets use the `value` key. Unavailable discovery retains active sources and applied DNS configuration. +Compilation requires the live source project identity to match its worker. Admission resolves the authenticated project parent against its live identity, and acknowledgement projection follows the discovered source clients. diff --git a/internal/internaldns/controlplane/context_reconciler_test.go b/internal/internaldns/controlplane/context_reconciler_test.go index 0ef59496..bce75e05 100644 --- a/internal/internaldns/controlplane/context_reconciler_test.go +++ b/internal/internaldns/controlplane/context_reconciler_test.go @@ -4,6 +4,7 @@ package controlplane import ( "context" + "errors" "fmt" "testing" "time" @@ -338,3 +339,24 @@ func TestAllNamespaceReconcileIncludesPlatformNamedSourceNamespace(t *testing.T) t.Fatal("source namespace was excluded because it shares the managed namespace name") } } + +func TestUnobservedProjectIdentityPreventsSourceCompilation(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + if err := dnsv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + resolver := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "context", Namespace: "project", UID: "context"}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "network"}} + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(resolver).WithStatusSubresource(resolver).Build() + unavailable := errors.New("identity unavailable") + r := &Reconciler{Client: cl, Options: ReconcilerOptions{ProjectUID: "project", VerifyProjectIdentity: func(context.Context) error { return unavailable }}} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(resolver)}); !errors.Is(err, unavailable) { + t.Fatalf("identity error not preserved: %v", err) + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(resolver), resolver); err != nil { + t.Fatal(err) + } + if len(resolver.Status.Conditions) != 0 { + t.Fatal("unobserved identity changed source state") + } +} diff --git a/internal/internaldns/controlplane/reconciler.go b/internal/internaldns/controlplane/reconciler.go index 9ca5a66b..5c5f2e97 100644 --- a/internal/internaldns/controlplane/reconciler.go +++ b/internal/internaldns/controlplane/reconciler.go @@ -19,19 +19,20 @@ import ( ) type ReconcilerOptions struct { - ProjectUID types.UID - ProjectNamespace string - PlatformNamespace string - Region string - Shard string - Identity string - ChunkSize int - LeaseDuration time.Duration - Now func() time.Time - AddressAllocator AddressAllocator - ManagedDomainSuffix string - PrivateZoneClassName string - ServingRegions []ServingRegion + ProjectUID types.UID + VerifyProjectIdentity func(context.Context) error + ProjectNamespace string + PlatformNamespace string + Region string + Shard string + Identity string + ChunkSize int + LeaseDuration time.Duration + Now func() time.Time + AddressAllocator AddressAllocator + ManagedDomainSuffix string + PrivateZoneClassName string + ServingRegions []ServingRegion } type ServingRegion struct { @@ -101,6 +102,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu if r.Options.ProjectUID == "" { return ctrl.Result{}, errors.New("projectUID is required") } + if r.Options.VerifyProjectIdentity != nil { + if err := r.Options.VerifyProjectIdentity(ctx); err != nil { + return ctrl.Result{}, err + } + } ns := r.Options.ProjectNamespace if ns == "" { ns = req.Namespace diff --git a/internal/internaldns/runtime/project_discovery.go b/internal/internaldns/runtime/project_discovery.go index 81d5e35e..62cb1349 100644 --- a/internal/internaldns/runtime/project_discovery.go +++ b/internal/internaldns/runtime/project_discovery.go @@ -12,6 +12,7 @@ import ( "go.miloapis.com/dns-operator/internal/internaldns/platform" resourcemanagerv1alpha1 "go.miloapis.com/milo/pkg/apis/resourcemanager/v1alpha1" apimeta "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" @@ -179,3 +180,14 @@ func projectCache(namespace string) cache.Options { } return cache.Options{DefaultNamespaces: map[string]cache.Config{namespace: {}}} } + +func verifyProjectIdentity(ctx context.Context, reader client.Reader, name string, expected types.UID) error { + var project resourcemanagerv1alpha1.Project + if err := reader.Get(ctx, client.ObjectKey{Name: name}, &project); err != nil { + return fmt.Errorf("observe source project identity: %w", err) + } + if project.UID != expected { + return errors.New("source project lifetime changed") + } + return nil +} diff --git a/internal/internaldns/runtime/project_discovery_test.go b/internal/internaldns/runtime/project_discovery_test.go index 4343247b..9440587e 100644 --- a/internal/internaldns/runtime/project_discovery_test.go +++ b/internal/internaldns/runtime/project_discovery_test.go @@ -24,6 +24,13 @@ type discoveryFixtureClient struct { unavailable bool } +func (c *discoveryFixtureClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if c.unavailable { + return errors.New("discovery unavailable") + } + return c.Client.Get(ctx, key, obj, opts...) +} + func (c *discoveryFixtureClient) List(ctx context.Context, obj client.ObjectList, opts ...client.ListOption) error { if c.unavailable { return errors.New("discovery unavailable") @@ -69,7 +76,13 @@ func TestProjectDiscoveryAddsSourcesAndPreservesThemDuringOutage(t *testing.T) { if len(scopes) != 2 || scopes[0].Namespace != "" || scopes[1].Namespace != "" { t.Fatalf("ACK scopes did not follow discovery: %#v", scopes) } + if err := verifyProjectIdentity(ctx, core, first.Name, first.UID); err != nil { + t.Fatal(err) + } core.unavailable = true + if err := verifyProjectIdentity(ctx, core, first.Name, first.UID); err == nil { + t.Fatal("unobserved source identity permitted compilation") + } if err := d.step(ctx); err == nil { t.Fatal("expected discovery error") } @@ -105,6 +118,9 @@ func TestProjectDiscoveryAddsSourcesAndPreservesThemDuringOutage(t *testing.T) { if err := core.Create(ctx, recreated); err != nil { t.Fatal(err) } + if err := verifyProjectIdentity(ctx, core, recreated.Name, second.UID); err == nil { + t.Fatal("recreated project used previous compilation identity") + } if _, err := directory.selectProject(ctx, request); err == nil { t.Fatal("recreated project used previous source") } diff --git a/internal/internaldns/runtime/run.go b/internal/internaldns/runtime/run.go index 586c56e0..fbf8f535 100644 --- a/internal/internaldns/runtime/run.go +++ b/internal/internaldns/runtime/run.go @@ -86,6 +86,7 @@ func runControlPlane(ctx context.Context, cfg Config, kubeconfig string) error { } identity := holderIdentity() + var discoveryReader client.Reader makeProject := func(p ProjectConfig, sourceREST *rest.Config) (projectRuntime, error) { projectREST := rest.CopyConfig(sourceREST) if p.Kubeconfig != "" { @@ -118,7 +119,13 @@ func runControlPlane(ctx context.Context, cfg Config, kubeconfig string) error { // contexts, access bindings, associations, registrations, or producer observations. routed := &RoutingClient{Client: platformClient, Source: direct, PlatformNamespace: cfg.PlatformNamespace} reconciler := &controlplane.Reconciler{Client: routed, APIReader: direct, Scheme: scheme, Options: controlplane.ReconcilerOptions{ - ProjectUID: project.ProjectUID, + ProjectUID: project.ProjectUID, + VerifyProjectIdentity: func(ctx context.Context) error { + if discoveryReader == nil { + return nil + } + return verifyProjectIdentity(ctx, discoveryReader, project.Name, project.ProjectUID) + }, ProjectNamespace: project.Namespace, PlatformNamespace: cfg.PlatformNamespace, Region: cfg.Region, Shard: cfg.Shard, Identity: identity, LeaseDuration: cfg.ownershipLease(), @@ -153,6 +160,7 @@ func runControlPlane(ctx context.Context, cfg Config, kubeconfig string) error { return fmt.Errorf("create project discovery client: %w", clientErr) } discoveryClient = &deadlineClient{Client: discoveryClient, Timeout: cfg.ProjectDiscovery.KubeAPI.requestTimeout()} + discoveryReader = discoveryClient discovery = &projectDiscovery{client: discoveryClient, directory: directory, rest: discoveryREST, kubeAPI: cfg.ProjectDiscovery.KubeAPI, makeProject: makeProject} } From 78c1f200226bf8ea5247c81b9218f52cd49e9996 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 17:08:55 -0500 Subject: [PATCH 10/23] Run DNS daemons and readiness independently of the agent --- cmd/internal-dns/main.go | 4 +- config/internal-dns/Dockerfile | 6 +-- config/internal-dns/README.md | 27 +++++----- .../commands/internal-dns-daemons | 30 ----------- .../internal-dns/commands/internal-dns-gate | 13 +++++ config/internal-dns/fleet.example.yaml | 53 +++++++++---------- internal/internaldns/runtime/config.go | 17 +++--- internal/internaldns/runtime/config_test.go | 18 +++++-- .../runtime/deployment_examples_test.go | 2 +- internal/internaldns/runtime/run.go | 9 ++-- 10 files changed, 91 insertions(+), 88 deletions(-) delete mode 100755 config/internal-dns/commands/internal-dns-daemons diff --git a/cmd/internal-dns/main.go b/cmd/internal-dns/main.go index 3c8d7c74..efe6fc8e 100644 --- a/cmd/internal-dns/main.go +++ b/cmd/internal-dns/main.go @@ -15,12 +15,12 @@ import ( func main() { var role, configPath, kubeconfig string - flag.StringVar(&role, "role", "", "Process role: control-plane, agent, or watchdog") + flag.StringVar(&role, "role", "", "Process role: control-plane, agent, serving, or readiness") flag.StringVar(&configPath, "config", "", "Absolute path to the internal DNS JSON configuration") // controller-runtime/client-go may install this standard flag from an init // hook. Reuse it when present so the command keeps one stable contract. if flag.Lookup("kubeconfig") == nil { - flag.StringVar(&kubeconfig, "kubeconfig", "", "Control-plane Kubernetes configuration; ignored by agent and watchdog") + flag.StringVar(&kubeconfig, "kubeconfig", "", "Control-plane Kubernetes configuration; ignored by serving processes") } logOptions := zap.Options{Development: false} logOptions.BindFlags(flag.CommandLine) diff --git a/config/internal-dns/Dockerfile b/config/internal-dns/Dockerfile index 6416aa75..e739a004 100644 --- a/config/internal-dns/Dockerfile +++ b/config/internal-dns/Dockerfile @@ -21,14 +21,14 @@ ENTRYPOINT ["/internal-dns"] FROM debian:trixie-slim AS serving RUN apt-get update && apt-get install --no-install-recommends -y \ - bind9 bind9-utils dnsdist iproute2 procps netcat-openbsd ca-certificates bash jq && \ + bind9 bind9-utils dnsdist iproute2 procps netcat-openbsd ca-certificates bash && \ rm -rf /var/lib/apt/lists/* COPY --from=build /out/internal-dns /internal-dns COPY config/internal-dns/commands/ /usr/local/sbin/ USER 65532:65532 -RUN for tool in bash jq date ip nc named named-checkconf named-checkzone dnsdist rndc; do \ +RUN for tool in bash ip nc named named-checkconf named-checkzone dnsdist rndc; do \ command -v "$tool" >/dev/null || exit 1; \ - done && jq --version && test -x /internal-dns + done && test -x /internal-dns RUN for helper in /usr/local/sbin/internal-dns-*; do \ test -x "$helper" && bash -n "$helper" || exit 1; \ done diff --git a/config/internal-dns/README.md b/config/internal-dns/README.md index 9dcc5f01..10d4e977 100644 --- a/config/internal-dns/README.md +++ b/config/internal-dns/README.md @@ -32,7 +32,7 @@ these processes. They never create workloads per VPC or zone. The legacy used for the multi-member production layout. Each member needs a distinct member ID, NATS identity and durable consumers, -checkpoint PVC, configuration PVC, and independent watchdog lease. Treat that +checkpoint PVC, configuration PVC, and durable activation state. Treat that set as a single-writer unit. A rolling update must allocate a new member identity and storage; do not run two pods against one checkpoint. @@ -135,18 +135,21 @@ from persistent checkpoints before waiting for messaging connectivity. New nodes require a complete configuration before accepting queries. Replay cannot regress writer or configuration versions or restore a withdrawn lifetime. -The local watchdog lease is at most five seconds. An independent health loop -checks local DNS daemons over UDP and TCP and renews the lease for the activated -configuration. Staging and validation can continue while that configuration -serves. A rejected expansion retains the activated configuration; a withdrawal -that cannot be applied gates serving. The gate runs without Kubernetes or NATS. +Readiness verifies the approved checkpoint and installed files, then queries +local DNS daemons over UDP and TCP. Healthy daemons continue through normal +agent stops and restarts. Staging and validation can continue while approved +configuration serves. A rejected expansion retains that configuration; a +withdrawal that cannot be applied gates serving. The serving supervisor accepts +approved configuration independently of the agent. Incomplete installation +keeps a durable activation marker; candidate verification requires a live +transaction lock until the approved checkpoint is committed. Orphaned +activation remains gated until recovery verifies the saved configuration. +Readiness and activation gates use local state without Kubernetes or NATS. Mount the transaction stage, active BIND configuration, zone files, checkpoint, -and watchdog lease on low-latency storage local to the serving host. Do not put -the hot transaction or lease path on a desktop file-sharing mount or a remote -filesystem whose fsync latency can consume the watchdog budget. Replication is -provided by the durable publication stream and independent regional members; -each member still has its own crash-safe checkpoint volume. +and activation state on storage local to the serving host. Replication uses +the durable publication stream and independent regional members; each member +has its own crash-safe checkpoint volume. The regional BIND reload helper must reload existing primary zones, for example with `rndc reload`. `rndc reconfig` only guarantees configuration and @@ -157,7 +160,7 @@ deployment's normal configuration reload because that BIND tier is recursive. ## Capacity and transport Set `maxBindings` from measured BIND view memory, validation/reload time, -dnsdist rule cost, NATS payload size, and watchdog timing. The example caps a +dnsdist rule cost, NATS payload size, and activation recovery time. The example caps a shard at 200 contexts and budgets 2 MiB per view. Add another fixed shard before raising those bounds. diff --git a/config/internal-dns/commands/internal-dns-daemons b/config/internal-dns/commands/internal-dns-daemons deleted file mode 100755 index d530d7c8..00000000 --- a/config/internal-dns/commands/internal-dns-daemons +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/sh -# One member's daemon supervisor. Only its local agent can enable a listener. -set -eu -role=$1 -supervise() { - daemon=$1 - config=$2 - shift 2 - while :; do - if [ -f "/run/internal-dns/allow-$daemon" ] && [ -s "$config" ]; then - "$@" || true - fi - sleep 0.05 - done -} -trap 'pkill -TERM -x named || true; pkill -TERM -x dnsdist || true; exit 0' INT TERM -case "$role" in - node) - supervise named /etc/internal-dns/node-bind.conf named -g -c /etc/internal-dns/node-bind.conf -n 2 & - supervise dnsdist /etc/internal-dns/node-dnsdist.conf dnsdist --supervised -C /etc/internal-dns/node-dnsdist.conf & - ;; - regional-dnsdist) - supervise dnsdist /etc/internal-dns/cluster-dnsdist.conf dnsdist --supervised -C /etc/internal-dns/cluster-dnsdist.conf & - ;; - regional-bind) - supervise named /etc/internal-dns/cluster-bind.conf named -g -c /etc/internal-dns/cluster-bind.conf -n 2 & - ;; - *) exit 2 ;; -esac -wait diff --git a/config/internal-dns/commands/internal-dns-gate b/config/internal-dns/commands/internal-dns-gate index c2ad3563..bf46ac9a 100755 --- a/config/internal-dns/commands/internal-dns-gate +++ b/config/internal-dns/commands/internal-dns-gate @@ -4,3 +4,16 @@ set -eu rm -f /run/internal-dns/allow-named /run/internal-dns/allow-dnsdist pkill -TERM -x named || true pkill -TERM -x dnsdist || true +for daemon in named dnsdist; do + for attempt in $(seq 1 40); do + if ! pgrep -x "$daemon" >/dev/null; then break; fi + sleep 0.05 + done + if pgrep -x "$daemon" >/dev/null; then + pkill -KILL -x "$daemon" || true + fi + if pgrep -x "$daemon" >/dev/null; then + echo "DNS process did not stop: $daemon" >&2 + exit 1 + fi +done diff --git a/config/internal-dns/fleet.example.yaml b/config/internal-dns/fleet.example.yaml index ca6638c4..d53b3e8c 100644 --- a/config/internal-dns/fleet.example.yaml +++ b/config/internal-dns/fleet.example.yaml @@ -1,6 +1,6 @@ # Fixed shared fleet members for one regional shard. Deploy at least two copies # of the regional-bind member with distinct IDs, addresses, state, config, NATS -# credentials, and watchdog leases. Contexts and zones never create workloads. +# credentials, and activation state. Contexts and zones never create workloads. apiVersion: v1 kind: ConfigMap metadata: {name: internal-dns-node-0-config, namespace: internal-dns-system} @@ -12,10 +12,9 @@ data: "agent":{ "region":"us-central1","shard":"shared-0","memberID":"node-us-central1-0","replicaID":"node-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"resolver", "render":{"role":"node","nodeDNSDist":{"path":"/etc/internal-dns/node-dnsdist.conf","listenAddress":"[::]:53","acls":["fd71:100::/64","::1/128"],"poolName":"node_shared","backends":[{"memberID":"node-bind-0","address":"fd70:10::10","port":5300}]},"nodeBIND":{"path":"/etc/internal-dns/node-bind.conf","listenAddress":"fd70:10::10","port":5300,"proxyPeers":["fd70:10::20","fd70:10::10"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, - "transaction":{"stageDir":"/var/lib/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-dnsdist","args":["node","{stageDir}/node-dnsdist.conf"],"whenChanged":["/etc/internal-dns/node-dnsdist.conf"]},{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["node","{stageDir}/node-bind.conf"],"whenChanged":["/etc/internal-dns/node-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["node"],"whenChanged":["/etc/internal-dns/node-bind.conf"]},{"path":"/usr/local/sbin/internal-dns-reload-dnsdist","args":["node"],"whenChanged":["/etc/internal-dns/node-dnsdist.conf"]}]}, - "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["node","{name}"]}],"reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"resolverDNSProbe":{"timeout":3000000000},"resolverVerifyConcurrency":16 - }, - "watchdog":{"leasePath":"/run/internal-dns/lease.json","memberID":"node-us-central1-0","replicaID":"node-us-central1-0","maxLeaseSeconds":5,"startupGraceSeconds":0,"intervalMillis":250,"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["node"]}]} + "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-dnsdist","args":["node","{stageDir}/node-dnsdist.conf"],"whenChanged":["/etc/internal-dns/node-dnsdist.conf"]},{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["node","{stageDir}/node-bind.conf"],"whenChanged":["/etc/internal-dns/node-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["node"],"whenChanged":["/etc/internal-dns/node-bind.conf"]},{"path":"/usr/local/sbin/internal-dns-reload-dnsdist","args":["node"],"whenChanged":["/etc/internal-dns/node-dnsdist.conf"]}],"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["node"]}]}, + "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["node","{name}"]}],"reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"resolverDNSProbe":{"timeout":3000000000},"resolverVerifyConcurrency":16 + } } --- apiVersion: v1 @@ -29,10 +28,9 @@ data: "agent":{ "region":"us-central1","shard":"shared-0","memberID":"regional-front-us-central1-0","replicaID":"regional-front-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"resolver", "render":{"role":"regional-dnsdist","clusterDNSDist":{"path":"/etc/internal-dns/cluster-dnsdist.conf","listenAddress":"[::]:53","acls":["fd71:200::/64","::1/128"],"poolName":"regional_shared","backends":[{"memberID":"regional-bind-us-central1-0","address":"fd70:20::10","port":5300},{"memberID":"regional-bind-us-central1-1","address":"fd70:20::11","port":5300}]}}, - "transaction":{"stageDir":"/var/lib/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-dnsdist","args":["regional","{stageDir}/cluster-dnsdist.conf"],"whenChanged":["/etc/internal-dns/cluster-dnsdist.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-dnsdist","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-dnsdist.conf"]}]}, - "reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"resolverDNSProbe":{"timeout":3000000000},"resolverVerifyConcurrency":16 - }, - "watchdog":{"leasePath":"/run/internal-dns/lease.json","memberID":"regional-front-us-central1-0","replicaID":"regional-front-us-central1-0","maxLeaseSeconds":5,"startupGraceSeconds":0,"intervalMillis":250,"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["regional-front"]}]} + "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-dnsdist","args":["regional","{stageDir}/cluster-dnsdist.conf"],"whenChanged":["/etc/internal-dns/cluster-dnsdist.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-dnsdist","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-dnsdist.conf"]}],"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["regional-front"]}]}, + "reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"resolverDNSProbe":{"timeout":3000000000},"resolverVerifyConcurrency":16 + } } --- apiVersion: v1 @@ -46,14 +44,13 @@ data: "agent":{ "region":"us-central1","shard":"shared-0","memberID":"regional-bind-us-central1-0","replicaID":"regional-bind-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"combined", "render":{"role":"regional-bind","clusterBIND":{"path":"/etc/internal-dns/cluster-bind.conf","listenAddress":"fd70:20::10","port":5300,"proxyPeers":["fd70:20::20","fd70:20::30","fd70:20::10"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, - "transaction":{"stageDir":"/var/lib/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["regional","-z","{stageDir}/cluster-bind.conf"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}]}, - "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["regional","{name}"]}],"reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"publicationDNSProbe":{"server":"[fd70:20::10]:5300","timeout":3000000000} - }, - "watchdog":{"leasePath":"/run/internal-dns/lease.json","memberID":"regional-bind-us-central1-0","replicaID":"regional-bind-us-central1-0","maxLeaseSeconds":5,"startupGraceSeconds":0,"intervalMillis":250,"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["regional-bind"]}]} + "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["regional","-z","{stageDir}/cluster-bind.conf"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}],"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["regional-bind"]}]}, + "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["regional","{name}"]}],"reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"publicationDNSProbe":{"server":"[fd70:20::10]:5300","timeout":3000000000} + } } --- # The second regional member has independent identity, listener, state, -# checkpoint, and watchdog storage. Its backend memberID exactly matches the +# checkpoint, and activation storage. Its backend memberID exactly matches the # second clusterBackends entry in the control-plane configuration. apiVersion: v1 kind: ConfigMap @@ -66,10 +63,9 @@ data: "agent":{ "region":"us-central1","shard":"shared-0","memberID":"regional-bind-us-central1-1","replicaID":"regional-bind-us-central1-1","stateDir":"/var/lib/internal-dns","mode":"combined", "render":{"role":"regional-bind","clusterBIND":{"path":"/etc/internal-dns/cluster-bind.conf","listenAddress":"fd70:20::11","port":5300,"proxyPeers":["fd70:20::20","fd70:20::31","fd70:20::11"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, - "transaction":{"stageDir":"/var/lib/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["regional","-z","{stageDir}/cluster-bind.conf"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}]}, - "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["regional","{name}"]}],"reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"publicationDNSProbe":{"server":"[fd70:20::11]:5300","timeout":3000000000} - }, - "watchdog":{"leasePath":"/run/internal-dns/lease.json","memberID":"regional-bind-us-central1-1","replicaID":"regional-bind-us-central1-1","maxLeaseSeconds":5,"startupGraceSeconds":0,"intervalMillis":250,"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["regional-bind"]}]} + "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["regional","-z","{stageDir}/cluster-bind.conf"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}],"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["regional-bind"]}]}, + "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["regional","{name}"]}],"reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"publicationDNSProbe":{"server":"[fd70:20::11]:5300","timeout":3000000000} + } } --- # This Deployment template is instantiated once for each ConfigMap above and a @@ -98,26 +94,29 @@ spec: volumeMounts: &member-mounts - {name: config, mountPath: /config, readOnly: true} - {name: nats, mountPath: /var/run/internal-dns/nats, readOnly: true} - - {name: lease, mountPath: /run/internal-dns} + - {name: activation, mountPath: /run/internal-dns} - {name: state, mountPath: /var/lib/internal-dns} - {name: daemon-config, mountPath: /etc/internal-dns} - - name: watchdog - image: example.invalid/dns-operator/internal-dns-serving:VERSION - args: ["--role=watchdog","--config=/config/config.json"] - securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, runAsNonRoot: true, capabilities: {drop: [ALL]}} - volumeMounts: *member-mounts - name: serving - image: example.invalid/dns-operator/internal-dns-bind:VERSION - args: ["regional-bind"] + image: example.invalid/dns-operator/internal-dns-serving:VERSION + command: ["/internal-dns"] + args: ["--role=serving","--config=/config/config.json"] + readinessProbe: + exec: {command: ["/internal-dns", "--role=readiness", "--config=/config/config.json"]} + periodSeconds: 2 + timeoutSeconds: 2 securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, runAsUser: 65532, runAsGroup: 65532, capabilities: {drop: [ALL], add: [NET_BIND_SERVICE]}} volumeMounts: + - {name: config, mountPath: /config, readOnly: true} + - {name: state, mountPath: /var/lib/internal-dns} + - {name: activation, mountPath: /run/internal-dns} - {name: daemon-config, mountPath: /etc/internal-dns} # BIND requires a writable working directory with a read-only root filesystem. - {name: runtime, mountPath: /tmp} volumes: - {name: config, configMap: {name: internal-dns-regional-bind-0-config}} - {name: nats, secret: {secretName: internal-dns-regional-bind-0-nats}} - - {name: lease, emptyDir: {medium: Memory}} + - {name: activation, emptyDir: {medium: Memory}} - {name: runtime, emptyDir: {}} - {name: daemon-config, persistentVolumeClaim: {claimName: internal-dns-regional-bind-0-config}} - {name: state, persistentVolumeClaim: {claimName: internal-dns-regional-bind-0-state}} diff --git a/internal/internaldns/runtime/config.go b/internal/internaldns/runtime/config.go index a5a4d024..8d1e9fae 100644 --- a/internal/internaldns/runtime/config.go +++ b/internal/internaldns/runtime/config.go @@ -14,12 +14,12 @@ import ( "go.miloapis.com/dns-operator/internal/internaldns/platform" "go.miloapis.com/dns-operator/internal/internaldns/serving" "go.miloapis.com/dns-operator/internal/internaldns/transport" - "go.miloapis.com/dns-operator/internal/internaldns/watchdog" "k8s.io/apimachinery/pkg/types" ) const ( - roleWatchdog = "watchdog" + roleReadiness = "readiness" + roleServing = "serving" roleControlPlane = "control-plane" memberRoleRegional = "regional" roleAgent = "agent" @@ -80,7 +80,6 @@ type Config struct { ManagedDomainSuffix string `json:"managedDomainSuffix,omitempty"` PrivateZoneClassName string `json:"privateZoneClassName,omitempty"` Agent serving.Config `json:"agent,omitempty"` - Watchdog watchdog.Config `json:"watchdog,omitempty"` Shards []platform.PlannerConfig `json:"shards,omitempty"` } @@ -165,12 +164,18 @@ func validateKubeAPI(name string, c KubeAPIClientConfig) error { func (c *Config) Validate(role string) error { c.defaults() switch role { - case roleControlPlane, roleAgent, roleWatchdog: + case roleControlPlane, roleAgent, roleReadiness, roleServing: default: return fmt.Errorf("unsupported role %q", role) } - if role == roleWatchdog { - return c.Watchdog.Validate() + if role == roleReadiness || role == roleServing { + if c.Agent.Region == "" { + c.Agent.Region = c.Region + } + if c.Agent.Shard == "" { + c.Agent.Shard = c.Shard + } + return c.Agent.Validate() } if len(c.RegionalTransports) == 0 { if err := validateTransportConfig(c.NATS, c.Stream, c.EnsureStream); err != nil { diff --git a/internal/internaldns/runtime/config_test.go b/internal/internaldns/runtime/config_test.go index 6ce6a902..c271e30e 100644 --- a/internal/internaldns/runtime/config_test.go +++ b/internal/internaldns/runtime/config_test.go @@ -155,7 +155,7 @@ func TestConfigLoaderRejectsRemovedPrivateServingFields(t *testing.T) { func TestAgentRestoresCheckpointWhileBrokerUnavailable(t *testing.T) { dir := t.TempDir() - cfg := Config{Region: "central", Shard: "one", NATS: transport.Config{URL: "nats://127.0.0.1:1", AllowInsecure: true}, Stream: transport.StreamConfig{Name: "DNS", SnapshotsBucket: "DNS_BOOTSTRAP"}, Agent: serving.Config{MemberID: "node-0", StateDir: filepath.Join(dir, "state"), Mode: serving.ModeResolver, Render: serving.RenderConfig{Role: serving.RenderNode, NodeDNSDist: serving.DNSDistRenderConfig{Path: filepath.Join(dir, "dnsdist.conf"), ListenAddress: "[::1]:53", PoolName: "node"}, NodeBIND: serving.BINDRenderConfig{Path: filepath.Join(dir, "named.conf"), ListenAddress: "::1", Port: 5353}}, Transaction: serving.TransactionConfig{StageDir: filepath.Join(dir, "stage")}, UnsafeAllowNoFailClosed: true, UnsafeAllowNoDNSVerify: true, WatchdogLeasePath: filepath.Join(dir, "lease.json"), ReconcileInterval: 10 * time.Millisecond}} + cfg := Config{Region: "central", Shard: "one", NATS: transport.Config{URL: "nats://127.0.0.1:1", AllowInsecure: true}, Stream: transport.StreamConfig{Name: "DNS", SnapshotsBucket: "DNS_BOOTSTRAP"}, Agent: serving.Config{MemberID: "node-0", StateDir: filepath.Join(dir, "state"), Mode: serving.ModeResolver, Render: serving.RenderConfig{Role: serving.RenderNode, NodeDNSDist: serving.DNSDistRenderConfig{Path: filepath.Join(dir, "dnsdist.conf"), ListenAddress: "[::1]:53", PoolName: "node"}, NodeBIND: serving.BINDRenderConfig{Path: filepath.Join(dir, "named.conf"), ListenAddress: "::1", Port: 5353}}, Transaction: serving.TransactionConfig{StageDir: filepath.Join(dir, "stage")}, UnsafeAllowNoFailClosed: true, UnsafeAllowNoDNSVerify: true, ReconcileInterval: 10 * time.Millisecond}} if err := cfg.Validate(roleAgent); err != nil { t.Fatal(err) } @@ -171,7 +171,7 @@ func TestAgentRestoresCheckpointWhileBrokerUnavailable(t *testing.T) { if err := seed.Handle(context.Background(), env); err != nil { t.Fatal(err) } - if err := os.Remove(cfg.Agent.WatchdogLeasePath); err != nil { + if err := os.Remove(cfg.Agent.Render.NodeDNSDist.Path); err != nil { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) @@ -180,7 +180,7 @@ func TestAgentRestoresCheckpointWhileBrokerUnavailable(t *testing.T) { go func() { done <- runAgent(ctx, cfg) }() deadline := time.Now().Add(3 * time.Second) for { - if _, err := os.Stat(cfg.Agent.WatchdogLeasePath); err == nil { + if _, err := os.Stat(cfg.Agent.Render.NodeDNSDist.Path); err == nil { break } if time.Now().After(deadline) { @@ -225,3 +225,15 @@ func TestControlPlaneProjectDiscoveryValidation(t *testing.T) { t.Fatal("relative discovery credential accepted") } } + +func TestLocalServingRolesDoNotRequireDeliveryConfiguration(t *testing.T) { + for _, role := range []string{"serving", "readiness"} { + t.Run(role, func(t *testing.T) { + dir := t.TempDir() + cfg := Config{Region: "central", Shard: "one", Agent: serving.Config{MemberID: "node", StateDir: filepath.Join(dir, "state"), Mode: serving.ModeResolver, CacheFlush: []serving.Command{{Path: "/bin/true"}}, Render: serving.RenderConfig{Role: serving.RenderNode, NodeDNSDist: serving.DNSDistRenderConfig{Path: filepath.Join(dir, "dnsdist.conf"), ListenAddress: "[::1]:53", PoolName: "node"}, NodeBIND: serving.BINDRenderConfig{Path: filepath.Join(dir, "named.conf"), ListenAddress: "::1", Port: 5353}}, Transaction: serving.TransactionConfig{StageDir: filepath.Join(dir, "stage"), FailClosed: []serving.Command{{Path: "/bin/true"}}}}} + if err := cfg.Validate(role); err != nil { + t.Fatalf("local role required messaging or control-plane configuration: %v", err) + } + }) + } +} diff --git a/internal/internaldns/runtime/deployment_examples_test.go b/internal/internaldns/runtime/deployment_examples_test.go index 94ea0eeb..3a25f807 100644 --- a/internal/internaldns/runtime/deployment_examples_test.go +++ b/internal/internaldns/runtime/deployment_examples_test.go @@ -21,7 +21,7 @@ func TestDeploymentExampleConfigsValidate(t *testing.T) { roles []string }{ {file: "control-plane.example.yaml", namePrefix: "internal-dns-control-plane-config", roles: []string{"control-plane"}}, - {file: "fleet.example.yaml", namePrefix: "internal-dns-", roles: []string{"agent", "watchdog"}}, + {file: "fleet.example.yaml", namePrefix: "internal-dns-", roles: []string{"agent", "readiness"}}, } for _, tc := range cases { t.Run(tc.file, func(t *testing.T) { diff --git a/internal/internaldns/runtime/run.go b/internal/internaldns/runtime/run.go index fbf8f535..fd6c445a 100644 --- a/internal/internaldns/runtime/run.go +++ b/internal/internaldns/runtime/run.go @@ -21,7 +21,6 @@ import ( "go.miloapis.com/dns-operator/internal/internaldns/platform" "go.miloapis.com/dns-operator/internal/internaldns/serving" "go.miloapis.com/dns-operator/internal/internaldns/transport" - "go.miloapis.com/dns-operator/internal/internaldns/watchdog" resourcemanagerv1alpha1 "go.miloapis.com/milo/pkg/apis/resourcemanager/v1alpha1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -40,7 +39,7 @@ import ( const parentNameExtra = "iam.miloapis.com/parent-name" // Run starts one of the deliberately separate production roles. Agent and -// watchdog never load Kubernetes credentials. +// readiness never load Kubernetes credentials. func Run(ctx context.Context, role string, cfg Config, kubeconfig string) error { if err := cfg.Validate(role); err != nil { return err @@ -50,8 +49,10 @@ func Run(ctx context.Context, role string, cfg Config, kubeconfig string) error return runControlPlane(ctx, cfg, kubeconfig) case roleAgent: return runAgent(ctx, cfg) - case roleWatchdog: - return (&watchdog.Watchdog{Config: cfg.Watchdog}).Start(ctx) + case roleReadiness: + return serving.Readiness(ctx, cfg.Agent) + case roleServing: + return serving.RunDaemons(ctx, cfg.Agent) default: return fmt.Errorf("unsupported role %q", role) } From fc1f3ad6a00b5f52dc25ed7f0dee1a3367e1394a Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 14 Oct 2026 17:01:13 -0500 Subject: [PATCH 11/23] feat: package the CoreDNS serving fleet --- cmd/internal-dns-coredns/main.go | 48 ++++++++++++ config/internal-dns/Dockerfile | 9 ++- config/internal-dns/README.md | 78 +++++++++---------- .../commands/internal-dns-flush-bind | 3 - .../internal-dns/commands/internal-dns-gate | 6 +- .../commands/internal-dns-reload-bind | 11 --- .../commands/internal-dns-reload-coredns | 3 + .../commands/internal-dns-validate-bind | 4 - .../commands/internal-dns-validate-coredns | 3 + .../internal-dns/control-plane.example.yaml | 13 ++-- config/internal-dns/fleet.example.yaml | 54 ++++++------- config/internal-dns/nats-server.conf.example | 65 ++++++++-------- internal/internaldns/runtime/config_test.go | 6 +- .../runtime/deployment_examples_test.go | 2 +- .../runtime/deployment_permissions_test.go | 34 +++++--- 15 files changed, 195 insertions(+), 144 deletions(-) create mode 100644 cmd/internal-dns-coredns/main.go delete mode 100755 config/internal-dns/commands/internal-dns-flush-bind delete mode 100755 config/internal-dns/commands/internal-dns-reload-bind create mode 100755 config/internal-dns/commands/internal-dns-reload-coredns delete mode 100755 config/internal-dns/commands/internal-dns-validate-bind create mode 100755 config/internal-dns/commands/internal-dns-validate-coredns diff --git a/cmd/internal-dns-coredns/main.go b/cmd/internal-dns-coredns/main.go new file mode 100644 index 00000000..2112164a --- /dev/null +++ b/cmd/internal-dns-coredns/main.go @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package main + +import ( + "context" + "flag" + "fmt" + "os" + "os/signal" + "syscall" + + "go.miloapis.com/dns-operator/internal/internaldns/coredns" + "runtime/debug" +) + +func main() { + var path string + var validate, version bool + flag.StringVar(&path, "config", "", "Absolute path to the DNS context snapshot") + flag.BoolVar(&validate, "validate", false, "Validate the DNS context snapshot and exit") + flag.BoolVar(&version, "version", false, "Print the CoreDNS version and exit") + flag.Parse() + if version { + version := "unknown" + if info, ok := debug.ReadBuildInfo(); ok { + for _, dep := range info.Deps { + if dep.Path == "github.com/coredns/coredns" { + version = dep.Version + } + } + } + fmt.Printf("CoreDNS %s; dnscontext plugin\n", version) + return + } + var err error + if validate { + _, err = coredns.Load(path) + } else { + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + err = coredns.Run(ctx, path) + } + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "CoreDNS: %v\n", err) + os.Exit(1) + } +} diff --git a/config/internal-dns/Dockerfile b/config/internal-dns/Dockerfile index e739a004..e5194791 100644 --- a/config/internal-dns/Dockerfile +++ b/config/internal-dns/Dockerfile @@ -7,12 +7,14 @@ WORKDIR /src COPY go.mod go.sum ./ RUN --mount=type=cache,target=/go/pkg/mod go mod download COPY cmd/internal-dns ./cmd/internal-dns +COPY cmd/internal-dns-coredns ./cmd/internal-dns-coredns COPY api ./api COPY internal ./internal RUN --mount=type=cache,target=/root/.cache/go-build \ CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH \ go build -trimpath -tags netgo,osusergo -ldflags='-s -w' \ - -o /out/internal-dns ./cmd/internal-dns + -o /out/internal-dns ./cmd/internal-dns && \ + CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -trimpath -o /out/coredns ./cmd/internal-dns-coredns FROM scratch AS runtime COPY --from=build /out/internal-dns /internal-dns @@ -21,12 +23,13 @@ ENTRYPOINT ["/internal-dns"] FROM debian:trixie-slim AS serving RUN apt-get update && apt-get install --no-install-recommends -y \ - bind9 bind9-utils dnsdist iproute2 procps netcat-openbsd ca-certificates bash && \ + dnsdist pdns-recursor iproute2 procps netcat-openbsd ca-certificates bash && \ rm -rf /var/lib/apt/lists/* COPY --from=build /out/internal-dns /internal-dns +COPY --from=build /out/coredns /usr/local/bin/coredns COPY config/internal-dns/commands/ /usr/local/sbin/ USER 65532:65532 -RUN for tool in bash ip nc named named-checkconf named-checkzone dnsdist rndc; do \ +RUN for tool in bash ip nc coredns dnsdist pdns_recursor; do \ command -v "$tool" >/dev/null || exit 1; \ done && test -x /internal-dns RUN for helper in /usr/local/sbin/internal-dns-*; do \ diff --git a/config/internal-dns/README.md b/config/internal-dns/README.md index 10d4e977..ecb09899 100644 --- a/config/internal-dns/README.md +++ b/config/internal-dns/README.md @@ -5,8 +5,8 @@ Replace the example images, addresses, certificates, storage classes, admission CA injection, and site command wrappers before deployment. CI publishes `ghcr.io/datum-cloud/internal-dns` for the control plane and -`ghcr.io/datum-cloud/internal-dns-serving` for fleet agents, BIND, and dnsdist. -The serving image includes the validation, reload, cache-flush, and gating +`ghcr.io/datum-cloud/internal-dns-serving` for fleet agents, CoreDNS, and dnsdist. +The serving image includes the validation, activation, and gating commands in `config/internal-dns/commands`. The Kubernetes qualification environment uses the same commands. @@ -18,18 +18,18 @@ feature-branch tags and uses these immutable image references. The fixed fleet has three roles: -- A `node` resolver agent owns the shared node dnsdist/BIND pair. +- A `node` resolver agent owns the shared node dnsdist/CoreDNS pair. - A `regional-dnsdist` resolver agent owns the shared regional frontend and its - pool of regional BIND members. -- Two or more `regional-bind` combined agents independently render the complete - private primary-zone set for their member, validate it with - `named-checkconf -z`, activate it in isolated BIND context views, prove the + pool of regional CoreDNS members. +- Two or more `regional-coredns` combined agents independently compile the complete + DNS context snapshot for their member, validate it with + `coredns -validate`, load it atomically into the context plugin, prove the exact version over UDP and TCP, and publish a leased ACK. -Contexts, access bindings, and zones add destinations, views, and zone files to -these processes. They never create workloads per VPC or zone. The legacy -`cluster` and `both` render roles remain development fallbacks and should not be -used for the multi-member production layout. +Contexts, access bindings, and zones add destinations and record data to +these processes. They never create workloads per VPC or zone. Use `node`, +`regional-dnsdist`, and `regional-coredns` roles for the multi-member production +layout. The `cluster` and `both` roles are limited to development. Each member needs a distinct member ID, NATS identity and durable consumers, checkpoint PVC, configuration PVC, and durable activation state. Treat that @@ -69,15 +69,15 @@ Publishers write addresses and eligibility together in contribution spec using optimistic concurrency. DNS owns status and uses API-assigned generations to reject stale desired state across takeover and replay. -## Serving identity and BIND views +## Serving identity and DNS contexts Consumer and regional listener prefixes must be local routes in their dnsdist network namespaces. Galactic authorizes and translates the consumer path to a service-side destination. dnsdist preserves that destination in a trusted -PROXYv2 header. Node BIND selects a context view and forwards to its regional +PROXYv2 header. Node CoreDNS selects a context and forwards to its regional destination. Regional dnsdist selects the same context and load-balances the -healthy BIND members. Every regional BIND member directly hosts the context's -private primary zones. +healthy CoreDNS members. Every regional CoreDNS member hosts the context's +private zones. Public queries forward to a shared PowerDNS Recursor service. An owned name without eligible answers carries service-only `TYPE65280` metadata at that exact name. This private-use record preserves NODATA without @@ -88,21 +88,20 @@ retain their explicit RRset TTL, or the default of 300 seconds. Entries in one static RRset must use the same TTL. A context without private zones proves recursion with the root SOA over UDP and -TCP. `verifier.publicName` can override this probe. BIND members need a writable +TCP. `verifier.publicName` can override this probe. CoreDNS members need a writable `/tmp` runtime volume even when the image root filesystem is read-only. -Private packet caching stays disabled in dnsdist. Each BIND view has its own -positive and negative cache. `max-ncache-ttl` defaults to five seconds and is -bounded at 60 seconds. Public recursion retains DNSSEC validation; each view -adds validation exceptions only for its private apexes. Unknown or withdrawn -destinations are rejected before cache lookup. +Private packet caching stays disabled in dnsdist and CoreDNS. The context plugin +answers private queries from the accepted in-memory zone data. Endpoint changes +replace that data without retaining cached private answers. Each context has a +separate public cache. PowerDNS Recursor validates DNSSEC for public queries. +Unknown destinations are rejected before cache lookup. -Regional publication proof connects directly to that member's BIND listener, +Regional publication proof connects directly to that member's CoreDNS listener, supplies the exact context destination in a protected PROXYv2 header over both UDP and TCP, and verifies the installed SOA serial and fingerprint. Restrict `proxyPeers` to regional dnsdist and the member's local publication agent -network. There are no PowerDNS views, network variants, or per-context source -markers. Local health probes also require the BIND listener address in +network. Local health probes also require the CoreDNS listener address in `proxyPeers` and loopback in dnsdist ACLs. These peers must remain inside the protected serving network. @@ -116,12 +115,12 @@ restores service with the same access mapping. Resolver access readiness reports the applied path independently of each zone's publication status. Explicit access withdrawal requires a newer access version to reactivate. -The agent writes a complete candidate set into its stage directory. Regional -zone files are siblings of `cluster-bind.conf`; validation rewrites file -references to the staged paths and runs `named-checkconf -z`. Installation is -atomic only after every candidate validates. Reload wrappers must wait for the -listener and backend pool before returning so the immediate DNS proof observes -the activated state. +The agent stages a complete JSON snapshot containing context destinations, +private zone data, and upstream destinations. The CoreDNS validator parses every +zone before installation. Atomic rename exposes one complete snapshot; the +plugin builds its replacement state before swapping the lookup table. Invalid +updates retain the accepted state. A record update does not restart CoreDNS. +The agent proves the installed publication over UDP and TCP before acknowledging it. Publication acknowledgements report verified configuration versions. The fleet retains accepted proofs during control plane or broker outages and continues @@ -146,22 +145,21 @@ transaction lock until the approved checkpoint is committed. Orphaned activation remains gated until recovery verifies the saved configuration. Readiness and activation gates use local state without Kubernetes or NATS. -Mount the transaction stage, active BIND configuration, zone files, checkpoint, +Mount the transaction stage, active CoreDNS configuration, checkpoint, and activation state on storage local to the serving host. Replication uses the durable publication stream and independent regional members; each member has its own crash-safe checkpoint volume. -The regional BIND reload helper must reload existing primary zones, for -example with `rndc reload`. `rndc reconfig` only guarantees configuration and -new-zone loading, so it is insufficient when an existing private apex moves to -a newly rendered immutable zone file. The node-tier helper may use the -deployment's normal configuration reload because that BIND tier is recursive. +The transaction stage and active configuration must share one filesystem: +installation uses atomic rename. The examples put the stage at +`/etc/internal-dns/stage` on the configuration PVC and keep checkpoints on the +separate state PVC. ## Capacity and transport -Set `maxBindings` from measured BIND view memory, validation/reload time, +Set `maxBindings` from measured CoreDNS context memory, snapshot activation time, dnsdist rule cost, NATS payload size, and activation recovery time. The example caps a -shard at 200 contexts and budgets 2 MiB per view. Add another fixed shard before +shard at 200 contexts. Add another fixed shard before raising those bounds. NATS uses TLS with a verified server name, client certificate, and one scoped @@ -170,9 +168,9 @@ subjects and consume ACKs. Each serving member can consume its assigned durable subjects and publish only its own hashed ACK subject. Broker partitions delay updates while the fleet continues serving its applied configuration. -`fleet.example.yaml` provides node, regional frontend, and regional BIND JSON +`fleet.example.yaml` provides node, regional frontend, and regional CoreDNS JSON configurations plus one Deployment template. Instantiate the template for each -member and create a second regional BIND member with distinct addresses and +member and create a second regional CoreDNS member with distinct addresses and storage. `control-plane.example.yaml` lists both regional members in `clusterBackends` and `members`. diff --git a/config/internal-dns/commands/internal-dns-flush-bind b/config/internal-dns/commands/internal-dns-flush-bind deleted file mode 100755 index 21bbe10c..00000000 --- a/config/internal-dns/commands/internal-dns-flush-bind +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -set -eu -exec rndc -s 127.0.0.1 -p 9953 -k /etc/bind/rndc.key flushtree "$2" diff --git a/config/internal-dns/commands/internal-dns-gate b/config/internal-dns/commands/internal-dns-gate index bf46ac9a..3a610076 100755 --- a/config/internal-dns/commands/internal-dns-gate +++ b/config/internal-dns/commands/internal-dns-gate @@ -1,10 +1,10 @@ #!/bin/sh # Clearing enable files before stopping processes prevents automatic restart. set -eu -rm -f /run/internal-dns/allow-named /run/internal-dns/allow-dnsdist -pkill -TERM -x named || true +rm -f /run/internal-dns/allow-coredns /run/internal-dns/allow-dnsdist +pkill -TERM -x coredns || true pkill -TERM -x dnsdist || true -for daemon in named dnsdist; do +for daemon in coredns dnsdist; do for attempt in $(seq 1 40); do if ! pgrep -x "$daemon" >/dev/null; then break; fi sleep 0.05 diff --git a/config/internal-dns/commands/internal-dns-reload-bind b/config/internal-dns/commands/internal-dns-reload-bind deleted file mode 100755 index d05937de..00000000 --- a/config/internal-dns/commands/internal-dns-reload-bind +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -set -eu -touch /run/internal-dns/allow-named -for attempt in $(seq 1 35); do - if rndc -s 127.0.0.1 -p 9953 -k /etc/bind/rndc.key status >/dev/null 2>&1; then - exec rndc -s 127.0.0.1 -p 9953 -k /etc/bind/rndc.key reload - fi - sleep 0.05 -done -echo 'BIND control listener did not become ready' >&2 -exit 1 diff --git a/config/internal-dns/commands/internal-dns-reload-coredns b/config/internal-dns/commands/internal-dns-reload-coredns new file mode 100755 index 00000000..a00050af --- /dev/null +++ b/config/internal-dns/commands/internal-dns-reload-coredns @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +touch /run/internal-dns/allow-coredns diff --git a/config/internal-dns/commands/internal-dns-validate-bind b/config/internal-dns/commands/internal-dns-validate-bind deleted file mode 100755 index 989cd2d5..00000000 --- a/config/internal-dns/commands/internal-dns-validate-bind +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -set -eu -shift -exec named-checkconf "$@" diff --git a/config/internal-dns/commands/internal-dns-validate-coredns b/config/internal-dns/commands/internal-dns-validate-coredns new file mode 100755 index 00000000..3a47c06d --- /dev/null +++ b/config/internal-dns/commands/internal-dns-validate-coredns @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +exec coredns -validate -config "$2" diff --git a/config/internal-dns/control-plane.example.yaml b/config/internal-dns/control-plane.example.yaml index 171d9d3c..8b963394 100644 --- a/config/internal-dns/control-plane.example.yaml +++ b/config/internal-dns/control-plane.example.yaml @@ -16,21 +16,20 @@ data: "kubeAPI": {"qps":20,"burst":40,"requestTimeoutSeconds":10} }, "nodeBackends": [ - {"memberID":"node-bind-0","address":"fd70:10::10","port":5300} + {"memberID":"node-coredns-0","address":"fd70:10::10","port":5300} ], "clusterBackends": [ - {"memberID":"regional-bind-us-central1-0","address":"fd70:20::10","port":5300}, - {"memberID":"regional-bind-us-central1-1","address":"fd70:20::11","port":5300} + {"memberID":"regional-coredns-us-central1-0","address":"fd70:20::10","port":5300}, + {"memberID":"regional-coredns-us-central1-1","address":"fd70:20::11","port":5300} ], "members": [ {"memberID":"node-us-central1-0","replicaID":"node-us-central1-0","role":"node"}, {"memberID":"regional-front-us-central1-0","replicaID":"regional-front-us-central1-0","role":"resolver"}, - {"memberID":"regional-bind-us-central1-0","replicaID":"regional-bind-us-central1-0","role":"regional"}, - {"memberID":"regional-bind-us-central1-1","replicaID":"regional-bind-us-central1-1","role":"regional"} + {"memberID":"regional-coredns-us-central1-0","replicaID":"regional-coredns-us-central1-0","role":"regional"}, + {"memberID":"regional-coredns-us-central1-1","replicaID":"regional-coredns-us-central1-1","role":"regional"} ], "clusterPrefix": "fd70:200::/48", - "managedDomainSuffix": "internal.example.net", - "privateZoneClassName": "private-bind", + "privateZoneClassName": "private-coredns", "ownershipLeaseSeconds": 30, "shards": [ {"region":"us-central1","shard":"shared-0","maxBindings":200,"maxSnapshotBytes":524288} diff --git a/config/internal-dns/fleet.example.yaml b/config/internal-dns/fleet.example.yaml index d53b3e8c..75d636d8 100644 --- a/config/internal-dns/fleet.example.yaml +++ b/config/internal-dns/fleet.example.yaml @@ -1,5 +1,5 @@ # Fixed shared fleet members for one regional shard. Deploy at least two copies -# of the regional-bind member with distinct IDs, addresses, state, config, NATS +# of the regional-coredns member with distinct IDs, addresses, state, config, NATS # credentials, and activation state. Contexts and zones never create workloads. apiVersion: v1 kind: ConfigMap @@ -11,9 +11,9 @@ data: "stream":{"name":"DNS_PRIVATE","snapshotsBucket":"DNS_PRIVATE_SNAPSHOTS"}, "agent":{ "region":"us-central1","shard":"shared-0","memberID":"node-us-central1-0","replicaID":"node-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"resolver", - "render":{"role":"node","nodeDNSDist":{"path":"/etc/internal-dns/node-dnsdist.conf","listenAddress":"[::]:53","acls":["fd71:100::/64","::1/128"],"poolName":"node_shared","backends":[{"memberID":"node-bind-0","address":"fd70:10::10","port":5300}]},"nodeBIND":{"path":"/etc/internal-dns/node-bind.conf","listenAddress":"fd70:10::10","port":5300,"proxyPeers":["fd70:10::20","fd70:10::10"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, - "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-dnsdist","args":["node","{stageDir}/node-dnsdist.conf"],"whenChanged":["/etc/internal-dns/node-dnsdist.conf"]},{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["node","{stageDir}/node-bind.conf"],"whenChanged":["/etc/internal-dns/node-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["node"],"whenChanged":["/etc/internal-dns/node-bind.conf"]},{"path":"/usr/local/sbin/internal-dns-reload-dnsdist","args":["node"],"whenChanged":["/etc/internal-dns/node-dnsdist.conf"]}],"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["node"]}]}, - "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["node","{name}"]}],"reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"resolverDNSProbe":{"timeout":3000000000},"resolverVerifyConcurrency":16 + "render":{"role":"node","nodeDNSDist":{"path":"/etc/internal-dns/node-dnsdist.conf","listenAddress":"[::]:53","acls":["fd71:100::/64","::1/128"],"poolName":"node_shared","backends":[{"memberID":"node-coredns-0","address":"fd70:10::10","port":5300}]},"nodeCoreDNS":{"path":"/etc/internal-dns/node-coredns.json","listenAddress":"fd70:10::10","port":5300,"proxyPeers":["fd70:10::20","fd70:10::10"],"publicUpstreams":["[fd70:30::10]:5353"]}}, + "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-dnsdist","args":["node","{stageDir}/node-dnsdist.conf"],"whenChanged":["/etc/internal-dns/node-dnsdist.conf"]},{"path":"/usr/local/sbin/internal-dns-validate-coredns","args":["node","{stageDir}/node-coredns.json"],"whenChanged":["/etc/internal-dns/node-coredns.json"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-coredns","args":["node"],"whenChanged":["/etc/internal-dns/node-coredns.json"]},{"path":"/usr/local/sbin/internal-dns-reload-dnsdist","args":["node"],"whenChanged":["/etc/internal-dns/node-dnsdist.conf"]}],"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["node"]}]}, + "reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"resolverDNSProbe":{"timeout":3000000000},"resolverVerifyConcurrency":16 } } --- @@ -27,7 +27,7 @@ data: "stream":{"name":"DNS_PRIVATE","snapshotsBucket":"DNS_PRIVATE_SNAPSHOTS"}, "agent":{ "region":"us-central1","shard":"shared-0","memberID":"regional-front-us-central1-0","replicaID":"regional-front-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"resolver", - "render":{"role":"regional-dnsdist","clusterDNSDist":{"path":"/etc/internal-dns/cluster-dnsdist.conf","listenAddress":"[::]:53","acls":["fd71:200::/64","::1/128"],"poolName":"regional_shared","backends":[{"memberID":"regional-bind-us-central1-0","address":"fd70:20::10","port":5300},{"memberID":"regional-bind-us-central1-1","address":"fd70:20::11","port":5300}]}}, + "render":{"role":"regional-dnsdist","clusterDNSDist":{"path":"/etc/internal-dns/cluster-dnsdist.conf","listenAddress":"[::]:53","acls":["fd71:200::/64","::1/128"],"poolName":"regional_shared","backends":[{"memberID":"regional-coredns-us-central1-0","address":"fd70:20::10","port":5300},{"memberID":"regional-coredns-us-central1-1","address":"fd70:20::11","port":5300}]}}, "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-dnsdist","args":["regional","{stageDir}/cluster-dnsdist.conf"],"whenChanged":["/etc/internal-dns/cluster-dnsdist.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-dnsdist","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-dnsdist.conf"]}],"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["regional-front"]}]}, "reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"resolverDNSProbe":{"timeout":3000000000},"resolverVerifyConcurrency":16 } @@ -35,17 +35,17 @@ data: --- apiVersion: v1 kind: ConfigMap -metadata: {name: internal-dns-regional-bind-0-config, namespace: internal-dns-system} +metadata: {name: internal-dns-regional-coredns-0-config, namespace: internal-dns-system} data: config.json: | { - "nats":{"url":"tls://nats.internal-dns-system.svc:4222","name":"regional-bind-us-central1-0","username":"regional-bind-us-central1-0","passwordFile":"/var/run/internal-dns/nats/password","tls":{"caFile":"/var/run/internal-dns/nats/ca.crt","certificateFile":"/var/run/internal-dns/nats/tls.crt","keyFile":"/var/run/internal-dns/nats/tls.key","serverName":"nats.internal-dns-system.svc"}}, + "nats":{"url":"tls://nats.internal-dns-system.svc:4222","name":"regional-coredns-us-central1-0","username":"regional-coredns-us-central1-0","passwordFile":"/var/run/internal-dns/nats/password","tls":{"caFile":"/var/run/internal-dns/nats/ca.crt","certificateFile":"/var/run/internal-dns/nats/tls.crt","keyFile":"/var/run/internal-dns/nats/tls.key","serverName":"nats.internal-dns-system.svc"}}, "stream":{"name":"DNS_PRIVATE","snapshotsBucket":"DNS_PRIVATE_SNAPSHOTS"}, "agent":{ - "region":"us-central1","shard":"shared-0","memberID":"regional-bind-us-central1-0","replicaID":"regional-bind-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"combined", - "render":{"role":"regional-bind","clusterBIND":{"path":"/etc/internal-dns/cluster-bind.conf","listenAddress":"fd70:20::10","port":5300,"proxyPeers":["fd70:20::20","fd70:20::30","fd70:20::10"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, - "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["regional","-z","{stageDir}/cluster-bind.conf"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}],"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["regional-bind"]}]}, - "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["regional","{name}"]}],"reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"publicationDNSProbe":{"server":"[fd70:20::10]:5300","timeout":3000000000} + "region":"us-central1","shard":"shared-0","memberID":"regional-coredns-us-central1-0","replicaID":"regional-coredns-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"combined", + "render":{"role":"regional-coredns","clusterCoreDNS":{"path":"/etc/internal-dns/cluster-coredns.json","listenAddress":"fd70:20::10","port":5300,"proxyPeers":["fd70:20::20","fd70:20::30","fd70:20::10"],"publicUpstreams":["[fd70:30::10]:5353"]}}, + "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-coredns","args":["regional","{stageDir}/cluster-coredns.json"],"whenChanged":["/etc/internal-dns/cluster-coredns.json"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-coredns","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-coredns.json"]}],"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["regional-coredns"]}]}, + "reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"publicationDNSProbe":{"server":"[fd70:20::10]:5300","timeout":3000000000} } } --- @@ -54,33 +54,33 @@ data: # second clusterBackends entry in the control-plane configuration. apiVersion: v1 kind: ConfigMap -metadata: {name: internal-dns-regional-bind-1-config, namespace: internal-dns-system} +metadata: {name: internal-dns-regional-coredns-1-config, namespace: internal-dns-system} data: config.json: | { - "nats":{"url":"tls://nats.internal-dns-system.svc:4222","name":"regional-bind-us-central1-1","username":"regional-bind-us-central1-1","passwordFile":"/var/run/internal-dns/nats/password","tls":{"caFile":"/var/run/internal-dns/nats/ca.crt","certificateFile":"/var/run/internal-dns/nats/tls.crt","keyFile":"/var/run/internal-dns/nats/tls.key","serverName":"nats.internal-dns-system.svc"}}, + "nats":{"url":"tls://nats.internal-dns-system.svc:4222","name":"regional-coredns-us-central1-1","username":"regional-coredns-us-central1-1","passwordFile":"/var/run/internal-dns/nats/password","tls":{"caFile":"/var/run/internal-dns/nats/ca.crt","certificateFile":"/var/run/internal-dns/nats/tls.crt","keyFile":"/var/run/internal-dns/nats/tls.key","serverName":"nats.internal-dns-system.svc"}}, "stream":{"name":"DNS_PRIVATE","snapshotsBucket":"DNS_PRIVATE_SNAPSHOTS"}, "agent":{ - "region":"us-central1","shard":"shared-0","memberID":"regional-bind-us-central1-1","replicaID":"regional-bind-us-central1-1","stateDir":"/var/lib/internal-dns","mode":"combined", - "render":{"role":"regional-bind","clusterBIND":{"path":"/etc/internal-dns/cluster-bind.conf","listenAddress":"fd70:20::11","port":5300,"proxyPeers":["fd70:20::20","fd70:20::31","fd70:20::11"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, - "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["regional","-z","{stageDir}/cluster-bind.conf"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}],"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["regional-bind"]}]}, - "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["regional","{name}"]}],"reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"publicationDNSProbe":{"server":"[fd70:20::11]:5300","timeout":3000000000} + "region":"us-central1","shard":"shared-0","memberID":"regional-coredns-us-central1-1","replicaID":"regional-coredns-us-central1-1","stateDir":"/var/lib/internal-dns","mode":"combined", + "render":{"role":"regional-coredns","clusterCoreDNS":{"path":"/etc/internal-dns/cluster-coredns.json","listenAddress":"fd70:20::11","port":5300,"proxyPeers":["fd70:20::20","fd70:20::31","fd70:20::11"],"publicUpstreams":["[fd70:30::10]:5353"]}}, + "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-coredns","args":["regional","{stageDir}/cluster-coredns.json"],"whenChanged":["/etc/internal-dns/cluster-coredns.json"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-coredns","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-coredns.json"]}],"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["regional-coredns"]}]}, + "reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"publicationDNSProbe":{"server":"[fd70:20::11]:5300","timeout":3000000000} } } --- # This Deployment template is instantiated once for each ConfigMap above and a -# second time for regional-bind-1. The serving image contains the local dnsdist -# and/or BIND process selected by MEMBER_ROLE plus the validation/reload/gate +# second time for regional-coredns-1. The serving image contains the local dnsdist +# and/or CoreDNS process selected by MEMBER_ROLE plus the validation/reload/gate # wrappers referenced by that member's config. apiVersion: apps/v1 kind: Deployment -metadata: {name: internal-dns-regional-bind-0, namespace: internal-dns-system} +metadata: {name: internal-dns-regional-coredns-0, namespace: internal-dns-system} spec: replicas: 1 strategy: {type: Recreate} - selector: {matchLabels: {app.kubernetes.io/name: internal-dns-regional-bind-0}} + selector: {matchLabels: {app.kubernetes.io/name: internal-dns-regional-coredns-0}} template: - metadata: {labels: {app.kubernetes.io/name: internal-dns-regional-bind-0, internal-dns.datum.net/tier: regional}} + metadata: {labels: {app.kubernetes.io/name: internal-dns-regional-coredns-0, internal-dns.datum.net/tier: regional}} spec: serviceAccountName: internal-dns-fleet automountServiceAccountToken: false @@ -111,12 +111,12 @@ spec: - {name: state, mountPath: /var/lib/internal-dns} - {name: activation, mountPath: /run/internal-dns} - {name: daemon-config, mountPath: /etc/internal-dns} - # BIND requires a writable working directory with a read-only root filesystem. + # CoreDNS requires a writable working directory with a read-only root filesystem. - {name: runtime, mountPath: /tmp} volumes: - - {name: config, configMap: {name: internal-dns-regional-bind-0-config}} - - {name: nats, secret: {secretName: internal-dns-regional-bind-0-nats}} + - {name: config, configMap: {name: internal-dns-regional-coredns-0-config}} + - {name: nats, secret: {secretName: internal-dns-regional-coredns-0-nats}} - {name: activation, emptyDir: {medium: Memory}} - {name: runtime, emptyDir: {}} - - {name: daemon-config, persistentVolumeClaim: {claimName: internal-dns-regional-bind-0-config}} - - {name: state, persistentVolumeClaim: {claimName: internal-dns-regional-bind-0-state}} + - {name: daemon-config, persistentVolumeClaim: {claimName: internal-dns-regional-coredns-0-config}} + - {name: state, persistentVolumeClaim: {claimName: internal-dns-regional-coredns-0-state}} diff --git a/config/internal-dns/nats-server.conf.example b/config/internal-dns/nats-server.conf.example index c75850a0..573d8eac 100644 --- a/config/internal-dns/nats-server.conf.example +++ b/config/internal-dns/nats-server.conf.example @@ -20,7 +20,10 @@ authorization { permissions: { publish: { allow: [ "dns.private.serving.>", "dns.private.records.>", - "$KV.DNS_PRIVATE_SNAPSHOTS.>", "$JS.API.>", "_INBOX.>" + "$KV.DNS_PRIVATE_SNAPSHOTS.>", "$JS.API.>", + # Match the control-plane ACK durable for this region and shard. + "$JS.ACK.DNS_PRIVATE.control-plane-us-central1-shared-0-acks.>", + "_INBOX.>" ] } subscribe: { allow: ["dns.private.acks.>", "_INBOX.>"] } } @@ -86,24 +89,24 @@ authorization { } }, { - user: "regional-bind-us-central1-0" - password: $REGIONAL_BIND_0_PASSWORD + user: "regional-coredns-us-central1-0" + password: $REGIONAL_COREDNS_0_PASSWORD permissions: { publish: { allow: [ - "dns.private.acks.us-central1.shared-0.b75afe155467f45fdab6d0a1", + "dns.private.acks.us-central1.shared-0.7c90d3973cc5ead390c680d8", "$JS.API.STREAM.INFO.DNS_PRIVATE", - "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-bind-us-central1-0-serving", - "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-bind-us-central1-0-records", - "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-bind-us-central1-0-acks", - "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-bind-us-central1-0-serving", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-bind-us-central1-0-serving.>", - "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-bind-us-central1-0-records", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-bind-us-central1-0-records.>", - "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-bind-us-central1-0-acks", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-bind-us-central1-0-acks.>", - "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-bind-us-central1-0-serving", - "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-bind-us-central1-0-records", - "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-bind-us-central1-0-acks", - "$JS.ACK.DNS_PRIVATE.regional-bind-us-central1-0-serving.>", - "$JS.ACK.DNS_PRIVATE.regional-bind-us-central1-0-records.>", - "$JS.ACK.DNS_PRIVATE.regional-bind-us-central1-0-acks.>", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-coredns-us-central1-0-serving", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-coredns-us-central1-0-records", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-coredns-us-central1-0-acks", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-coredns-us-central1-0-serving", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-coredns-us-central1-0-serving.>", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-coredns-us-central1-0-records", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-coredns-us-central1-0-records.>", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-coredns-us-central1-0-acks", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-coredns-us-central1-0-acks.>", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-coredns-us-central1-0-serving", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-coredns-us-central1-0-records", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-coredns-us-central1-0-acks", + "$JS.ACK.DNS_PRIVATE.regional-coredns-us-central1-0-serving.>", + "$JS.ACK.DNS_PRIVATE.regional-coredns-us-central1-0-records.>", + "$JS.ACK.DNS_PRIVATE.regional-coredns-us-central1-0-acks.>", "$JS.API.DIRECT.GET.KV_DNS_PRIVATE_SNAPSHOTS.>", "$JS.API.STREAM.INFO.KV_DNS_PRIVATE_SNAPSHOTS", "$JS.API.CONSUMER.CREATE.KV_DNS_PRIVATE_SNAPSHOTS.>", @@ -116,24 +119,24 @@ authorization { } }, { - user: "regional-bind-us-central1-1" - password: $REGIONAL_BIND_1_PASSWORD + user: "regional-coredns-us-central1-1" + password: $REGIONAL_COREDNS_1_PASSWORD permissions: { publish: { allow: [ - "dns.private.acks.us-central1.shared-0.2600654ae9645c6e4439e8ac", + "dns.private.acks.us-central1.shared-0.f7f1afc061d6adfe80acb36b", "$JS.API.STREAM.INFO.DNS_PRIVATE", - "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-bind-us-central1-1-serving", - "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-bind-us-central1-1-records", - "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-bind-us-central1-1-acks", - "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-bind-us-central1-1-serving", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-bind-us-central1-1-serving.>", - "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-bind-us-central1-1-records", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-bind-us-central1-1-records.>", - "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-bind-us-central1-1-acks", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-bind-us-central1-1-acks.>", - "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-bind-us-central1-1-serving", - "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-bind-us-central1-1-records", - "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-bind-us-central1-1-acks", - "$JS.ACK.DNS_PRIVATE.regional-bind-us-central1-1-serving.>", - "$JS.ACK.DNS_PRIVATE.regional-bind-us-central1-1-records.>", - "$JS.ACK.DNS_PRIVATE.regional-bind-us-central1-1-acks.>", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-coredns-us-central1-1-serving", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-coredns-us-central1-1-records", + "$JS.API.CONSUMER.INFO.DNS_PRIVATE.regional-coredns-us-central1-1-acks", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-coredns-us-central1-1-serving", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-coredns-us-central1-1-serving.>", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-coredns-us-central1-1-records", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-coredns-us-central1-1-records.>", + "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-coredns-us-central1-1-acks", "$JS.API.CONSUMER.CREATE.DNS_PRIVATE.regional-coredns-us-central1-1-acks.>", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-coredns-us-central1-1-serving", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-coredns-us-central1-1-records", + "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE.regional-coredns-us-central1-1-acks", + "$JS.ACK.DNS_PRIVATE.regional-coredns-us-central1-1-serving.>", + "$JS.ACK.DNS_PRIVATE.regional-coredns-us-central1-1-records.>", + "$JS.ACK.DNS_PRIVATE.regional-coredns-us-central1-1-acks.>", "$JS.API.DIRECT.GET.KV_DNS_PRIVATE_SNAPSHOTS.>", "$JS.API.STREAM.INFO.KV_DNS_PRIVATE_SNAPSHOTS", "$JS.API.CONSUMER.CREATE.KV_DNS_PRIVATE_SNAPSHOTS.>", diff --git a/internal/internaldns/runtime/config_test.go b/internal/internaldns/runtime/config_test.go index c271e30e..0457c89e 100644 --- a/internal/internaldns/runtime/config_test.go +++ b/internal/internaldns/runtime/config_test.go @@ -25,7 +25,7 @@ func TestAgentUsesRootRegionAndDoesNotNeedKubernetes(t *testing.T) { Agent: serving.Config{MemberID: "resolver-1", ReplicaID: "resolver-1", StateDir: t.TempDir(), Mode: serving.ModeResolver, Render: serving.RenderConfig{Role: serving.RenderNode, NodeDNSDist: serving.DNSDistRenderConfig{Path: t.TempDir() + "/dnsdist.conf", ListenAddress: "[::1]:53", PoolName: "node"}, - NodeBIND: serving.BINDRenderConfig{Path: t.TempDir() + "/named.conf", ListenAddress: "::1", Port: 5353}}, + NodeCoreDNS: serving.CoreDNSRenderConfig{Path: t.TempDir() + "/named.conf", ListenAddress: "::1", Port: 5353, ProxyPeers: []string{"::1"}}}, Transaction: serving.TransactionConfig{StageDir: t.TempDir()}, UnsafeAllowNoFailClosed: true, UnsafeAllowNoDNSVerify: true}, } if err := c.Validate("agent"); err != nil { @@ -155,7 +155,7 @@ func TestConfigLoaderRejectsRemovedPrivateServingFields(t *testing.T) { func TestAgentRestoresCheckpointWhileBrokerUnavailable(t *testing.T) { dir := t.TempDir() - cfg := Config{Region: "central", Shard: "one", NATS: transport.Config{URL: "nats://127.0.0.1:1", AllowInsecure: true}, Stream: transport.StreamConfig{Name: "DNS", SnapshotsBucket: "DNS_BOOTSTRAP"}, Agent: serving.Config{MemberID: "node-0", StateDir: filepath.Join(dir, "state"), Mode: serving.ModeResolver, Render: serving.RenderConfig{Role: serving.RenderNode, NodeDNSDist: serving.DNSDistRenderConfig{Path: filepath.Join(dir, "dnsdist.conf"), ListenAddress: "[::1]:53", PoolName: "node"}, NodeBIND: serving.BINDRenderConfig{Path: filepath.Join(dir, "named.conf"), ListenAddress: "::1", Port: 5353}}, Transaction: serving.TransactionConfig{StageDir: filepath.Join(dir, "stage")}, UnsafeAllowNoFailClosed: true, UnsafeAllowNoDNSVerify: true, ReconcileInterval: 10 * time.Millisecond}} + cfg := Config{Region: "central", Shard: "one", NATS: transport.Config{URL: "nats://127.0.0.1:1", AllowInsecure: true}, Stream: transport.StreamConfig{Name: "DNS", SnapshotsBucket: "DNS_BOOTSTRAP"}, Agent: serving.Config{MemberID: "node-0", StateDir: filepath.Join(dir, "state"), Mode: serving.ModeResolver, Render: serving.RenderConfig{Role: serving.RenderNode, NodeDNSDist: serving.DNSDistRenderConfig{Path: filepath.Join(dir, "dnsdist.conf"), ListenAddress: "[::1]:53", PoolName: "node"}, NodeCoreDNS: serving.CoreDNSRenderConfig{Path: filepath.Join(dir, "named.conf"), ListenAddress: "::1", Port: 5353, ProxyPeers: []string{"::1"}}}, Transaction: serving.TransactionConfig{StageDir: filepath.Join(dir, "stage")}, UnsafeAllowNoFailClosed: true, UnsafeAllowNoDNSVerify: true, ReconcileInterval: 10 * time.Millisecond}} if err := cfg.Validate(roleAgent); err != nil { t.Fatal(err) } @@ -230,7 +230,7 @@ func TestLocalServingRolesDoNotRequireDeliveryConfiguration(t *testing.T) { for _, role := range []string{"serving", "readiness"} { t.Run(role, func(t *testing.T) { dir := t.TempDir() - cfg := Config{Region: "central", Shard: "one", Agent: serving.Config{MemberID: "node", StateDir: filepath.Join(dir, "state"), Mode: serving.ModeResolver, CacheFlush: []serving.Command{{Path: "/bin/true"}}, Render: serving.RenderConfig{Role: serving.RenderNode, NodeDNSDist: serving.DNSDistRenderConfig{Path: filepath.Join(dir, "dnsdist.conf"), ListenAddress: "[::1]:53", PoolName: "node"}, NodeBIND: serving.BINDRenderConfig{Path: filepath.Join(dir, "named.conf"), ListenAddress: "::1", Port: 5353}}, Transaction: serving.TransactionConfig{StageDir: filepath.Join(dir, "stage"), FailClosed: []serving.Command{{Path: "/bin/true"}}}}} + cfg := Config{Region: "central", Shard: "one", Agent: serving.Config{MemberID: "node", StateDir: filepath.Join(dir, "state"), Mode: serving.ModeResolver, Render: serving.RenderConfig{Role: serving.RenderNode, NodeDNSDist: serving.DNSDistRenderConfig{Path: filepath.Join(dir, "dnsdist.conf"), ListenAddress: "[::1]:53", PoolName: "node"}, NodeCoreDNS: serving.CoreDNSRenderConfig{Path: filepath.Join(dir, "named.conf"), ListenAddress: "::1", Port: 5353, ProxyPeers: []string{"::1"}}}, Transaction: serving.TransactionConfig{StageDir: filepath.Join(dir, "stage"), FailClosed: []serving.Command{{Path: "/bin/true"}}}}} if err := cfg.Validate(role); err != nil { t.Fatalf("local role required messaging or control-plane configuration: %v", err) } diff --git a/internal/internaldns/runtime/deployment_examples_test.go b/internal/internaldns/runtime/deployment_examples_test.go index 3a25f807..ba85464a 100644 --- a/internal/internaldns/runtime/deployment_examples_test.go +++ b/internal/internaldns/runtime/deployment_examples_test.go @@ -80,7 +80,7 @@ func TestNATSExampleUsesExactMemberAckSubjects(t *testing.T) { if err != nil { t.Fatal(err) } - for _, memberID := range []string{"node-us-central1-0", "regional-front-us-central1-0", "regional-bind-us-central1-0", "regional-bind-us-central1-1"} { + for _, memberID := range []string{"node-us-central1-0", "regional-front-us-central1-0", "regional-coredns-us-central1-0", "regional-coredns-us-central1-1"} { subject := model.AckSubject("us-central1", "shared-0", memberID) if !bytes.Contains(contents, []byte(`"`+subject+`"`)) { t.Errorf("NATS policy is missing exact ACK subject %q", subject) diff --git a/internal/internaldns/runtime/deployment_permissions_test.go b/internal/internaldns/runtime/deployment_permissions_test.go index 84fb8270..ed105150 100644 --- a/internal/internaldns/runtime/deployment_permissions_test.go +++ b/internal/internaldns/runtime/deployment_permissions_test.go @@ -161,7 +161,19 @@ func TestEveryMemberCanCreateAllRuntimeConsumers(t *testing.T) { t.Fatal(err) } contents := string(data) - for _, member := range []string{"node-us-central1-0", "regional-front-us-central1-0", "regional-bind-us-central1-0", "regional-bind-us-central1-1"} { + start := strings.Index(contents, `user: "internal-dns-control-plane"`) + if start < 0 { + t.Fatal("missing control-plane broker identity") + } + controlPolicy := strings.SplitN(contents[start:], "\n },", 2)[0] + controlACK := "$JS.ACK.DNS_PRIVATE." + durable(roleControlPlane, "us-central1", "shared-0", "acks") + ".>" + if !strings.Contains(controlPolicy, `"`+controlACK+`"`) { + t.Errorf("control plane cannot acknowledge its member-ACK consumer: %s", controlACK) + } + if strings.Contains(controlPolicy, `"$JS.ACK.DNS_PRIVATE.>"`) { + t.Error("control plane can acknowledge another serving member's consumers") + } + for _, member := range []string{"node-us-central1-0", "regional-front-us-central1-0", "regional-coredns-us-central1-0", "regional-coredns-us-central1-1"} { start := strings.Index(contents, `user: "`+member+`"`) if start < 0 { t.Fatalf("missing member %s", member) @@ -189,7 +201,7 @@ func TestEveryMemberCanCreateAllRuntimeConsumers(t *testing.T) { } } -func TestReadOnlyServingImageHasWritableBINDWorkingDirectory(t *testing.T) { +func TestReadOnlyServingImageHasWritableCoreDNSWorkingDirectory(t *testing.T) { found := false for _, object := range deploymentObjects(t, "fleet.example.yaml") { if object.GetKind() != "Deployment" { @@ -217,7 +229,7 @@ func TestReadOnlyServingImageHasWritableBINDWorkingDirectory(t *testing.T) { } } if runtimeVolume == "" { - t.Fatal("read-only serving image lacks writable BIND /tmp mount") + t.Fatal("read-only serving image lacks writable CoreDNS /tmp mount") } volumes, _, err := unstructured.NestedSlice(object.Object, "spec", "template", "spec", "volumes") if err != nil { @@ -231,7 +243,7 @@ func TestReadOnlyServingImageHasWritableBINDWorkingDirectory(t *testing.T) { } } if !writable { - t.Fatal("BIND working-directory mount is not a per-member writable runtime volume") + t.Fatal("CoreDNS working-directory mount is not a per-member writable runtime volume") } } } @@ -253,13 +265,13 @@ func verifyProjectReferenceSchemas(t *testing.T, ctx context.Context, cl client. object client.Object valid bool }{ - {"registration-valid", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: ref, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 30}}, true}, - {"registration-unpinned", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name}, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 30}}, false}, - {"registration-ttl31", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: ref, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 31}}, false}, - {"association-valid", &dnsv1alpha1.DNSZoneAssociation{Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: ref, ResolverContextRef: contextRef}}, true}, - {"association-unpinned", &dnsv1alpha1.DNSZoneAssociation{Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name}, ResolverContextRef: contextRef}}, false}, - {"naming-valid", &dnsv1alpha1.DNSNamingPolicy{Spec: dnsv1alpha1.DNSNamingPolicySpec{ResolverContextRef: contextRef, AdditionalNames: []dnsv1alpha1.DNSAdditionalNameRule{{Selector: nil, DNSZoneRef: ref, NamePrefix: "services"}}}}, true}, - {"naming-unpinned", &dnsv1alpha1.DNSNamingPolicy{Spec: dnsv1alpha1.DNSNamingPolicySpec{ResolverContextRef: contextRef, AdditionalNames: []dnsv1alpha1.DNSAdditionalNameRule{{Selector: nil, DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name}, NamePrefix: "services"}}}}, false}, + {"registration-name-only", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 30}}, true}, + {"registration-empty-zone", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{}, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 30}}, false}, + {"registration-ttl31", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: ref.Name}, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 31}}, false}, + {"association-valid", &dnsv1alpha1.DNSZoneAssociation{Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: ref.Name}, ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: contextRef.Name}}}, true}, + {"association-empty-zone", &dnsv1alpha1.DNSZoneAssociation{Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{}, ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: contextRef.Name}}}, false}, + {"naming-valid", &dnsv1alpha1.DNSNamingPolicy{Spec: dnsv1alpha1.DNSNamingPolicySpec{ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: contextRef.Name}, AdditionalNames: []dnsv1alpha1.DNSAdditionalNameRule{{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: ref.Name}, NamePrefix: "services"}}}}, true}, + {"naming-empty-zone", &dnsv1alpha1.DNSNamingPolicy{Spec: dnsv1alpha1.DNSNamingPolicySpec{ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: contextRef.Name}, AdditionalNames: []dnsv1alpha1.DNSAdditionalNameRule{{DNSZoneRef: dnsv1alpha1.DNSNameReference{}, NamePrefix: "services"}}}}, false}, } for _, tc := range cases { tc.object.SetName(tc.name) From 3a4297eef0d565d58d950e8401e2bbd2315cd5d5 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 14 Oct 2026 17:04:23 -0500 Subject: [PATCH 12/23] test: scope runtime checks to its API --- .../runtime/deployment_permissions_test.go | 26 +++++-------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/internal/internaldns/runtime/deployment_permissions_test.go b/internal/internaldns/runtime/deployment_permissions_test.go index ed105150..e47d4536 100644 --- a/internal/internaldns/runtime/deployment_permissions_test.go +++ b/internal/internaldns/runtime/deployment_permissions_test.go @@ -161,18 +161,6 @@ func TestEveryMemberCanCreateAllRuntimeConsumers(t *testing.T) { t.Fatal(err) } contents := string(data) - start := strings.Index(contents, `user: "internal-dns-control-plane"`) - if start < 0 { - t.Fatal("missing control-plane broker identity") - } - controlPolicy := strings.SplitN(contents[start:], "\n },", 2)[0] - controlACK := "$JS.ACK.DNS_PRIVATE." + durable(roleControlPlane, "us-central1", "shared-0", "acks") + ".>" - if !strings.Contains(controlPolicy, `"`+controlACK+`"`) { - t.Errorf("control plane cannot acknowledge its member-ACK consumer: %s", controlACK) - } - if strings.Contains(controlPolicy, `"$JS.ACK.DNS_PRIVATE.>"`) { - t.Error("control plane can acknowledge another serving member's consumers") - } for _, member := range []string{"node-us-central1-0", "regional-front-us-central1-0", "regional-coredns-us-central1-0", "regional-coredns-us-central1-1"} { start := strings.Index(contents, `user: "`+member+`"`) if start < 0 { @@ -265,13 +253,13 @@ func verifyProjectReferenceSchemas(t *testing.T, ctx context.Context, cl client. object client.Object valid bool }{ - {"registration-name-only", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 30}}, true}, - {"registration-empty-zone", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{}, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 30}}, false}, - {"registration-ttl31", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: ref.Name}, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 31}}, false}, - {"association-valid", &dnsv1alpha1.DNSZoneAssociation{Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: ref.Name}, ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: contextRef.Name}}}, true}, - {"association-empty-zone", &dnsv1alpha1.DNSZoneAssociation{Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{}, ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: contextRef.Name}}}, false}, - {"naming-valid", &dnsv1alpha1.DNSNamingPolicy{Spec: dnsv1alpha1.DNSNamingPolicySpec{ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: contextRef.Name}, AdditionalNames: []dnsv1alpha1.DNSAdditionalNameRule{{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: ref.Name}, NamePrefix: "services"}}}}, true}, - {"naming-empty-zone", &dnsv1alpha1.DNSNamingPolicy{Spec: dnsv1alpha1.DNSNamingPolicySpec{ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: contextRef.Name}, AdditionalNames: []dnsv1alpha1.DNSAdditionalNameRule{{DNSZoneRef: dnsv1alpha1.DNSNameReference{}, NamePrefix: "services"}}}}, false}, + {"registration-valid", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: ref, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 30}}, true}, + {"registration-unpinned", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name}, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 30}}, false}, + {"registration-ttl31", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: ref, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 31}}, false}, + {"association-valid", &dnsv1alpha1.DNSZoneAssociation{Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: ref, ResolverContextRef: contextRef}}, true}, + {"association-unpinned", &dnsv1alpha1.DNSZoneAssociation{Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name}, ResolverContextRef: contextRef}}, false}, + {"naming-valid", &dnsv1alpha1.DNSNamingPolicy{Spec: dnsv1alpha1.DNSNamingPolicySpec{ResolverContextRef: contextRef, AdditionalNames: []dnsv1alpha1.DNSAdditionalNameRule{{Selector: nil, DNSZoneRef: ref, NamePrefix: "services"}}}}, true}, + {"naming-unpinned", &dnsv1alpha1.DNSNamingPolicy{Spec: dnsv1alpha1.DNSNamingPolicySpec{ResolverContextRef: contextRef, AdditionalNames: []dnsv1alpha1.DNSAdditionalNameRule{{Selector: nil, DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name}, NamePrefix: "services"}}}}, false}, } for _, tc := range cases { tc.object.SetName(tc.name) From c917421e7681d9fdbd462f2804c31251703af1b9 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 14 Oct 2026 17:07:32 -0500 Subject: [PATCH 13/23] fix: validate the packaged CoreDNS entry point --- cmd/internal-dns-coredns/main.go | 11 +++++++---- config/internal-dns/Dockerfile | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/cmd/internal-dns-coredns/main.go b/cmd/internal-dns-coredns/main.go index 2112164a..6a96edd4 100644 --- a/cmd/internal-dns-coredns/main.go +++ b/cmd/internal-dns-coredns/main.go @@ -15,12 +15,15 @@ import ( ) func main() { + flags := flag.NewFlagSet("coredns", flag.ExitOnError) var path string var validate, version bool - flag.StringVar(&path, "config", "", "Absolute path to the DNS context snapshot") - flag.BoolVar(&validate, "validate", false, "Validate the DNS context snapshot and exit") - flag.BoolVar(&version, "version", false, "Print the CoreDNS version and exit") - flag.Parse() + flags.StringVar(&path, "config", "", "Absolute path to the DNS context snapshot") + flags.BoolVar(&validate, "validate", false, "Validate the DNS context snapshot and exit") + flags.BoolVar(&version, "version", false, "Print the CoreDNS version and exit") + if err := flags.Parse(os.Args[1:]); err != nil { + os.Exit(2) + } if version { version := "unknown" if info, ok := debug.ReadBuildInfo(); ok { diff --git a/config/internal-dns/Dockerfile b/config/internal-dns/Dockerfile index e5194791..c4e0c2c9 100644 --- a/config/internal-dns/Dockerfile +++ b/config/internal-dns/Dockerfile @@ -31,7 +31,7 @@ COPY config/internal-dns/commands/ /usr/local/sbin/ USER 65532:65532 RUN for tool in bash ip nc coredns dnsdist pdns_recursor; do \ command -v "$tool" >/dev/null || exit 1; \ - done && test -x /internal-dns + done && test -x /internal-dns && coredns -version RUN for helper in /usr/local/sbin/internal-dns-*; do \ test -x "$helper" && bash -n "$helper" || exit 1; \ done From cc7791a8d5543e36b55db0197895db9ab18ea5a0 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 14 Oct 2026 17:34:41 -0500 Subject: [PATCH 14/23] style: format the CoreDNS command imports --- cmd/internal-dns-coredns/main.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/internal-dns-coredns/main.go b/cmd/internal-dns-coredns/main.go index 6a96edd4..b87dbcdc 100644 --- a/cmd/internal-dns-coredns/main.go +++ b/cmd/internal-dns-coredns/main.go @@ -10,8 +10,9 @@ import ( "os/signal" "syscall" - "go.miloapis.com/dns-operator/internal/internaldns/coredns" "runtime/debug" + + "go.miloapis.com/dns-operator/internal/internaldns/coredns" ) func main() { From 75ec4a519037cc751ef2a01d1213404484bea33b Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 15 Oct 2026 07:27:50 -0500 Subject: [PATCH 15/23] feat: run upstream dnsdist directly Apply validated routing snapshots through the authenticated loopback console. Keep accepted routing available across configuration agent restarts and let Kubernetes manage the upstream dnsdist container. --- cmd/internal-dns/main.go | 2 +- config/internal-dns/Dockerfile | 5 +- config/internal-dns/README.md | 22 +- .../internal-dns/commands/internal-dns-gate | 31 +- .../commands/internal-dns-reload-dnsdist | 15 +- .../commands/internal-dns-validate-dnsdist | 2 +- config/internal-dns/kustomization.yaml | 1 + config/internal-dns/routers.example.yaml | 290 ++++++++++++++++++ internal/internaldns/runtime/config.go | 5 +- internal/internaldns/runtime/run.go | 2 + internal/internaldns/serving/coredns_test.go | 2 +- internal/internaldns/serving/daemons.go | 8 +- internal/internaldns/serving/dnsdist.go | 100 ++++++ internal/internaldns/serving/dnsdist_test.go | 78 +++++ internal/internaldns/serving/render.go | 87 ++++-- internal/internaldns/serving/transaction.go | 2 +- 16 files changed, 589 insertions(+), 63 deletions(-) create mode 100644 config/internal-dns/routers.example.yaml create mode 100644 internal/internaldns/serving/dnsdist.go create mode 100644 internal/internaldns/serving/dnsdist_test.go diff --git a/cmd/internal-dns/main.go b/cmd/internal-dns/main.go index efe6fc8e..41a98c9a 100644 --- a/cmd/internal-dns/main.go +++ b/cmd/internal-dns/main.go @@ -15,7 +15,7 @@ import ( func main() { var role, configPath, kubeconfig string - flag.StringVar(&role, "role", "", "Process role: control-plane, agent, serving, or readiness") + flag.StringVar(&role, "role", "", "Process role: control-plane, agent, serving, readiness, or prepare") flag.StringVar(&configPath, "config", "", "Absolute path to the internal DNS JSON configuration") // controller-runtime/client-go may install this standard flag from an init // hook. Reuse it when present so the command keeps one stable contract. diff --git a/config/internal-dns/Dockerfile b/config/internal-dns/Dockerfile index c4e0c2c9..07e9d1f4 100644 --- a/config/internal-dns/Dockerfile +++ b/config/internal-dns/Dockerfile @@ -21,9 +21,10 @@ COPY --from=build /out/internal-dns /internal-dns USER 65532:65532 ENTRYPOINT ["/internal-dns"] -FROM debian:trixie-slim AS serving +FROM powerdns/dnsdist-20:2.0.10@sha256:24ac720af997aff6b98cb5b50d748754f815428e07b5368dfb0eccd5aa8d07e8 AS serving +USER 0:0 RUN apt-get update && apt-get install --no-install-recommends -y \ - dnsdist pdns-recursor iproute2 procps netcat-openbsd ca-certificates bash && \ + pdns-recursor iproute2 procps netcat-openbsd ca-certificates bash && \ rm -rf /var/lib/apt/lists/* COPY --from=build /out/internal-dns /internal-dns COPY --from=build /out/coredns /usr/local/bin/coredns diff --git a/config/internal-dns/README.md b/config/internal-dns/README.md index ecb09899..3ff577cd 100644 --- a/config/internal-dns/README.md +++ b/config/internal-dns/README.md @@ -5,7 +5,7 @@ Replace the example images, addresses, certificates, storage classes, admission CA injection, and site command wrappers before deployment. CI publishes `ghcr.io/datum-cloud/internal-dns` for the control plane and -`ghcr.io/datum-cloud/internal-dns-serving` for fleet agents, CoreDNS, and dnsdist. +`ghcr.io/datum-cloud/internal-dns-serving` for fleet agents and CoreDNS. dnsdist runs directly from its upstream image. The serving image includes the validation, activation, and gating commands in `config/internal-dns/commands`. The Kubernetes qualification environment uses the same commands. @@ -18,8 +18,8 @@ feature-branch tags and uses these immutable image references. The fixed fleet has three roles: -- A `node` resolver agent owns the shared node dnsdist/CoreDNS pair. -- A `regional-dnsdist` resolver agent owns the shared regional frontend and its +- A `node` resolver agent configures the shared node dnsdist/CoreDNS pair. +- A `regional-dnsdist` resolver agent configures the shared regional frontend and its pool of regional CoreDNS members. - Two or more `regional-coredns` combined agents independently compile the complete DNS context snapshot for their member, validate it with @@ -36,8 +36,24 @@ checkpoint PVC, configuration PVC, and durable activation state. Treat that set as a single-writer unit. A rolling update must allocate a new member identity and storage; do not run two pods against one checkpoint. +Kubernetes manages the dnsdist process. An init container creates a private, +pod-local control key and an initial configuration that refuses every context. +The agent validates complete routing snapshots and replaces the rule chain over +the authenticated loopback console. Failed validation retains active rules. +Backend eligibility uses immutable pools shared by identical member sets; +context identity remains the original trusted destination. Listener or backend +topology changes require a replacement member with independent storage. Agent and broker outages retain applied +routing. The control key is generated locally and is never a DNS API field. + +`routers.example.yaml` supplies the node and regional frontend workloads. +`fleet.example.yaml` supplies their configurations and the regional CoreDNS +workload template. Give every member independent configuration and checkpoint +storage. CoreDNS process activation uses the serving role; dnsdist runs directly +as its container process. + ## Control-plane and API boundaries + Control-plane replicas discover ready Milo projects through the core API and watch every namespace in each project control plane. New projects are included without restarting the service. `projectDiscovery.kubeconfig` authenticates diff --git a/config/internal-dns/commands/internal-dns-gate b/config/internal-dns/commands/internal-dns-gate index 3a610076..79b5def9 100755 --- a/config/internal-dns/commands/internal-dns-gate +++ b/config/internal-dns/commands/internal-dns-gate @@ -1,19 +1,20 @@ #!/bin/sh -# Clearing enable files before stopping processes prevents automatic restart. set -eu -rm -f /run/internal-dns/allow-coredns /run/internal-dns/allow-dnsdist +case "$1" in + node) config=/etc/internal-dns/node-dnsdist.conf; port=5199 ;; + regional-front) config=/etc/internal-dns/cluster-dnsdist.conf; port=5299 ;; + regional-coredns) config= ;; + *) echo 'unknown fleet role' >&2; exit 2 ;; +esac +if [ -n "$config" ] && nc -z -w 1 127.0.0.1 "$port" >/dev/null 2>&1; then + output=$(timeout 5s dnsdist -C "$config" -c -e 'setRules({newRuleAction(AllRule(),RCodeAction(DNSRCode.SERVFAIL))}); print("INTERNAL_DNS_GATED")') + [ "$output" = 'INTERNAL_DNS_GATED' ] || { printf '%s\n' "$output" >&2; exit 1; } +fi +rm -f /run/internal-dns/allow-coredns pkill -TERM -x coredns || true -pkill -TERM -x dnsdist || true -for daemon in coredns dnsdist; do - for attempt in $(seq 1 40); do - if ! pgrep -x "$daemon" >/dev/null; then break; fi - sleep 0.05 - done - if pgrep -x "$daemon" >/dev/null; then - pkill -KILL -x "$daemon" || true - fi - if pgrep -x "$daemon" >/dev/null; then - echo "DNS process did not stop: $daemon" >&2 - exit 1 - fi +for attempt in $(seq 1 40); do + if ! pgrep -x coredns >/dev/null; then exit 0; fi + sleep 0.05 done +pkill -KILL -x coredns || true +if pgrep -x coredns >/dev/null; then echo 'CoreDNS did not stop' >&2; exit 1; fi diff --git a/config/internal-dns/commands/internal-dns-reload-dnsdist b/config/internal-dns/commands/internal-dns-reload-dnsdist index 562b584d..7f4db1e9 100755 --- a/config/internal-dns/commands/internal-dns-reload-dnsdist +++ b/config/internal-dns/commands/internal-dns-reload-dnsdist @@ -1,10 +1,9 @@ #!/bin/sh set -eu -touch /run/internal-dns/allow-dnsdist -pkill -TERM -x dnsdist || true -for attempt in $(seq 1 35); do - if nc -z -w 1 ::1 53 >/dev/null 2>&1; then exit 0; fi - sleep 0.05 -done -echo 'dnsdist listener did not become ready' >&2 -exit 1 +case "$1" in + node) config=/etc/internal-dns/node-dnsdist.conf ;; + regional) config=/etc/internal-dns/cluster-dnsdist.conf ;; + *) echo 'unknown dnsdist role' >&2; exit 2 ;; +esac +output=$(timeout 5s dnsdist -C "$config" -c -e "dofile(\"$config\"); print(\"INTERNAL_DNS_APPLIED\")") +[ "$output" = 'INTERNAL_DNS_APPLIED' ] || { printf '%s\n' "$output" >&2; exit 1; } diff --git a/config/internal-dns/commands/internal-dns-validate-dnsdist b/config/internal-dns/commands/internal-dns-validate-dnsdist index 20ecff4e..77e1f323 100755 --- a/config/internal-dns/commands/internal-dns-validate-dnsdist +++ b/config/internal-dns/commands/internal-dns-validate-dnsdist @@ -1,3 +1,3 @@ #!/bin/sh set -eu -exec dnsdist --check-config -C "$2" +INTERNAL_DNS_VALIDATE=1 exec dnsdist --check-config -C "$2" diff --git a/config/internal-dns/kustomization.yaml b/config/internal-dns/kustomization.yaml index 32a75931..4fca26ba 100644 --- a/config/internal-dns/kustomization.yaml +++ b/config/internal-dns/kustomization.yaml @@ -5,4 +5,5 @@ resources: - rbac.yaml - control-plane.example.yaml - fleet.example.yaml +- routers.example.yaml - admission.example.yaml diff --git a/config/internal-dns/routers.example.yaml b/config/internal-dns/routers.example.yaml new file mode 100644 index 00000000..31355a8e --- /dev/null +++ b/config/internal-dns/routers.example.yaml @@ -0,0 +1,290 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: internal-dns-node-0 + namespace: internal-dns-system +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: internal-dns-node-0 + template: + metadata: + labels: + app.kubernetes.io/name: internal-dns-node-0 + internal-dns.datum.net/tier: node + spec: + serviceAccountName: internal-dns-fleet + automountServiceAccountToken: false + shareProcessNamespace: true + securityContext: + fsGroup: 65532 + fsGroupChangePolicy: OnRootMismatch + containers: + - name: agent + image: example.invalid/dns-operator/internal-dns-serving:VERSION + args: + - "--role=agent" + - "--config=/config/config.json" + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + capabilities: + drop: + - ALL + volumeMounts: + - name: config + mountPath: "/config" + readOnly: true + - name: nats + mountPath: "/var/run/internal-dns/nats" + readOnly: true + - name: activation + mountPath: "/run/internal-dns" + - name: state + mountPath: "/var/lib/internal-dns" + - name: daemon-config + mountPath: "/etc/internal-dns" + - name: serving + image: example.invalid/dns-operator/internal-dns-serving:VERSION + command: + - "/internal-dns" + args: + - "--role=serving" + - "--config=/config/config.json" + readinessProbe: + exec: + command: + - "/internal-dns" + - "--role=readiness" + - "--config=/config/config.json" + periodSeconds: 2 + timeoutSeconds: 2 + securityContext: &1 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsUser: 65532 + runAsGroup: 65532 + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + volumeMounts: + - name: config + mountPath: "/config" + readOnly: true + - name: state + mountPath: "/var/lib/internal-dns" + - name: activation + mountPath: "/run/internal-dns" + - name: daemon-config + mountPath: "/etc/internal-dns" + - name: runtime + mountPath: "/tmp" + - name: dnsdist + image: powerdns/dnsdist-20:2.0.10@sha256:24ac720af997aff6b98cb5b50d748754f815428e07b5368dfb0eccd5aa8d07e8 + command: + - "/usr/local/bin/dnsdist" + args: + - "--supervised" + - "--disable-syslog" + - "-C" + - "/etc/internal-dns/node-dnsdist.conf" + readinessProbe: + tcpSocket: + port: 53 + periodSeconds: 2 + securityContext: *1 + volumeMounts: + - name: daemon-config + mountPath: "/etc/internal-dns" + readOnly: true + - name: activation + mountPath: "/run/internal-dns" + readOnly: true + - name: runtime + mountPath: "/tmp" + volumes: + - name: config + configMap: + name: internal-dns-node-0-config + - name: nats + secret: + secretName: internal-dns-node-0-nats + - name: activation + emptyDir: + medium: Memory + - name: runtime + emptyDir: {} + - name: daemon-config + persistentVolumeClaim: + claimName: internal-dns-node-0-config + - name: state + persistentVolumeClaim: + claimName: internal-dns-node-0-state + initContainers: + - name: dnsdist-config + image: example.invalid/dns-operator/internal-dns-serving:VERSION + args: + - "--role=prepare" + - "--config=/config/config.json" + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + capabilities: + drop: + - ALL + volumeMounts: + - name: config + mountPath: "/config" + readOnly: true + - name: nats + mountPath: "/var/run/internal-dns/nats" + readOnly: true + - name: activation + mountPath: "/run/internal-dns" + - name: state + mountPath: "/var/lib/internal-dns" + - name: daemon-config + mountPath: "/etc/internal-dns" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: internal-dns-regional-front-0 + namespace: internal-dns-system +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: internal-dns-regional-front-0 + template: + metadata: + labels: + app.kubernetes.io/name: internal-dns-regional-front-0 + internal-dns.datum.net/tier: regional + spec: + serviceAccountName: internal-dns-fleet + automountServiceAccountToken: false + shareProcessNamespace: true + securityContext: + fsGroup: 65532 + fsGroupChangePolicy: OnRootMismatch + containers: + - name: agent + image: example.invalid/dns-operator/internal-dns-serving:VERSION + args: + - "--role=agent" + - "--config=/config/config.json" + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + capabilities: + drop: + - ALL + volumeMounts: + - name: config + mountPath: "/config" + readOnly: true + - name: nats + mountPath: "/var/run/internal-dns/nats" + readOnly: true + - name: activation + mountPath: "/run/internal-dns" + - name: state + mountPath: "/var/lib/internal-dns" + - name: daemon-config + mountPath: "/etc/internal-dns" + readinessProbe: + exec: + command: + - "/internal-dns" + - "--role=readiness" + - "--config=/config/config.json" + periodSeconds: 2 + timeoutSeconds: 2 + - name: dnsdist + image: powerdns/dnsdist-20:2.0.10@sha256:24ac720af997aff6b98cb5b50d748754f815428e07b5368dfb0eccd5aa8d07e8 + command: + - "/usr/local/bin/dnsdist" + args: + - "--supervised" + - "--disable-syslog" + - "-C" + - "/etc/internal-dns/cluster-dnsdist.conf" + readinessProbe: + tcpSocket: + port: 53 + periodSeconds: 2 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsUser: 65532 + runAsGroup: 65532 + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + volumeMounts: + - name: daemon-config + mountPath: "/etc/internal-dns" + readOnly: true + - name: activation + mountPath: "/run/internal-dns" + readOnly: true + - name: runtime + mountPath: "/tmp" + volumes: + - name: config + configMap: + name: internal-dns-regional-front-0-config + - name: nats + secret: + secretName: internal-dns-regional-front-0-nats + - name: activation + emptyDir: + medium: Memory + - name: runtime + emptyDir: {} + - name: daemon-config + persistentVolumeClaim: + claimName: internal-dns-regional-front-0-config + - name: state + persistentVolumeClaim: + claimName: internal-dns-regional-front-0-state + initContainers: + - name: dnsdist-config + image: example.invalid/dns-operator/internal-dns-serving:VERSION + args: + - "--role=prepare" + - "--config=/config/config.json" + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + capabilities: + drop: + - ALL + volumeMounts: + - name: config + mountPath: "/config" + readOnly: true + - name: nats + mountPath: "/var/run/internal-dns/nats" + readOnly: true + - name: activation + mountPath: "/run/internal-dns" + - name: state + mountPath: "/var/lib/internal-dns" + - name: daemon-config + mountPath: "/etc/internal-dns" diff --git a/internal/internaldns/runtime/config.go b/internal/internaldns/runtime/config.go index 8d1e9fae..7083b548 100644 --- a/internal/internaldns/runtime/config.go +++ b/internal/internaldns/runtime/config.go @@ -20,6 +20,7 @@ import ( const ( roleReadiness = "readiness" roleServing = "serving" + rolePrepare = "prepare" roleControlPlane = "control-plane" memberRoleRegional = "regional" roleAgent = "agent" @@ -164,11 +165,11 @@ func validateKubeAPI(name string, c KubeAPIClientConfig) error { func (c *Config) Validate(role string) error { c.defaults() switch role { - case roleControlPlane, roleAgent, roleReadiness, roleServing: + case roleControlPlane, roleAgent, roleReadiness, roleServing, rolePrepare: default: return fmt.Errorf("unsupported role %q", role) } - if role == roleReadiness || role == roleServing { + if role == roleReadiness || role == roleServing || role == rolePrepare { if c.Agent.Region == "" { c.Agent.Region = c.Region } diff --git a/internal/internaldns/runtime/run.go b/internal/internaldns/runtime/run.go index fd6c445a..803c01b2 100644 --- a/internal/internaldns/runtime/run.go +++ b/internal/internaldns/runtime/run.go @@ -51,6 +51,8 @@ func Run(ctx context.Context, role string, cfg Config, kubeconfig string) error return runAgent(ctx, cfg) case roleReadiness: return serving.Readiness(ctx, cfg.Agent) + case rolePrepare: + return serving.PrepareDNSDist(cfg.Agent.Render) case roleServing: return serving.RunDaemons(ctx, cfg.Agent) default: diff --git a/internal/internaldns/serving/coredns_test.go b/internal/internaldns/serving/coredns_test.go index f2942faf..20502f07 100644 --- a/internal/internaldns/serving/coredns_test.go +++ b/internal/internaldns/serving/coredns_test.go @@ -307,7 +307,7 @@ func TestRegionalPoolRequiresOneMemberWithEveryCurrentZone(t *testing.T) { t.Fatal(err) } conf := string(rendered.Files[c.ClusterDNSDist.Path]) - if strings.Count(conf, "server:addPool(") != 1 { + if strings.Count(conf, "]:addPool(") != 1 { t.Fatalf("stale live member remains in context pool:\n%s", conf) } // A newer ACK without the matching publication plan gates even the formerly diff --git a/internal/internaldns/serving/daemons.go b/internal/internaldns/serving/daemons.go index a11db457..a20830ac 100644 --- a/internal/internaldns/serving/daemons.go +++ b/internal/internaldns/serving/daemons.go @@ -63,20 +63,14 @@ func RunDaemons(ctx context.Context, config Config) error { addBind := func(bind CoreDNSRenderConfig) { processes = append(processes, &daemonProcess{name: "coredns", config: bind.Path, args: []string{"-config", bind.Path}}) } - addRouter := func(router DNSDistRenderConfig) { - processes = append(processes, &daemonProcess{name: "dnsdist", config: router.Path, args: []string{"--supervised", "-C", router.Path}}) - } + switch config.Render.Role { case RenderNode: addBind(config.Render.NodeCoreDNS) - addRouter(config.Render.NodeDNSDist) case RenderRegionalCoreDNS: addBind(config.Render.ClusterCoreDNS) - case RenderRegionalDNSDist: - addRouter(config.Render.ClusterDNSDist) case RenderCluster: addBind(config.Render.ClusterCoreDNS) - addRouter(config.Render.ClusterDNSDist) default: return fmt.Errorf("unsupported serving process role %q", config.Render.Role) } diff --git a/internal/internaldns/serving/dnsdist.go b/internal/internaldns/serving/dnsdist.go new file mode 100644 index 00000000..e08b2f07 --- /dev/null +++ b/internal/internaldns/serving/dnsdist.go @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package serving + +import ( + "crypto/rand" + "encoding/base64" + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +const DNSDistKeyPath = "/run/internal-dns/dnsdist.key" +const DNSDistImage = "powerdns/dnsdist-20:2.0.10@sha256:24ac720af997aff6b98cb5b50d748754f815428e07b5368dfb0eccd5aa8d07e8" + +func dnsdistConsole(path string) string { + if strings.Contains(filepath.Base(path), "node") { + return "127.0.0.1:5199" + } + return "127.0.0.1:5299" +} + +func PrepareDNSDist(config RenderConfig) error { + if err := validateRenderConfig(config); err != nil { + return err + } + var routers []DNSDistRenderConfig + switch config.Role { + case RenderNode: + routers = append(routers, config.NodeDNSDist) + case RenderRegionalDNSDist: + routers = append(routers, config.ClusterDNSDist) + case RenderRegionalCoreDNS: + return nil + default: + return fmt.Errorf("unsupported Kubernetes fleet role %q", config.Role) + } + if err := prepareDNSDistKey(DNSDistKeyPath); err != nil { + return err + } + for _, router := range routers { + if err := prepareDNSDistRouter(router); err != nil { + return err + } + } + return nil +} + +func prepareDNSDistRouter(router DNSDistRenderConfig) error { + if len(router.Backends) == 0 { + return errors.New("dnsdist startup requires fixed shared backends") + } + if err := os.MkdirAll(filepath.Dir(router.Path), 0750); err != nil { + return err + } + if _, err := os.Stat(router.Path); err == nil { + return nil + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + temporary := router.Path + ".bootstrap" + if err := writeDurable(temporary, renderDNSDist(router, router.Backends, nil, nil, false, nil, nil), 0644); err != nil { + return err + } + if err := os.Rename(temporary, router.Path); err != nil { + return err + } + if err := syncDirs([]string{router.Path}); err != nil { + return err + } + return nil +} + +func prepareDNSDistKey(path string) error { + if err := os.MkdirAll(filepath.Dir(path), 0750); err != nil { + return err + } + data, err := os.ReadFile(path) + if err == nil { + key, decodeErr := base64.StdEncoding.DecodeString(strings.TrimSpace(string(data))) + if decodeErr != nil || len(key) != 32 { + return errors.New("invalid dnsdist control key") + } + return nil + } + if !errors.Is(err, os.ErrNotExist) { + return err + } + key := make([]byte, 32) + if _, err := rand.Read(key); err != nil { + return err + } + temporary := path + ".tmp" + if err := writeDurable(temporary, []byte(base64.StdEncoding.EncodeToString(key)+"\n"), 0600); err != nil { + return err + } + return os.Rename(temporary, path) +} diff --git a/internal/internaldns/serving/dnsdist_test.go b/internal/internaldns/serving/dnsdist_test.go new file mode 100644 index 00000000..f6573db9 --- /dev/null +++ b/internal/internaldns/serving/dnsdist_test.go @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package serving + +import ( + "encoding/base64" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestDNSDistKeyPersistsAcrossAgentRestarts(t *testing.T) { + path := filepath.Join(t.TempDir(), "runtime", "dnsdist.key") + if err := prepareDNSDistKey(path); err != nil { + t.Fatal(err) + } + first, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if err := prepareDNSDistKey(path); err != nil { + t.Fatal(err) + } + second, err := os.ReadFile(path) + if err != nil || string(first) != string(second) { + t.Fatal("control identity changed", err) + } + decoded, err := base64.StdEncoding.DecodeString(strings.TrimSpace(string(first))) + if err != nil || len(decoded) != 32 { + t.Fatal("invalid control identity", err) + } + info, err := os.Stat(path) + if err != nil || info.Mode().Perm() != 0600 { + t.Fatal("control key must be private", err) + } + if err := os.WriteFile(path, []byte("invalid"), 0600); err != nil { + t.Fatal(err) + } + if err := prepareDNSDistKey(path); err == nil { + t.Fatal("invalid retained key was accepted") + } +} + +func TestDNSDistBootstrapPublishesOnlyCompleteConfiguration(t *testing.T) { + config, _ := renderFixture(t) + router := config.NodeDNSDist + router.Path = filepath.Join(t.TempDir(), "node-dnsdist.conf") + if err := os.Mkdir(router.Path+".bootstrap", 0700); err != nil { + t.Fatal(err) + } + if err := prepareDNSDistRouter(router); err == nil { + t.Fatal("failed staging was accepted") + } + if _, err := os.Stat(router.Path); !os.IsNotExist(err) { + t.Fatal("failed staging exposed a startup file", err) + } + if err := os.Remove(router.Path + ".bootstrap"); err != nil { + t.Fatal(err) + } + if err := prepareDNSDistRouter(router); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(router.Path) + if err != nil || !strings.Contains(string(data), "DNSAction.Refused") || !strings.Contains(string(data), "setRules(rules)") { + t.Fatal("bootstrap must refuse unknown contexts", err) + } + if err := os.WriteFile(router.Path, []byte("accepted snapshot"), 0644); err != nil { + t.Fatal(err) + } + if err := prepareDNSDistRouter(router); err != nil { + t.Fatal(err) + } + retained, err := os.ReadFile(router.Path) + if err != nil || string(retained) != "accepted snapshot" { + t.Fatal("bootstrap replaced accepted routing", err) + } +} diff --git a/internal/internaldns/serving/render.go b/internal/internaldns/serving/render.go index e61aa3ea..ceec28e5 100644 --- a/internal/internaldns/serving/render.go +++ b/internal/internaldns/serving/render.go @@ -10,6 +10,7 @@ import ( "path/filepath" "sort" "strconv" + "strings" contextdns "go.miloapis.com/dns-operator/internal/internaldns/coredns" "go.miloapis.com/dns-operator/internal/internaldns/model" @@ -200,28 +201,54 @@ func collectBackends(bindings []model.Binding, get func(model.Binding) []model.B func renderDNSDist(c DNSDistRenderConfig, backends []model.Backend, bindings []model.Binding, active map[string]bool, cluster bool, eligible map[string]map[string]bool, zones map[string]map[string]map[string]bool) []byte { var out bytes.Buffer - fmt.Fprintf(&out, "setLocal(%q)\n", c.ListenAddress) - fmt.Fprint(&out, "setACL({") + endpoint := dnsdistConsole(c.Path) + fmt.Fprint(&out, "local client = isClient()\nif client or not internalDNSInitialized then\n") + fmt.Fprintf(&out, " local key\n if os.getenv(\"INTERNAL_DNS_VALIDATE\") == \"1\" then key = string.rep(\"A\",43)..\"=\" else\n local handle = assert(io.open(%q,\"r\"))\n key = handle:read(\"*l\"); handle:close()\n end\n setKey(key)\n controlSocket(%q)\n setConsoleACL({\"127.0.0.1/32\"})\nend\nif not client then\n", DNSDistKeyPath, endpoint) + topology, _ := json.Marshal(struct { + Listen string + ACLs []string + Backends []model.Backend + }{c.ListenAddress, c.ACLs, backends}) + fingerprint := model.OpaqueToken(string(topology)) + fmt.Fprint(&out, "if not internalDNSInitialized then\n") + fmt.Fprintf(&out, "setLocal(%q)\nsetACL({", c.ListenAddress) for i, acl := range c.ACLs { if i > 0 { fmt.Fprint(&out, ",") } fmt.Fprintf(&out, "%q", acl) } - fmt.Fprint(&out, "})\nsetSecurityPollSuffix(\"\")\n\n") - for _, b := range backends { - fmt.Fprintf(&out, "local server = newServer({address=%q, name=%q, pool=%q, useProxyProtocol=true, checkName=\".\", checkType=\"SOA\", mustResolve=false})\n", net.JoinHostPort(b.Address, strconv.Itoa(int(b.Port))), "b_"+model.OpaqueToken(b.MemberID), c.PoolName) - if cluster { - for _, binding := range bindings { - for _, zone := range binding.Zones { - if zones != nil && active[binding.BindingUID] && zones[binding.BindingUID][zone.ZoneUID][b.MemberID] { - fmt.Fprintf(&out, "server:addPool(%q)\n", binding.ViewName()+"_"+model.OpaqueToken(zone.ZoneUID)) - } + fmt.Fprint(&out, "})\nsetSecurityPollSuffix(\"\")\ninternalDNSServers = {}\ninternalDNSPools = {}\n") + for _, backend := range backends { + fmt.Fprintf(&out, "internalDNSServers[%q] = newServer({address=%q,name=%q,pool=%q,useProxyProtocol=true,checkName=\".\",checkType=\"SOA\",mustResolve=false})\n", backend.MemberID, net.JoinHostPort(backend.Address, strconv.Itoa(int(backend.Port))), "b_"+model.OpaqueToken(backend.MemberID), c.PoolName) + } + fmt.Fprintf(&out, "internalDNSTopology = %q\ninternalDNSInitialized = true\nend\nassert(internalDNSTopology == %q,\"dnsdist topology requires a replacement member\")\n", fingerprint, fingerprint) + fmt.Fprint(&out, "local rules = {}\nlocal function appendRule(rule,action) rules[#rules+1] = newRuleAction(rule,action) end\n") + if cluster { + pools := map[string]map[string]bool{} + for _, binding := range bindings { + if eligible != nil { + pools[eligiblePool(backends, eligible[binding.BindingUID])] = eligible[binding.BindingUID] + } + for _, zone := range binding.Zones { + if zones != nil { + pools[eligiblePool(backends, zones[binding.BindingUID][zone.ZoneUID])] = zones[binding.BindingUID][zone.ZoneUID] } - if !binding.ConfigurationPending && active[binding.BindingUID] && (eligible == nil || eligible[binding.BindingUID][b.MemberID]) { - fmt.Fprintf(&out, "server:addPool(%q)\n", binding.ViewName()) + } + } + names := make([]string, 0, len(pools)) + for name := range pools { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + fmt.Fprintf(&out, "if not internalDNSPools[%q] then\ngetPool(%q)\n", name, name) + for _, backend := range backends { + if pools[name][backend.MemberID] { + fmt.Fprintf(&out, "internalDNSServers[%q]:addPool(%q)\n", backend.MemberID, name) } } + fmt.Fprintf(&out, "internalDNSPools[%q] = true\nend\n", name) } } fmt.Fprint(&out, "\nlocal bindings = {\n") @@ -232,33 +259,49 @@ func renderDNSDist(c DNSDistRenderConfig, backends []model.Backend, bindings []m } fmt.Fprintf(&out, " [%q] = {context=%q, ready=%t},\n", addr, b.ViewName(), !b.ConfigurationPending && active[b.BindingUID]) } - fmt.Fprint(&out, "}\n\nfunction classifyContext(dq)\n local binding = bindings[dq.localaddr:toString()]\n if binding == nil then\n return DNSAction.Refused, \"\"\n end\n dq:setTag(\"dns-context\", binding.context)\n if not binding.ready then dq:setTag(\"context-unready\", \"true\") end\n return DNSAction.None, \"\"\nend\n\n") - fmt.Fprint(&out, "addAction(AllRule(), LuaAction(classifyContext))\n") - fmt.Fprint(&out, "addAction(TagRule(\"context-unready\", \"true\"), RCodeAction(DNSRCode.SERVFAIL))\n") + fmt.Fprint(&out, "}\n\nlocal function classifyContext(dq)\n local binding = bindings[dq.localaddr:toString()]\n if binding == nil then\n return DNSAction.Refused, \"\"\n end\n dq:setTag(\"dns-context\", binding.context)\n if not binding.ready then dq:setTag(\"context-unready\", \"true\") end\n return DNSAction.None, \"\"\nend\n\n") + fmt.Fprint(&out, "appendRule(AllRule(), LuaAction(classifyContext))\n") + fmt.Fprint(&out, "appendRule(TagRule(\"context-unready\", \"true\"), RCodeAction(DNSRCode.SERVFAIL))\n") for _, binding := range bindings { attached := append([]model.ZoneAttachment(nil), binding.Zones...) sort.Slice(attached, func(i, j int) bool { return len(attached[i].Apex) > len(attached[j].Apex) }) for _, zone := range attached { rule := fmt.Sprintf("AndRule({TagRule(\"dns-context\",%q),QNameSuffixRule(%q)})", binding.ViewName(), model.AbsoluteName(zone.Apex)) if zone.Pending || zones != nil && len(zones[binding.BindingUID][zone.ZoneUID]) == 0 { - fmt.Fprintf(&out, "addAction(%s,RCodeAction(DNSRCode.SERVFAIL))\n", rule) + fmt.Fprintf(&out, "appendRule(%s,RCodeAction(DNSRCode.SERVFAIL))\n", rule) } else if cluster && zones != nil { - pool := binding.ViewName() + "_" + model.OpaqueToken(zone.ZoneUID) - fmt.Fprintf(&out, "getPool(%q)\naddAction(AndRule({%s,NotRule(PoolAvailableRule(%q))}),RCodeAction(DNSRCode.SERVFAIL))\naddAction(%s,PoolAction(%q))\n", pool, rule, pool, rule, pool) + pool := eligiblePool(backends, zones[binding.BindingUID][zone.ZoneUID]) + fmt.Fprintf(&out, "getPool(%q)\nappendRule(AndRule({%s,NotRule(PoolAvailableRule(%q))}),RCodeAction(DNSRCode.SERVFAIL))\nappendRule(%s,PoolAction(%q))\n", pool, rule, pool, rule, pool) } } } if cluster { for _, binding := range bindings { - fmt.Fprintf(&out, "getPool(%q)\naddAction(AndRule({TagRule(\"dns-context\",%q),NotRule(PoolAvailableRule(%q))}), RCodeAction(DNSRCode.SERVFAIL))\naddAction(TagRule(\"dns-context\",%q),PoolAction(%q))\n", binding.ViewName(), binding.ViewName(), binding.ViewName(), binding.ViewName(), binding.ViewName()) + pool := eligiblePool(backends, eligible[binding.BindingUID]) + if eligible == nil { + pool = c.PoolName + } + fmt.Fprintf(&out, "getPool(%q)\nappendRule(AndRule({TagRule(\"dns-context\",%q),NotRule(PoolAvailableRule(%q))}), RCodeAction(DNSRCode.SERVFAIL))\nappendRule(TagRule(\"dns-context\",%q),PoolAction(%q))\n", pool, binding.ViewName(), pool, binding.ViewName(), pool) } - fmt.Fprint(&out, "addAction(AllRule(),RCodeAction(DNSRCode.SERVFAIL))\n") + fmt.Fprint(&out, "appendRule(AllRule(),RCodeAction(DNSRCode.SERVFAIL))\n") } else { - fmt.Fprintf(&out, "addAction(NotRule(PoolAvailableRule(%q)), RCodeAction(DNSRCode.SERVFAIL))\naddAction(AllRule(),PoolAction(%q))\n", c.PoolName, c.PoolName) + fmt.Fprintf(&out, "appendRule(NotRule(PoolAvailableRule(%q)), RCodeAction(DNSRCode.SERVFAIL))\nappendRule(AllRule(),PoolAction(%q))\n", c.PoolName, c.PoolName) } + fmt.Fprint(&out, "setRules(rules)\nend\n") return out.Bytes() } +func eligiblePool(backends []model.Backend, allowed map[string]bool) string { + members := make([]string, 0, len(allowed)) + for _, backend := range backends { + if allowed[backend.MemberID] { + members = append(members, backend.MemberID) + } + } + sort.Strings(members) + return "ready_" + model.OpaqueToken(strings.Join(members, "\x00")) +} + func renderNodeCoreDNS(c CoreDNSRenderConfig, bindings []model.Binding) []byte { cfg := coreDNSConfig(c) for _, b := range bindings { diff --git a/internal/internaldns/serving/transaction.go b/internal/internaldns/serving/transaction.go index bd8f756a..d504812a 100644 --- a/internal/internaldns/serving/transaction.go +++ b/internal/internaldns/serving/transaction.go @@ -344,5 +344,5 @@ func restoreFiles(paths []string, old map[string]oldFile) error { } func routingConfigurationChanged(path string, before, after []byte) bool { - return strings.HasSuffix(path, ".conf") && !bytes.Equal(before, after) + return strings.HasSuffix(path, ".conf") && !strings.HasSuffix(path, "-dnsdist.conf") && !bytes.Equal(before, after) } From fbcf54ae716e633eff060ab4fd6ff1f8e2adf312 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 15 Oct 2026 07:31:29 -0500 Subject: [PATCH 16/23] fix: detect native dnsdist console clients Use the supported client startup predicate in the shared configuration. --- internal/internaldns/serving/render.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/internaldns/serving/render.go b/internal/internaldns/serving/render.go index ceec28e5..8a4ad7f3 100644 --- a/internal/internaldns/serving/render.go +++ b/internal/internaldns/serving/render.go @@ -202,7 +202,7 @@ func collectBackends(bindings []model.Binding, get func(model.Binding) []model.B func renderDNSDist(c DNSDistRenderConfig, backends []model.Backend, bindings []model.Binding, active map[string]bool, cluster bool, eligible map[string]map[string]bool, zones map[string]map[string]map[string]bool) []byte { var out bytes.Buffer endpoint := dnsdistConsole(c.Path) - fmt.Fprint(&out, "local client = isClient()\nif client or not internalDNSInitialized then\n") + fmt.Fprint(&out, "local client = inClientStartup()\nif client or not internalDNSInitialized then\n") fmt.Fprintf(&out, " local key\n if os.getenv(\"INTERNAL_DNS_VALIDATE\") == \"1\" then key = string.rep(\"A\",43)..\"=\" else\n local handle = assert(io.open(%q,\"r\"))\n key = handle:read(\"*l\"); handle:close()\n end\n setKey(key)\n controlSocket(%q)\n setConsoleACL({\"127.0.0.1/32\"})\nend\nif not client then\n", DNSDistKeyPath, endpoint) topology, _ := json.Marshal(struct { Listen string From e891d5353846062fdae16db8408a47122eee729d Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 15 Oct 2026 07:38:03 -0500 Subject: [PATCH 17/23] fix: confirm native DNS routing activation Verify the control response before accepting routing updates and reuse module downloads in the runtime build. --- config/internal-dns/Dockerfile | 2 +- config/internal-dns/commands/internal-dns-gate | 2 +- config/internal-dns/commands/internal-dns-reload-dnsdist | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/internal-dns/Dockerfile b/config/internal-dns/Dockerfile index 07e9d1f4..1c86bf29 100644 --- a/config/internal-dns/Dockerfile +++ b/config/internal-dns/Dockerfile @@ -10,7 +10,7 @@ COPY cmd/internal-dns ./cmd/internal-dns COPY cmd/internal-dns-coredns ./cmd/internal-dns-coredns COPY api ./api COPY internal ./internal -RUN --mount=type=cache,target=/root/.cache/go-build \ +RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \ CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH \ go build -trimpath -tags netgo,osusergo -ldflags='-s -w' \ -o /out/internal-dns ./cmd/internal-dns && \ diff --git a/config/internal-dns/commands/internal-dns-gate b/config/internal-dns/commands/internal-dns-gate index 79b5def9..68862c6b 100755 --- a/config/internal-dns/commands/internal-dns-gate +++ b/config/internal-dns/commands/internal-dns-gate @@ -7,7 +7,7 @@ case "$1" in *) echo 'unknown fleet role' >&2; exit 2 ;; esac if [ -n "$config" ] && nc -z -w 1 127.0.0.1 "$port" >/dev/null 2>&1; then - output=$(timeout 5s dnsdist -C "$config" -c -e 'setRules({newRuleAction(AllRule(),RCodeAction(DNSRCode.SERVFAIL))}); print("INTERNAL_DNS_GATED")') + output=$(timeout 5s dnsdist -C "$config" -c -e 'setRules({newRuleAction(AllRule(),RCodeAction(DNSRCode.SERVFAIL))}); return "INTERNAL_DNS_GATED"') [ "$output" = 'INTERNAL_DNS_GATED' ] || { printf '%s\n' "$output" >&2; exit 1; } fi rm -f /run/internal-dns/allow-coredns diff --git a/config/internal-dns/commands/internal-dns-reload-dnsdist b/config/internal-dns/commands/internal-dns-reload-dnsdist index 7f4db1e9..717bd223 100755 --- a/config/internal-dns/commands/internal-dns-reload-dnsdist +++ b/config/internal-dns/commands/internal-dns-reload-dnsdist @@ -5,5 +5,5 @@ case "$1" in regional) config=/etc/internal-dns/cluster-dnsdist.conf ;; *) echo 'unknown dnsdist role' >&2; exit 2 ;; esac -output=$(timeout 5s dnsdist -C "$config" -c -e "dofile(\"$config\"); print(\"INTERNAL_DNS_APPLIED\")") +output=$(timeout 5s dnsdist -C "$config" -c -e "dofile(\"$config\"); return \"INTERNAL_DNS_APPLIED\"") [ "$output" = 'INTERNAL_DNS_APPLIED' ] || { printf '%s\n' "$output" >&2; exit 1; } From b8d2bf87034e70259a4849d6ad063b26a2b50cbf Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 15 Oct 2026 07:44:42 -0500 Subject: [PATCH 18/23] fix: use DNS query readiness for routers Verify the configured listener through the member readiness check. --- config/internal-dns/routers.example.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/config/internal-dns/routers.example.yaml b/config/internal-dns/routers.example.yaml index 31355a8e..a1b6472e 100644 --- a/config/internal-dns/routers.example.yaml +++ b/config/internal-dns/routers.example.yaml @@ -95,10 +95,6 @@ spec: - "--disable-syslog" - "-C" - "/etc/internal-dns/node-dnsdist.conf" - readinessProbe: - tcpSocket: - port: 53 - periodSeconds: 2 securityContext: *1 volumeMounts: - name: daemon-config @@ -221,10 +217,6 @@ spec: - "--disable-syslog" - "-C" - "/etc/internal-dns/cluster-dnsdist.conf" - readinessProbe: - tcpSocket: - port: 53 - periodSeconds: 2 securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true From f5852a02a0fe9f10a572dd02e85f764cdfa969be Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 21 Oct 2026 20:08:22 -0500 Subject: [PATCH 19/23] fix: isolate internal DNS deployment dependencies --- .github/workflows/publish.yml | 3 +- .../internal-dns/control-plane.example.yaml | 7 +++++ config/internal-dns/rbac.yaml | 29 ++++++++++++++++--- .../runtime/deployment_permissions_test.go | 26 +++++++++++++++++ internal/internaldns/runtime/run.go | 4 +++ 5 files changed, 64 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 27308d33..ba9996d9 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -24,7 +24,7 @@ jobs: publish-kustomize-bundles: # The bundle names the image by tag, so it must not be published until that image exists. - needs: [publish-container-image, publish-internal-dns-runtime, publish-internal-dns-serving] + needs: [publish-container-image] permissions: id-token: write contents: read @@ -73,6 +73,7 @@ jobs: secrets: inherit publish-internal-dns-builds: + if: github.event_name == 'release' || (github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/fix/internal-dns-name-references' || startsWith(github.ref, 'refs/tags/'))) needs: [publish-internal-dns-runtime, publish-internal-dns-serving, publish-kustomize-bundles] permissions: contents: read diff --git a/config/internal-dns/control-plane.example.yaml b/config/internal-dns/control-plane.example.yaml index 8b963394..58162d92 100644 --- a/config/internal-dns/control-plane.example.yaml +++ b/config/internal-dns/control-plane.example.yaml @@ -82,6 +82,13 @@ spec: args: ["--role=control-plane","--config=/etc/internal-dns/config.json","--kubeconfig=/var/run/internal-dns/platform/kubeconfig"] ports: - {name: admission, containerPort: 9443} + resources: + requests: {cpu: 100m, memory: 256Mi} + limits: {memory: 2Gi} + readinessProbe: + httpGet: {scheme: HTTPS, path: /readyz, port: admission} + periodSeconds: 5 + timeoutSeconds: 2 securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true diff --git a/config/internal-dns/rbac.yaml b/config/internal-dns/rbac.yaml index a050ad51..73f5394e 100644 --- a/config/internal-dns/rbac.yaml +++ b/config/internal-dns/rbac.yaml @@ -96,6 +96,26 @@ rules: - dnsresolvercontexts/status - dnsresolveraccessbindings/status verbs: [get, update, patch] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: internal-dns-control-plane +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: internal-dns-control-plane +subjects: +- kind: ServiceAccount + name: internal-dns-control-plane + namespace: internal-dns-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: internal-dns-control-plane-state + namespace: internal-dns-system +rules: - apiGroups: [coordination.k8s.io] resources: [leases] verbs: [get, list, watch, create, update, patch] @@ -104,13 +124,14 @@ rules: verbs: [get, list, watch, create, update, patch, delete] --- apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding +kind: RoleBinding metadata: - name: internal-dns-control-plane + name: internal-dns-control-plane-state + namespace: internal-dns-system roleRef: apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: internal-dns-control-plane + kind: Role + name: internal-dns-control-plane-state subjects: - kind: ServiceAccount name: internal-dns-control-plane diff --git a/internal/internaldns/runtime/deployment_permissions_test.go b/internal/internaldns/runtime/deployment_permissions_test.go index e47d4536..6cfab9ee 100644 --- a/internal/internaldns/runtime/deployment_permissions_test.go +++ b/internal/internaldns/runtime/deployment_permissions_test.go @@ -87,6 +87,7 @@ func TestDeploymentRolesAuthorizeWorkerAndDiscoveryBoundaries(t *testing.T) { t.Fatal(err) } } + verifyStateNamespaceBoundary(t, ctx, cl) for _, identity := range []struct{ name, role string }{{"product", "internal-dns-product-writer"}, {"integration", "internal-dns-vpc-integration"}, {"customer", "internal-dns-customer"}} { binding := &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: identity.name + "-test", Namespace: "example-project"}, Subjects: []rbacv1.Subject{{Kind: "ServiceAccount", Name: identity.name, Namespace: "example-project"}}, RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "Role", Name: identity.role}} if err := cl.Create(ctx, binding); err != nil { @@ -273,3 +274,28 @@ func verifyProjectReferenceSchemas(t *testing.T, ctx context.Context, cl client. } } } + +func verifyStateNamespaceBoundary(t *testing.T, ctx context.Context, cl client.Client) { + t.Helper() + for _, resource := range []struct{ group, name string }{{"", "configmaps"}, {"coordination.k8s.io", "leases"}} { + for _, namespace := range []string{"internal-dns-system", "example-project"} { + for _, verb := range []string{"get", "create", "patch"} { + allowed := namespace == "internal-dns-system" + deadline := time.Now().Add(5 * time.Second) + for { + review := &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{User: "system:serviceaccount:internal-dns-system:internal-dns-control-plane", Groups: []string{"system:authenticated", "system:serviceaccounts", "system:serviceaccounts:internal-dns-system"}, ResourceAttributes: &authorizationv1.ResourceAttributes{Namespace: namespace, Group: resource.group, Resource: resource.name, Verb: verb}}} + if err := cl.Create(ctx, review); err != nil { + t.Fatal(err) + } + if review.Status.Allowed == allowed { + break + } + if time.Now().After(deadline) { + t.Fatalf("state boundary %s %s in %s allowed=%t want=%t", verb, resource.name, namespace, review.Status.Allowed, allowed) + } + time.Sleep(20 * time.Millisecond) + } + } + } + } +} diff --git a/internal/internaldns/runtime/run.go b/internal/internaldns/runtime/run.go index 803c01b2..b24e687e 100644 --- a/internal/internaldns/runtime/run.go +++ b/internal/internaldns/runtime/run.go @@ -9,6 +9,7 @@ import ( "encoding/hex" "errors" "fmt" + "net/http" "os" "strings" "sync" @@ -183,6 +184,9 @@ func runControlPlane(ctx context.Context, cfg Config, kubeconfig string) error { ClientForRequest: directory.selectProject, } admissionManager.GetWebhookServer().Register(dnsadmission.Path, &kubeadmission.Webhook{Handler: handler}) + admissionManager.GetWebhookServer().Register("/readyz", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) } errCh := make(chan error, len(projects)+len(planners)*3+8) From 0d0397c8a333ea1f5568fe32a62a61e9b1855582 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 21 Oct 2026 20:29:30 -0500 Subject: [PATCH 20/23] fix: separate DNS query processes from broker credentials --- config/internal-dns/fleet.example.yaml | 4 +- config/internal-dns/routers.example.yaml | 16 +++++- .../runtime/deployment_examples_test.go | 52 +++++++++++++++++++ internal/internaldns/serving/activation.go | 4 +- .../internaldns/serving/activation_test.go | 22 ++++++++ internal/internaldns/serving/agent.go | 3 ++ internal/internaldns/serving/dnsdist.go | 4 +- internal/internaldns/serving/dnsdist_test.go | 4 +- internal/internaldns/serving/state.go | 23 +++++++- 9 files changed, 121 insertions(+), 11 deletions(-) diff --git a/config/internal-dns/fleet.example.yaml b/config/internal-dns/fleet.example.yaml index 75d636d8..c8503f94 100644 --- a/config/internal-dns/fleet.example.yaml +++ b/config/internal-dns/fleet.example.yaml @@ -90,7 +90,7 @@ spec: - name: agent image: example.invalid/dns-operator/internal-dns-serving:VERSION args: ["--role=agent","--config=/config/config.json"] - securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, runAsNonRoot: true, capabilities: {drop: [ALL]}} + securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, runAsNonRoot: true, runAsUser: 65532, runAsGroup: 65532, capabilities: {drop: [ALL], add: [KILL]}} volumeMounts: &member-mounts - {name: config, mountPath: /config, readOnly: true} - {name: nats, mountPath: /var/run/internal-dns/nats, readOnly: true} @@ -105,7 +105,7 @@ spec: exec: {command: ["/internal-dns", "--role=readiness", "--config=/config/config.json"]} periodSeconds: 2 timeoutSeconds: 2 - securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, runAsUser: 65532, runAsGroup: 65532, capabilities: {drop: [ALL], add: [NET_BIND_SERVICE]}} + securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, runAsUser: 65533, runAsGroup: 65532, capabilities: {drop: [ALL], add: [NET_BIND_SERVICE]}} volumeMounts: - {name: config, mountPath: /config, readOnly: true} - {name: state, mountPath: /var/lib/internal-dns} diff --git a/config/internal-dns/routers.example.yaml b/config/internal-dns/routers.example.yaml index a1b6472e..12ed3762 100644 --- a/config/internal-dns/routers.example.yaml +++ b/config/internal-dns/routers.example.yaml @@ -33,9 +33,13 @@ spec: allowPrivilegeEscalation: false readOnlyRootFilesystem: true runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 capabilities: drop: - ALL + add: + - KILL volumeMounts: - name: config mountPath: "/config" @@ -67,7 +71,7 @@ spec: securityContext: &1 allowPrivilegeEscalation: false readOnlyRootFilesystem: true - runAsUser: 65532 + runAsUser: 65533 runAsGroup: 65532 capabilities: drop: @@ -133,6 +137,8 @@ spec: allowPrivilegeEscalation: false readOnlyRootFilesystem: true runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 capabilities: drop: - ALL @@ -184,9 +190,13 @@ spec: allowPrivilegeEscalation: false readOnlyRootFilesystem: true runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 capabilities: drop: - ALL + add: + - KILL volumeMounts: - name: config mountPath: "/config" @@ -220,7 +230,7 @@ spec: securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true - runAsUser: 65532 + runAsUser: 65533 runAsGroup: 65532 capabilities: drop: @@ -264,6 +274,8 @@ spec: allowPrivilegeEscalation: false readOnlyRootFilesystem: true runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 capabilities: drop: - ALL diff --git a/internal/internaldns/runtime/deployment_examples_test.go b/internal/internaldns/runtime/deployment_examples_test.go index ba85464a..cb288ac1 100644 --- a/internal/internaldns/runtime/deployment_examples_test.go +++ b/internal/internaldns/runtime/deployment_examples_test.go @@ -9,6 +9,7 @@ import ( "testing" "go.miloapis.com/dns-operator/internal/internaldns/model" + appsv1 "k8s.io/api/apps/v1" k8syaml "k8s.io/apimachinery/pkg/util/yaml" ) @@ -87,3 +88,54 @@ func TestNATSExampleUsesExactMemberAckSubjects(t *testing.T) { } } } + +func TestQueryDaemonsCannotShareCredentialAgentIdentity(t *testing.T) { + for _, file := range []string{"fleet.example.yaml", "routers.example.yaml"} { + contents, err := os.ReadFile(filepath.Join("..", "..", "..", "config", "internal-dns", file)) + if err != nil { + t.Fatal(err) + } + decoder := k8syaml.NewYAMLOrJSONDecoder(bytes.NewReader(contents), 4096) + count := 0 + for { + var deployment appsv1.Deployment + if err := decoder.Decode(&deployment); err == io.EOF { + break + } else if err != nil { + t.Fatal(err) + } + if deployment.Kind != "Deployment" { + continue + } + for _, container := range deployment.Spec.Template.Spec.Containers { + if container.Name != "agent" && container.Name != "serving" && container.Name != "dnsdist" { + continue + } + count++ + expected := int64(65533) + if container.Name == "agent" { + expected = 65532 + } + sc := container.SecurityContext + if sc == nil || sc.RunAsUser == nil || *sc.RunAsUser != expected { + t.Fatalf("%s/%s must use UID %d", deployment.Name, container.Name, expected) + } + for _, capability := range sc.Capabilities.Add { + if capability == "SYS_PTRACE" { + t.Fatal("query Pods must not grant process inspection") + } + } + if container.Name != "agent" { + for _, mount := range container.VolumeMounts { + if mount.Name == "nats" { + t.Fatal("query daemon mounts broker credentials") + } + } + } + } + } + if count == 0 { + t.Fatalf("no query containers in %s", file) + } + } +} diff --git a/internal/internaldns/serving/activation.go b/internal/internaldns/serving/activation.go index 74a2c6dd..d9f8d1f4 100644 --- a/internal/internaldns/serving/activation.go +++ b/internal/internaldns/serving/activation.go @@ -28,7 +28,7 @@ func (t *FileTransaction) beginActivation() error { return err } if t.activationLock == nil { - lock, err := os.OpenFile(t.MarkerPath+".lock", os.O_CREATE|os.O_RDWR, 0o600) + lock, err := os.OpenFile(t.MarkerPath+".lock", os.O_CREATE|os.O_RDWR, 0o640) if err != nil { return err } @@ -98,7 +98,7 @@ func (t *FileTransaction) abandonActivation() error { // installation cannot race with a process loading its configuration files. func daemonPermission(stateDir string) (start, retain bool, release func()) { marker := activationMarkerPath(stateDir) - lock, err := os.OpenFile(marker+".lock", os.O_CREATE|os.O_RDWR, 0o600) + lock, err := os.Open(marker + ".lock") if err != nil { return false, false, func() {} } diff --git a/internal/internaldns/serving/activation_test.go b/internal/internaldns/serving/activation_test.go index 6ee5421f..e6af66e3 100644 --- a/internal/internaldns/serving/activation_test.go +++ b/internal/internaldns/serving/activation_test.go @@ -132,3 +132,25 @@ func TestRestoreActiveUsesInstalledConfigurationHash(t *testing.T) { t.Fatal("approved installed hash was not checkpointed") } } + +func TestServingStateUsesReadOnlyGroupAccess(t *testing.T) { + dir := t.TempDir() + checkpoint := dir + "/checkpoint.json" + if err := os.WriteFile(checkpoint, []byte("{}"), 0600); err != nil { + t.Fatal(err) + } + if err := prepareServingState(dir); err != nil { + t.Fatal(err) + } + for _, path := range []string{checkpoint, activationMarkerPath(dir) + ".lock"} { + info, err := os.Stat(path) + if err != nil || info.Mode().Perm() != 0640 { + t.Fatalf("serving group permissions on %s: %v %v", path, info, err) + } + } + start, retain, release := daemonPermission(dir) + release() + if !start || !retain { + t.Fatal("read-only descriptor cannot check accepted activation") + } +} diff --git a/internal/internaldns/serving/agent.go b/internal/internaldns/serving/agent.go index ad627a4f..4e384173 100644 --- a/internal/internaldns/serving/agent.go +++ b/internal/internaldns/serving/agent.go @@ -227,6 +227,9 @@ func NewAgent(c Config, d Dependencies) (*Agent, error) { } } store := stateStore{path: filepath.Join(c.StateDir, "checkpoint.json")} + if err := prepareServingState(c.StateDir); err != nil { + return nil, err + } state, err := store.Load() if err != nil { gateErr := (FileTransaction{Config: c.Transaction, Runner: d.Runner}).FailClosed(context.Background()) diff --git a/internal/internaldns/serving/dnsdist.go b/internal/internaldns/serving/dnsdist.go index e08b2f07..071ff5f9 100644 --- a/internal/internaldns/serving/dnsdist.go +++ b/internal/internaldns/serving/dnsdist.go @@ -83,7 +83,7 @@ func prepareDNSDistKey(path string) error { if decodeErr != nil || len(key) != 32 { return errors.New("invalid dnsdist control key") } - return nil + return os.Chmod(path, 0640) } if !errors.Is(err, os.ErrNotExist) { return err @@ -93,7 +93,7 @@ func prepareDNSDistKey(path string) error { return err } temporary := path + ".tmp" - if err := writeDurable(temporary, []byte(base64.StdEncoding.EncodeToString(key)+"\n"), 0600); err != nil { + if err := writeDurable(temporary, []byte(base64.StdEncoding.EncodeToString(key)+"\n"), 0640); err != nil { return err } return os.Rename(temporary, path) diff --git a/internal/internaldns/serving/dnsdist_test.go b/internal/internaldns/serving/dnsdist_test.go index f6573db9..485311ba 100644 --- a/internal/internaldns/serving/dnsdist_test.go +++ b/internal/internaldns/serving/dnsdist_test.go @@ -31,8 +31,8 @@ func TestDNSDistKeyPersistsAcrossAgentRestarts(t *testing.T) { t.Fatal("invalid control identity", err) } info, err := os.Stat(path) - if err != nil || info.Mode().Perm() != 0600 { - t.Fatal("control key must be private", err) + if err != nil || info.Mode().Perm() != 0640 { + t.Fatal("control key must be readable only by its owner and serving group", err) } if err := os.WriteFile(path, []byte("invalid"), 0600); err != nil { t.Fatal(err) diff --git a/internal/internaldns/serving/state.go b/internal/internaldns/serving/state.go index ebc122bc..f5036ae8 100644 --- a/internal/internaldns/serving/state.go +++ b/internal/internaldns/serving/state.go @@ -126,7 +126,7 @@ func (s stateStore) Save(state checkpoint) error { return err } temp := s.path + ".tmp" - if err := writeDurable(temp, b, 0o600); err != nil { + if err := writeDurable(temp, b, 0o640); err != nil { return err } if err := os.Rename(temp, s.path); err != nil { @@ -141,3 +141,24 @@ func (s stateStore) Save(state checkpoint) error { } func chunkKey(index int) string { return fmt.Sprint(index) } + +func prepareServingState(dir string) error { + if err := os.MkdirAll(dir, 0750); err != nil { + return err + } + lock, err := os.OpenFile(activationMarkerPath(dir)+".lock", os.O_CREATE|os.O_RDWR, 0640) + if err != nil { + return err + } + if err := lock.Chmod(0640); err != nil { + _ = lock.Close() + return err + } + if err := lock.Close(); err != nil { + return err + } + if err := os.Chmod(filepath.Join(dir, "checkpoint.json"), 0640); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return nil +} From 15315f6c0b64f26104103e2b81f5b4dfe828a10d Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 21 Oct 2026 20:31:36 -0500 Subject: [PATCH 21/23] fix: limit process signaling to CoreDNS agents --- config/internal-dns/routers.example.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/config/internal-dns/routers.example.yaml b/config/internal-dns/routers.example.yaml index 12ed3762..25b27c68 100644 --- a/config/internal-dns/routers.example.yaml +++ b/config/internal-dns/routers.example.yaml @@ -195,8 +195,6 @@ spec: capabilities: drop: - ALL - add: - - KILL volumeMounts: - name: config mountPath: "/config" From c0a8af4a2f6bb3472e653c3eee03dda56076243d Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 21 Oct 2026 20:34:31 -0500 Subject: [PATCH 22/23] fix: retain serving permissions after interrupted state writes --- internal/internaldns/serving/activation_test.go | 14 ++++++++++++++ internal/internaldns/serving/transaction.go | 4 ++++ 2 files changed, 18 insertions(+) diff --git a/internal/internaldns/serving/activation_test.go b/internal/internaldns/serving/activation_test.go index e6af66e3..b0384830 100644 --- a/internal/internaldns/serving/activation_test.go +++ b/internal/internaldns/serving/activation_test.go @@ -154,3 +154,17 @@ func TestServingStateUsesReadOnlyGroupAccess(t *testing.T) { t.Fatal("read-only descriptor cannot check accepted activation") } } + +func TestServingCheckpointReplacesInterruptedWritePermissions(t *testing.T) { + path := t.TempDir() + "/checkpoint.json.tmp" + if err := os.WriteFile(path, []byte("partial"), 0600); err != nil { + t.Fatal(err) + } + if err := writeDurable(path, []byte("complete"), 0640); err != nil { + t.Fatal(err) + } + info, err := os.Stat(path) + if err != nil || info.Mode().Perm() != 0640 { + t.Fatalf("retained temporary file prevents serving group access: %v %v", info, err) + } +} diff --git a/internal/internaldns/serving/transaction.go b/internal/internaldns/serving/transaction.go index d504812a..cb1e69c8 100644 --- a/internal/internaldns/serving/transaction.go +++ b/internal/internaldns/serving/transaction.go @@ -287,6 +287,10 @@ func writeDurable(path string, data []byte, mode os.FileMode) error { if err != nil { return err } + if err := f.Chmod(mode); err != nil { + _ = f.Close() + return err + } if _, err = f.Write(data); err != nil { _ = f.Close() return err From cd344421cbc6a51e8513ec05982469286d9218d1 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 21 Oct 2026 20:35:03 -0500 Subject: [PATCH 23/23] docs: explain DNS process credential boundaries --- config/internal-dns/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/config/internal-dns/README.md b/config/internal-dns/README.md index 3ff577cd..5449bc3f 100644 --- a/config/internal-dns/README.md +++ b/config/internal-dns/README.md @@ -51,6 +51,13 @@ workload template. Give every member independent configuration and checkpoint storage. CoreDNS process activation uses the serving role; dnsdist runs directly as its container process. +The agent runs as UID 65532; CoreDNS and dnsdist run as UID 65533. Query +containers do not mount broker credentials and cannot inspect the agent's +filesystem through the shared process namespace. Group 65532 can read accepted +configuration, checkpoints, and the pod-local console key. Only agents that +manage CoreDNS receive `KILL` to stop query processes during activation; no +container receives `SYS_PTRACE`. + ## Control-plane and API boundaries