diff --git a/infra/dns/README.md b/infra/dns/README.md index 2e1d6078de..7b4bb218d6 100644 --- a/infra/dns/README.md +++ b/infra/dns/README.md @@ -8,7 +8,8 @@ Authoritative zone files, one per domain. Imported into Cloudflare via - Zone: [`esa-blueshell.nl.zone`](./esa-blueshell.nl.zone) - Migrated off TransIP on 2026-04-21. - Apex `esa-blueshell.nl` serves the NixOS + k3s stack at - `157.173.115.164` / `2a02:c207:2316:2642::1`. Every service hostname + `157.173.115.164`; its IPv6 address is not published until the server + can serve it (#2640). Every service hostname the platform advertises (`api` lives at apex `/api`, plus admin hosts `kube`, `vault`, `stalwart`, `status`, `traefik`) resolves through that node — admin records are external-dns managed off each @@ -26,7 +27,7 @@ After import, toggle the orange cloud per record: | `@`, `www` | on | Public HTTP(S) behind Cloudflare. | | `stalwart` | on | Webadmin only — pure HTTP, proxy gives cache + DDoS shielding. | | `mail` | off | SMTP/IMAP/POP3 are raw TCP. Cloudflare proxy is HTTP(S)-only. | -| `@` (MX target = `mail`) | off | Inherits from the `mail` A/AAAA above. | +| `@` (MX target = `mail`) | off | Inherits from the `mail` A record above. | | `_dmarc`, `default._domainkey` | off | TXT records — proxy status is irrelevant but Cloudflare convention is "off". | | `minecraft` | off | Game traffic — Cloudflare proxy is HTTP(S) only. | | `ftp` | off | Non-HTTP protocol. | @@ -36,11 +37,11 @@ After import, toggle the orange cloud per record: Outbound mail signing + inbound delivery for `esa-blueshell.nl` use five records, all in this zone file: -- **`mail`** A/AAAA — non-proxied, pointing at the Frankfurt VPS public +- **`mail`** A — non-proxied, pointing at the Frankfurt VPS public IP. Stalwart binds the mail ports (25/465/587/993) on the host via `hostPort` on this address. - **`@` MX → `mail`** — RFC-mandated indirection; the MX target's - A/AAAA must be non-proxied so SMTP delivery reaches the pod + A record must be non-proxied so SMTP delivery reaches the pod directly. - **`@` SPF** (`v=spf1 mx ~all`) — authorises the MX hosts to send. Soft-fail (`~all`) for the initial rollout; flip to `-all` once diff --git a/infra/dns/esa-blueshell.nl.zone b/infra/dns/esa-blueshell.nl.zone index c138d77a16..958c1c3f7e 100644 --- a/infra/dns/esa-blueshell.nl.zone +++ b/infra/dns/esa-blueshell.nl.zone @@ -45,16 +45,7 @@ v2.esa-blueshell.nl. 1 IN A 157.173.115.164 ; cf_tags=cf-proxied:true vault.esa-blueshell.nl. 1 IN A 157.173.115.164 ; cf_tags=cf-proxied:true ;; AAAA Records -auth.esa-blueshell.nl. 1 IN AAAA 2a02:c207:2316:2642::1 ; cf_tags=cf-proxied:true -esa-blueshell.nl. 1 IN AAAA 2a02:c207:2316:2642::1 ; cf_tags=cf-proxied:true -headlamp.esa-blueshell.nl. 1 IN AAAA 2a02:c207:2316:2642::1 ; cf_tags=cf-proxied:true -kube.esa-blueshell.nl. 1 IN AAAA 2a02:c207:2316:2642::1 ; cf_tags=cf-proxied:true -mail.esa-blueshell.nl. 1 IN AAAA 2a02:c207:2316:2642::1 ; cf_tags=cf-proxied:false -stalwart.esa-blueshell.nl. 1 IN AAAA 2a02:c207:2316:2642::1 ; cf_tags=cf-proxied:true -status.esa-blueshell.nl. 1 IN AAAA 2a02:c207:2316:2642::1 ; cf_tags=cf-proxied:true -traefik.esa-blueshell.nl. 1 IN AAAA 2a02:c207:2316:2642::1 ; cf_tags=cf-proxied:true -v2.esa-blueshell.nl. 1 IN AAAA 2a02:c207:2316:2642::1 ; cf_tags=cf-proxied:true -vault.esa-blueshell.nl. 1 IN AAAA 2a02:c207:2316:2642::1 ; cf_tags=cf-proxied:true +;; None: the server cannot serve IPv6 yet (#2640). ;; CNAME Records ftp.esa-blueshell.nl. 1 IN CNAME esa-blueshell.nl. ; cf_tags=cf-proxied:false @@ -62,7 +53,7 @@ www.esa-blueshell.nl. 1 IN CNAME esa-blueshell.nl. ; cf_tags=cf-proxied:true ;; MX Records ;; The MX target must NOT be Cloudflare-proxied (mail is raw TCP, not -;; HTTP) — point it at mail.esa-blueshell.nl, whose A/AAAA are +;; HTTP) — point it at mail.esa-blueshell.nl, whose A record is ;; explicitly cf-proxied:false. Mail flowing to *@esa-blueshell.nl ;; therefore reaches the Stalwart pod directly on the VPS public IP. esa-blueshell.nl. 1 IN MX 10 mail.esa-blueshell.nl. diff --git a/platform/bootstrap/lib/blueshell.sh b/platform/bootstrap/lib/blueshell.sh index 80244e6c61..36b2f0fbb1 100644 --- a/platform/bootstrap/lib/blueshell.sh +++ b/platform/bootstrap/lib/blueshell.sh @@ -14,7 +14,8 @@ CONTEXT=${KUBE_CONTEXT:-blueshell} DOMAIN=esa-blueshell.nl MAIL_HOST=mail.esa-blueshell.nl SERVER_V4=157.173.115.164 -SERVER_V6=2a02:c207:2316:2642::1 +# Empty while the server does not serve IPv6 (#2640); mail.sh then checks IPv4 only. +SERVER_V6="" VAULT_NS=data-system VAULT_POD=vault-v2-0 diff --git a/platform/bootstrap/mail.sh b/platform/bootstrap/mail.sh index e8f2d45644..f76d9c7911 100755 --- a/platform/bootstrap/mail.sh +++ b/platform/bootstrap/mail.sh @@ -64,6 +64,19 @@ stage_cluster() { fi } +# server_addresses prints the addresses the server serves on, one a line. +server_addresses() { printf '%s\n' "$SERVER_V4"; [[ -z "$SERVER_V6" ]] || printf '%s\n' "$SERVER_V6"; } +expected_addresses() { server_addresses | paste -sd ' ' - | sed 's/ / and /'; } + +# serves_expected "IPS": the addresses DNS gives are exactly the server's, +# so an IPv6 address published while SERVER_V6 is empty counts as wrong. +serves_expected() { + local got want + got=$(tr ' ' '\n' <<<"$1" | grep -v '^$' | sort | tr '\n' ' ') + want=$(server_addresses | sort | tr '\n' ' ') + [[ "$got" == "$want" ]] +} + stage_dns() { local mx ips spf dmarc rua sel type rec sts tlsrpt stage "DNS: MX, SPF, DKIM, DMARC, MTA-STS and TLS-RPT (resolver $RESOLVER)" @@ -71,17 +84,17 @@ stage_dns() { [[ -n "$mx" ]] || { problem "$DOMAIN has no MX record"; } if [[ -n "$mx" ]]; then ips="$(q A "$mx") $(q AAAA "$mx")" - if [[ " $ips " == *" $SERVER_V4 "* && " $ips " == *" $SERVER_V6 "* ]]; then - ok "MX $mx reaches $SERVER_V4 and $SERVER_V6" + if serves_expected "$ips"; then + ok "MX $mx reaches $(expected_addresses)" else - problem "MX $mx resolves to $ips, not $SERVER_V4 and $SERVER_V6" + problem "MX $mx resolves to $ips, not $(expected_addresses)" fi fi ips="$(q A "$MAIL_HOST") $(q AAAA "$MAIL_HOST")" - if [[ " $ips " == *" $SERVER_V4 "* && " $ips " == *" $SERVER_V6 "* ]]; then - ok "$MAIL_HOST is $SERVER_V4 and $SERVER_V6" + if serves_expected "$ips"; then + ok "$MAIL_HOST is $(expected_addresses)" else - problem "$MAIL_HOST resolves to $ips (proxied through Cloudflare? mail needs it DNS-only)" + problem "$MAIL_HOST resolves to $ips, not $(expected_addresses) (a Cloudflare address means it is proxied; mail needs it DNS-only)" fi spf=$(txt "$DOMAIN" | grep '^v=spf1' || true) @@ -236,9 +249,9 @@ stage_rdns() { say "Outlook.com and Gmail distrust a server whose address does not name the host it greets as." while true; do bad=() - for ip in "$SERVER_V4" "$SERVER_V6"; do + while IFS= read -r ip; do if got=$(check_ptr "$ip"); then ok "$ip → $MAIL_HOST → $ip"; else warn "$ip → $got"; bad+=("$ip"); fi - done + done < <(server_addresses) (( ${#bad[@]} == 0 )) && return 0 tries=$((tries + 1)) if (( tries > 2 )); then diff --git a/platform/cluster/flux/apps/edge/ingressroutes/frontend.yaml b/platform/cluster/flux/apps/edge/ingressroutes/frontend.yaml index 5f9247d3d8..9d45bee2d2 100644 --- a/platform/cluster/flux/apps/edge/ingressroutes/frontend.yaml +++ b/platform/cluster/flux/apps/edge/ingressroutes/frontend.yaml @@ -6,7 +6,7 @@ metadata: annotations: # Main site, one label deep — CF Universal SSL + our Let's # Encrypt wildcard both cover it, CF proxy stays on. - external-dns.alpha.kubernetes.io/target: 157.173.115.164,2a02:c207:2316:2642::1 + external-dns.alpha.kubernetes.io/target: 157.173.115.164 external-dns.alpha.kubernetes.io/cloudflare-proxied: 'true' kubernetes.io/ingress.class: traefik-public spec: diff --git a/platform/cluster/flux/apps/edge/ingressroutes/headlamp.yaml b/platform/cluster/flux/apps/edge/ingressroutes/headlamp.yaml index 8c1ff25c1d..8ae8f7f1f5 100644 --- a/platform/cluster/flux/apps/edge/ingressroutes/headlamp.yaml +++ b/platform/cluster/flux/apps/edge/ingressroutes/headlamp.yaml @@ -5,7 +5,7 @@ metadata: namespace: edge-system annotations: # One label deep — CF Universal SSL covers it, proxy stays on. - external-dns.alpha.kubernetes.io/target: 157.173.115.164,2a02:c207:2316:2642::1 + external-dns.alpha.kubernetes.io/target: 157.173.115.164 external-dns.alpha.kubernetes.io/cloudflare-proxied: 'true' kubernetes.io/ingress.class: traefik-public spec: diff --git a/platform/cluster/flux/apps/edge/ingressroutes/mta-sts.yaml b/platform/cluster/flux/apps/edge/ingressroutes/mta-sts.yaml index 629ae424af..6c84a90e7f 100644 --- a/platform/cluster/flux/apps/edge/ingressroutes/mta-sts.yaml +++ b/platform/cluster/flux/apps/edge/ingressroutes/mta-sts.yaml @@ -4,7 +4,7 @@ metadata: name: mta-sts namespace: edge-system annotations: - external-dns.alpha.kubernetes.io/target: 157.173.115.164,2a02:c207:2316:2642::1 + external-dns.alpha.kubernetes.io/target: 157.173.115.164 external-dns.alpha.kubernetes.io/cloudflare-proxied: 'true' kubernetes.io/ingress.class: traefik-public spec: diff --git a/platform/cluster/flux/apps/edge/ingressroutes/stalwart-admin.yaml b/platform/cluster/flux/apps/edge/ingressroutes/stalwart-admin.yaml index 1a9a32f16a..6206ffc1e4 100644 --- a/platform/cluster/flux/apps/edge/ingressroutes/stalwart-admin.yaml +++ b/platform/cluster/flux/apps/edge/ingressroutes/stalwart-admin.yaml @@ -5,7 +5,7 @@ metadata: namespace: edge-system annotations: # One label deep — CF Universal SSL covers it, proxy stays on. - external-dns.alpha.kubernetes.io/target: 157.173.115.164,2a02:c207:2316:2642::1 + external-dns.alpha.kubernetes.io/target: 157.173.115.164 external-dns.alpha.kubernetes.io/cloudflare-proxied: 'true' kubernetes.io/ingress.class: traefik-public spec: diff --git a/platform/cluster/flux/apps/edge/ingressroutes/status.yaml b/platform/cluster/flux/apps/edge/ingressroutes/status.yaml index 7e2db8acc1..4b12cf4552 100644 --- a/platform/cluster/flux/apps/edge/ingressroutes/status.yaml +++ b/platform/cluster/flux/apps/edge/ingressroutes/status.yaml @@ -5,7 +5,7 @@ metadata: namespace: edge-system annotations: # One label deep — CF Universal SSL covers it, proxy stays on. - external-dns.alpha.kubernetes.io/target: 157.173.115.164,2a02:c207:2316:2642::1 + external-dns.alpha.kubernetes.io/target: 157.173.115.164 external-dns.alpha.kubernetes.io/cloudflare-proxied: 'true' kubernetes.io/ingress.class: traefik-public spec: diff --git a/platform/cluster/flux/apps/edge/ingressroutes/traefik-dashboard.yaml b/platform/cluster/flux/apps/edge/ingressroutes/traefik-dashboard.yaml index 4de55f5fba..e89e111ead 100644 --- a/platform/cluster/flux/apps/edge/ingressroutes/traefik-dashboard.yaml +++ b/platform/cluster/flux/apps/edge/ingressroutes/traefik-dashboard.yaml @@ -5,7 +5,7 @@ metadata: namespace: edge-system annotations: # One label deep — fits CF Universal SSL + our wildcard cert. - external-dns.alpha.kubernetes.io/target: 157.173.115.164,2a02:c207:2316:2642::1 + external-dns.alpha.kubernetes.io/target: 157.173.115.164 external-dns.alpha.kubernetes.io/cloudflare-proxied: 'true' kubernetes.io/ingress.class: traefik-public spec: diff --git a/platform/cluster/flux/apps/edge/ingressroutes/vault.yaml b/platform/cluster/flux/apps/edge/ingressroutes/vault.yaml index 07e1ac0311..e1097f327d 100644 --- a/platform/cluster/flux/apps/edge/ingressroutes/vault.yaml +++ b/platform/cluster/flux/apps/edge/ingressroutes/vault.yaml @@ -5,7 +5,7 @@ metadata: namespace: edge-system annotations: # One label deep — CF Universal SSL covers it, proxy stays on. - external-dns.alpha.kubernetes.io/target: 157.173.115.164,2a02:c207:2316:2642::1 + external-dns.alpha.kubernetes.io/target: 157.173.115.164 external-dns.alpha.kubernetes.io/cloudflare-proxied: 'true' kubernetes.io/ingress.class: traefik-public spec: diff --git a/platform/docs/bringup-v2.md b/platform/docs/bringup-v2.md index dbe8f8c146..71d480ef79 100644 --- a/platform/docs/bringup-v2.md +++ b/platform/docs/bringup-v2.md @@ -70,7 +70,8 @@ git commit -am "authorized-keys: rotate bs-deploy" && git push The flake at `platform/nix/hosts/frankfurt-contabo-1/default.nix` carries the full production addressing baked in: IPv4 `157.173.115.164/24` gateway `157.173.115.1`, IPv6 -`2a02:c207:2316:2642::1/64` gateway `fe80::1` (link-local). No further +`2a02:c207:2316:2642::1/64` (not configured until #2640: Contabo does +not route it yet). No further nix edits are needed before install. ```bash @@ -249,7 +250,7 @@ S3/B2 bucket: dump → upload → `kubectl cp` from workstation → restore. Cloudflare zone `esa-blueshell.nl`: - `esa-blueshell.nl` A → `157.173.115.164`, AAAA → - `2a02:c207:2316:2642::1`, both proxied (orange cloud). + `2a02:c207:2316:2642::1` once #2640 serves IPv6, both proxied (orange cloud). - `*.esa-blueshell.nl` — external-dns auto-materialises records from every IngressRoute; no manual action needed. - `stalwart.esa-blueshell.nl` A → `157.173.115.164`, **grey cloud** diff --git a/platform/nix/hosts/frankfurt-contabo-1/default.nix b/platform/nix/hosts/frankfurt-contabo-1/default.nix index f4a54872c2..d1907e1e14 100644 --- a/platform/nix/hosts/frankfurt-contabo-1/default.nix +++ b/platform/nix/hosts/frankfurt-contabo-1/default.nix @@ -39,9 +39,10 @@ # and cidata network-config: # prefix /20 (netmask 255.255.240.0, NOT /24) # gw4 157.173.112.1 (in-subnet on /20; NOT .115.1) - # v6 2a02:c207:2316:2642::1/64 - # gw6 fe80::1 (link-local — Contabo does not answer ND for - # it, so the route MUST be `onlink`). + # v6 2a02:c207:2316:2642::1/64, not configured: Contabo does not + # route it yet and the IPv4-only cluster cannot serve it, so + # an address here only makes the host try IPv6 and time out + # (#2640). useDHCP = lib.mkForce false; interfaces.ens18 = { useDHCP = false; @@ -51,24 +52,6 @@ prefixLength = 20; } ]; - ipv6.addresses = [ - { - address = "2a02:c207:2316:2642::1"; - prefixLength = 64; - } - ]; - # `networking.defaultGateway6` doesn't expose `onlink`. Without - # onlink, the kernel tries ND for fe80::1, which times out - # (Contabo's router doesn't advertise on that address) and the - # default v6 route becomes unusable. Drive the route directly. - ipv6.routes = [ - { - address = "::"; - prefixLength = 0; - via = "fe80::1"; - options = { onlink = ""; }; - } - ]; }; defaultGateway = { address = "157.173.112.1";