diff --git a/hack/create-kind-cluster.sh b/hack/create-kind-cluster.sh index feb8732540..8e4b7c8aa6 100755 --- a/hack/create-kind-cluster.sh +++ b/hack/create-kind-cluster.sh @@ -21,6 +21,7 @@ KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}" KUBECTL_CONTEXT="kind-${KIND_CLUSTER_NAME}" reg_name="kind-registry" reg_port="${KIND_REGISTRY_PORT:-5001}" +IPV6_DNS_UPSTREAM="${IPV6_DNS_UPSTREAM:-2001:4860:4860::8888 2001:4860:4860::8844}" if [[ $# -gt 0 ]]; then case "$1" in @@ -31,6 +32,9 @@ if [[ $# -gt 0 ]]; then echo "Configured through the environment:" echo " KIND_CLUSTER_NAME Name of the cluster to create (default: kind)." echo " IP_FAMILY Address families for pods and Services: ipv4, ipv6 or dual (default: ipv4)." + echo " IPV6_DNS_UPSTREAM Space-separated IPv6 resolvers CoreDNS forwards to when IP_FAMILY=ipv6" + echo " (default: Google Public DNS). These replace the host's resolver, so any" + echo " split-horizon names it served stop resolving from pods." exit 0 ;; esac @@ -135,6 +139,50 @@ fi echo "Creating kind cluster '${KIND_CLUSTER_NAME}'..." "${ROOT}"/hack/kind.sh create cluster --name "${KIND_CLUSTER_NAME}" --config "${ROOT}/bin/kind-config.yaml" +# kind create returns before the apiserver answers, and every kubectl below races +# it. Poll from inside the node, where the answer does not depend on how the +# daemon published the port. +echo "Waiting for the control plane to answer..." +for attempt in $(seq 60); do + if docker exec "${KIND_CLUSTER_NAME}-control-plane" \ + kubectl --kubeconfig=/etc/kubernetes/admin.conf get --raw /healthz >/dev/null 2>&1; then + break + fi + if [[ "${attempt}" == 60 ]]; then + echo "error: the control plane did not answer /healthz within 2m of create:" >&2 + echo " docker logs ${KIND_CLUSTER_NAME}-control-plane" >&2 + exit 1 + fi + sleep 2 +done + +# Where the daemon lives in a VM (Lima on macOS), kind publishes the port inside +# the VM and limactl re-forwards it to the host's v4 loopback only, so kind's +# own kubeconfig entry is unreachable. See +# https://github.com/lima-vm/lima/issues/1540 for more details. +# +# Probe that entry once and, if it is refused, repoint at localhost — the only +# spelling that reaches it: it falls back to the v4 loopback limactl forwards, +# and unlike 127.0.0.1 it is a SAN on the cert an IPv6-only cluster issues. +answers() { + kubectl --context="${KUBECTL_CONTEXT}" --server="$1" \ + --request-timeout=5s get --raw /healthz >/dev/null 2>&1 +} +server="$(kubectl config view \ + -o jsonpath="{.clusters[?(@.name==\"${KUBECTL_CONTEXT}\")].cluster.server}")" +if ! answers "${server}"; then + fallback="https://localhost:${server##*:}" + if ! answers "${fallback}"; then + echo "error: the apiserver answers inside the node but neither ${server} nor" >&2 + echo " ${fallback} reaches it from here. Check where the daemon" >&2 + echo " published the port, and whether it reaches this host:" >&2 + echo " docker port ${KIND_CLUSTER_NAME}-control-plane 6443" >&2 + exit 1 + fi + echo "The kubeconfig server ${server} is unreachable; repointing at ${fallback}..." + kubectl config set-cluster "${KUBECTL_CONTEXT}" --server="${fallback}" >/dev/null +fi + # A daemon with IPv6 off hands kind a v4-only network whatever it asked for. if [[ "${IP_FAMILY}" != "ipv4" && "$(docker network inspect kind --format '{{.EnableIPv6}}')" != "true" ]]; then @@ -145,21 +193,6 @@ if [[ "${IP_FAMILY}" != "ipv4" && exit 1 fi -# For ipv6 kind writes a kubeconfig pointing at [::1], the address it published -# the apiserver on, which only works for a client on the Docker host itself: a -# VM-hosted daemon (Lima on macOS) forwards the port to the *v4* loopback, so -# every kubectl below fails at connect. localhost is a SAN on the apiserver -# cert and lets the client pick a family that works from either side. -if [[ "${IP_FAMILY}" == "ipv6" ]]; then - server="$(kubectl config view \ - -o jsonpath="{.clusters[?(@.name==\"${KUBECTL_CONTEXT}\")].cluster.server}")" - if [[ "${server}" == "https://[::1]:"* ]]; then - echo "Repointing the kubeconfig for '${KUBECTL_CONTEXT}' at localhost..." - kubectl config set-cluster "${KUBECTL_CONTEXT}" \ - --server="https://localhost:${server##*:}" >/dev/null - fi -fi - # 2.5 Enable Proxy ARP/NDP on kind nodes for gVisor loopback pod-to-pod networking echo "Enabling Proxy ARP/NDP on kind nodes..." for node in $("${ROOT}"/hack/kind.sh get nodes --name "${KIND_CLUSTER_NAME}"); do @@ -196,6 +229,50 @@ if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name} docker network connect "kind" "${reg_name}" fi +# 4.5. Point CoreDNS at an IPv6 resolver and teach it the registry's name +if [[ "${IP_FAMILY}" == "ipv6" ]]; then + echo "Repointing CoreDNS at an IPv6 resolver and teaching it '${reg_name}'..." + reg_v6="$(docker inspect "${reg_name}" \ + --format '{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}' 2>/dev/null || true)" + if [[ -z "${reg_v6}" ]]; then + echo "error: '${reg_name}' has no IPv6 address on the 'kind' network" >&2 + exit 1 + fi + + # CoreDNS runs dnsPolicy: Default and inherits the node's IPv4 resolver, which + # no pod here can reach. + corefile="$(kubectl --context="${KUBECTL_CONTEXT}" -n kube-system get cm coredns \ + -o jsonpath='{.data.Corefile}')" + search="forward . /etc/resolv.conf" + # $search unquoted: bash 3.2 splices the quotes in literally. + patched="${corefile/$search/forward . ${IPV6_DNS_UPSTREAM}}" + if [[ "${patched}" == "${corefile}" ]]; then + echo "error: '${search}' not found in the CoreDNS Corefile" >&2 + echo " the Corefile layout changed upstream; update this block" >&2 + exit 1 + fi + + # Step 3's registry wiring is node-side, while atelet pulls from its own netns, + # where "kind-registry" does not resolve. Own zone, so no fallthrough is needed: + # only this name reaches the hosts stanza. + patched="${patched} +${reg_name}:53 { + errors + hosts { + ${reg_v6} ${reg_name} + } +}" + + # A YAML patch file avoids escaping the Corefile's newlines into JSON. + { printf 'data:\n Corefile: |\n'; printf '%s\n' "${patched}" | sed 's/^/ /'; } \ + > "${ROOT}/bin/coredns-patch.yaml" + kubectl --context="${KUBECTL_CONTEXT}" -n kube-system patch cm coredns \ + --type=merge --patch-file "${ROOT}/bin/coredns-patch.yaml" + kubectl --context="${KUBECTL_CONTEXT}" -n kube-system rollout restart deploy/coredns + kubectl --context="${KUBECTL_CONTEXT}" -n kube-system rollout status deploy/coredns \ + --timeout=120s +fi + # 5. Document the local registry in kube-public ConfigMap echo "Documenting local registry in cluster..." cat <