Skip to content

[WIP] hack: repoint the kind kubeconfig when it cannot reach the apiserver #1292

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Yuan Gao (ygao-g) wants to merge 3 commits into
base: main
Choose a base branch
from
+92 −15
Open

[WIP] hack: repoint the kind kubeconfig when it cannot reach the apiserver #1292

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9084d8e
hack: drop the IPv6 kubeconfig repoint
ygao-g Aug 27, 2026
1cb5a78
hack: fix DNS on IPv6-only kind clusters
ygao-g Aug 27, 2026
84d3c80
hack: repoint the kind kubeconfig when it cannot reach the apiserver
ygao-g Aug 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 92 additions & 15 deletions hack/create-kind-cluster.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}"
KUBECTL_CONTEXT="kind-${KIND_CLUSTER_NAME}"
reg_name="kind-registry"
reg_port="${KIND_REGISTRY_PORT:-5001}"
IPV6_DNS_UPSTREAM="${IPV6_DNS_UPSTREAM:-2001:4860:4860::8888 2001:4860:4860::8844}"

if [[ $# -gt 0 ]]; then
case "$1" in
Expand All @@ -31,6 +32,9 @@ if [[ $# -gt 0 ]]; then
echo "Configured through the environment:"
echo " KIND_CLUSTER_NAME Name of the cluster to create (default: kind)."
echo " IP_FAMILY Address families for pods and Services: ipv4, ipv6 or dual (default: ipv4)."
echo " IPV6_DNS_UPSTREAM Space-separated IPv6 resolvers CoreDNS forwards to when IP_FAMILY=ipv6"
echo " (default: Google Public DNS). These replace the host's resolver, so any"
echo " split-horizon names it served stop resolving from pods."
exit 0
;;
esac
Expand Down Expand Up @@ -135,6 +139,50 @@ fi
echo "Creating kind cluster '${KIND_CLUSTER_NAME}'..."
"${ROOT}"/hack/kind.sh create cluster --name "${KIND_CLUSTER_NAME}" --config "${ROOT}/bin/kind-config.yaml"

# kind create returns before the apiserver answers, and every kubectl below races
# it. Poll from inside the node, where the answer does not depend on how the
# daemon published the port.
echo "Waiting for the control plane to answer..."
for attempt in $(seq 60); do
if docker exec "${KIND_CLUSTER_NAME}-control-plane" \
kubectl --kubeconfig=/etc/kubernetes/admin.conf get --raw /healthz >/dev/null 2>&1; then
break
fi
if [[ "${attempt}" == 60 ]]; then
echo "error: the control plane did not answer /healthz within 2m of create:" >&2
echo " docker logs ${KIND_CLUSTER_NAME}-control-plane" >&2
exit 1
fi
sleep 2
done

# Where the daemon lives in a VM (Lima on macOS), kind publishes the port inside
# the VM and limactl re-forwards it to the host's v4 loopback only, so kind's
# own kubeconfig entry is unreachable. See
# https://github.com/lima-vm/lima/issues/1540 for more details.
#
# Probe that entry once and, if it is refused, repoint at localhost — the only
# spelling that reaches it: it falls back to the v4 loopback limactl forwards,
# and unlike 127.0.0.1 it is a SAN on the cert an IPv6-only cluster issues.
answers() {
kubectl --context="${KUBECTL_CONTEXT}" --server="$1" \
--request-timeout=5s get --raw /healthz >/dev/null 2>&1
}
server="$(kubectl config view \
-o jsonpath="{.clusters[?(@.name==\"${KUBECTL_CONTEXT}\")].cluster.server}")"
if ! answers "${server}"; then
fallback="https://localhost:${server##*:}"
if ! answers "${fallback}"; then
echo "error: the apiserver answers inside the node but neither ${server} nor" >&2
echo " ${fallback} reaches it from here. Check where the daemon" >&2
echo " published the port, and whether it reaches this host:" >&2
echo " docker port ${KIND_CLUSTER_NAME}-control-plane 6443" >&2
exit 1
fi
echo "The kubeconfig server ${server} is unreachable; repointing at ${fallback}..."
kubectl config set-cluster "${KUBECTL_CONTEXT}" --server="${fallback}" >/dev/null
fi

# A daemon with IPv6 off hands kind a v4-only network whatever it asked for.
if [[ "${IP_FAMILY}" != "ipv4" &&
"$(docker network inspect kind --format '{{.EnableIPv6}}')" != "true" ]]; then
Expand All @@ -145,21 +193,6 @@ if [[ "${IP_FAMILY}" != "ipv4" &&
exit 1
fi

# For ipv6 kind writes a kubeconfig pointing at [::1], the address it published
# the apiserver on, which only works for a client on the Docker host itself: a
# VM-hosted daemon (Lima on macOS) forwards the port to the *v4* loopback, so
# every kubectl below fails at connect. localhost is a SAN on the apiserver
# cert and lets the client pick a family that works from either side.
if [[ "${IP_FAMILY}" == "ipv6" ]]; then
server="$(kubectl config view \
-o jsonpath="{.clusters[?(@.name==\"${KUBECTL_CONTEXT}\")].cluster.server}")"
if [[ "${server}" == "https://[::1]:"* ]]; then
echo "Repointing the kubeconfig for '${KUBECTL_CONTEXT}' at localhost..."
kubectl config set-cluster "${KUBECTL_CONTEXT}" \
--server="https://localhost:${server##*:}" >/dev/null
fi
fi

# 2.5 Enable Proxy ARP/NDP on kind nodes for gVisor loopback pod-to-pod networking
echo "Enabling Proxy ARP/NDP on kind nodes..."
for node in $("${ROOT}"/hack/kind.sh get nodes --name "${KIND_CLUSTER_NAME}"); do
Expand Down Expand Up @@ -196,6 +229,50 @@ if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name}
docker network connect "kind" "${reg_name}"
fi

# 4.5. Point CoreDNS at an IPv6 resolver and teach it the registry's name
if [[ "${IP_FAMILY}" == "ipv6" ]]; then
echo "Repointing CoreDNS at an IPv6 resolver and teaching it '${reg_name}'..."
reg_v6="$(docker inspect "${reg_name}" \
--format '{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}' 2>/dev/null || true)"
if [[ -z "${reg_v6}" ]]; then
echo "error: '${reg_name}' has no IPv6 address on the 'kind' network" >&2
exit 1
fi

# CoreDNS runs dnsPolicy: Default and inherits the node's IPv4 resolver, which
# no pod here can reach.
corefile="$(kubectl --context="${KUBECTL_CONTEXT}" -n kube-system get cm coredns \
-o jsonpath='{.data.Corefile}')"
search="forward . /etc/resolv.conf"
# $search unquoted: bash 3.2 splices the quotes in literally.
patched="${corefile/$search/forward . ${IPV6_DNS_UPSTREAM}}"
if [[ "${patched}" == "${corefile}" ]]; then
echo "error: '${search}' not found in the CoreDNS Corefile" >&2
echo " the Corefile layout changed upstream; update this block" >&2
exit 1
fi

# Step 3's registry wiring is node-side, while atelet pulls from its own netns,
# where "kind-registry" does not resolve. Own zone, so no fallthrough is needed:
# only this name reaches the hosts stanza.
patched="${patched}
${reg_name}:53 {
errors
hosts {
${reg_v6} ${reg_name}
}
}"

# A YAML patch file avoids escaping the Corefile's newlines into JSON.
{ printf 'data:\n Corefile: |\n'; printf '%s\n' "${patched}" | sed 's/^/ /'; } \
> "${ROOT}/bin/coredns-patch.yaml"
kubectl --context="${KUBECTL_CONTEXT}" -n kube-system patch cm coredns \
--type=merge --patch-file "${ROOT}/bin/coredns-patch.yaml"
kubectl --context="${KUBECTL_CONTEXT}" -n kube-system rollout restart deploy/coredns
kubectl --context="${KUBECTL_CONTEXT}" -n kube-system rollout status deploy/coredns \
--timeout=120s
fi

# 5. Document the local registry in kube-public ConfigMap
echo "Documenting local registry in cluster..."
cat <<EOF | kubectl --context="${KUBECTL_CONTEXT}" apply -f -
Expand Down
Loading