diff --git a/.taskfiles/talos/Taskfile.yaml b/.taskfiles/talos/Taskfile.yaml index 801d6f9f08..1278ac1c0e 100644 --- a/.taskfiles/talos/Taskfile.yaml +++ b/.taskfiles/talos/Taskfile.yaml @@ -22,9 +22,12 @@ tasks: - which talosctl sops yq envsubst diff-node: - desc: Show what applying the rendered config would change on a node (dry run) [IP=required] - cmd: talosctl --nodes {{.IP}} apply-config --dry-run --file <(./render.sh "$({{.TALOS_DIR}}/nodes.sh {{.IP}} | cut -d' ' -f1)") + desc: Summarise what applying the rendered config would change on a node (dry run, no values shown) [IP=required] [FULL=false] + cmd: FULL={{.FULL}} ./diff.sh "$({{.TALOS_DIR}}/nodes.sh {{.IP}} | cut -d' ' -f1)" dir: "{{.TALOS_DIR}}" + vars: + # FULL=true prints talosctl's raw diff, which contains CA keys and tokens. + FULL: '{{.FULL | default "false"}}' requires: vars: [IP] preconditions: @@ -33,12 +36,11 @@ tasks: - which talosctl sops yq envsubst diff-cluster: - desc: Dry-run the rendered config against every node + desc: Dry-run the rendered config against every node (summary only, no values shown) dir: "{{.TALOS_DIR}}" cmd: | - ./nodes.sh | while read -r host ip _; do - echo "### $host ($ip)" - talosctl --nodes "$ip" apply-config --dry-run --file <(./render.sh "$host") | grep -E '^(@@|[+-] )' || echo "no changes" + ./nodes.sh | while read -r host _; do + ./diff.sh "$host" || true done preconditions: - test -f {{.SOPS_AGE_KEY_FILE}} @@ -57,7 +59,7 @@ tasks: INSECURE: '{{.INSECURE | default "false"}}' requires: vars: [IP] - prompt: Apply the rendered Talos config to {{.IP}} (diff shown first)... continue? + prompt: Apply the rendered Talos config to {{.IP}} (summary of the change shown first)... continue? preconditions: - sh: '{{if eq .INSECURE "true"}}exit 0{{else}}talosctl --nodes {{.IP}} get machineconfig{{end}}' msg: "Node {{.IP}} is not reachable or not configured. Use INSECURE=true for first-time setup." diff --git a/docs/src/operations/sso-cluster-access.md b/docs/src/operations/sso-cluster-access.md index 14eaa2021a..a909f69f45 100644 --- a/docs/src/operations/sso-cluster-access.md +++ b/docs/src/operations/sso-cluster-access.md @@ -24,12 +24,11 @@ unreachable Authentik degrades only OIDC token validation, never apiserver start `Headlamp OIDC` (confidential, `client_id: headlamp`). - **Issuers**: `https://sso.chelonianlabs.com/application/o/kubectl/` and `…/o/headlamp/` (trailing slash required). -- **apiserver config**: `kubernetes/bootstrap/talos/patches/controller/apiserver-oidc.yaml` - — the auth config is written via `machine.files` to `/var/lib/kubernetes/authentication/` - (must be a writable `/var` path — a `machine.files` entry under `/etc/kubernetes` makes - Talos overlay that dir read-only and breaks the kubelet's PKI writes) and mounted into - the apiserver pod. Adding another OIDC app = one more `jwt:` block. Applying this patch - requires a per-node reboot (`machine.files` changes are not immediate-mode). +- **apiserver config**: `kubernetes/bootstrap/talos/patches/controller/30-apiserver-oidc.yaml` + — a Talos `KubeAuthenticationConfig` document. Talos renders it into the apiserver's + `--authentication-config` file and restarts the apiserver when it changes (no reboot). + Anonymous access is turned off in the same document. Adding another OIDC app = one more + `jwt:` block. - **RBAC**: `kubernetes/apps/kube-system/oidc-rbac/` binds `oidc:kubernetes-admins` → `cluster-admin` and `oidc:kubernetes-viewers` → `view`; both issuers map claims with the same `oidc:` prefix, so bindings apply to both. diff --git a/kubernetes/bootstrap/talos/README.md b/kubernetes/bootstrap/talos/README.md index a3653e1610..96a671fb9f 100644 --- a/kubernetes/bootstrap/talos/README.md +++ b/kubernetes/bootstrap/talos/README.md @@ -15,9 +15,10 @@ the rendered output contains cluster secrets and is never written to disk by the | `talsecret.sops.yaml` | `talosctl gen secrets` bundle, SOPS-encrypted | | `patches/controller/*.yaml` | Control-plane-only patches | | `patches/global/*.yaml` | Patches for every node | -| `nodes//.yaml` | Per-node patch: install disk/image, labels, hostname and network documents | +| `nodes//.yaml` | Per-node patch: install image/disk, labels, hostname and network documents | | `render.sh ` | Prints the full machine config for a node | | `nodes.sh [ip\|hostname]` | Lists nodes as ` ` | +| `diff.sh ` | Dry run against the node; prints a summary of the change without values | | `clusterconfig/talosconfig` | Admin client config (gitignored) | The directory a node file sits in (`controlplane/` or `worker/`) is its role. @@ -32,30 +33,64 @@ and lists are appended: 3. `patches/global/*.yaml` 4. `nodes//.yaml` -Role patches come before global ones so list order matches what talhelper produced. Files apply -in name order, hence the numeric prefixes. `${TALOS_VERSION}` and `${KUBERNETES_VERSION}` are -substituted from `versions.yaml`, which is how the install images pick up the Talos version. +Files apply in name order, hence the numeric prefixes. `${TALOS_VERSION}` and +`${KUBERNETES_VERSION}` are substituted from `versions.yaml`, which is how the install images pick +up the Talos version. -`configContract` in `versions.yaml` is passed to `talosctl gen config --talos-version`. It stays at -`v1.13` until the patches are converted to the Talos 1.14 multi-document format; with a `v1.14` -contract the base config emits the new documents and the v1alpha1 patches conflict with them. +Patches are Talos 1.14 configuration documents (`kind: KubeAPIServerConfig`, `KubeletConfig`, +`ResolverConfig`, `UnattendedInstallConfig`, …). A patch document merges into the base document of +the same kind (and `name`, where the kind has one): maps merge, lists append, and +`$patch: delete` removes the base document. Only what has no document yet stays in the v1alpha1 +`machine:` / `cluster:` form: etcd settings, `machine.certSANs`, and the containerd customization in +`patches/global/20-machine-files.yaml` (see the comment there). + +Every secret (CAs, tokens, the service-account signing key, the etcd encryption key) comes from the +secrets bundle through the generated base. Patches never hold secret material. + +## Config contract + +`configContract` in `versions.yaml` is passed to `talosctl gen config --talos-version`. It decides +which documents the generated base contains **and their defaults**, so raising it is a migration, +not a version bump: diff the new base against the old one first. + +The move from `v1.13` to `v1.14` (November 2026) was done without changing node behaviour. Three +things hold the old behaviour in place and are worth knowing about: + +- `patches/global/70-pinned-defaults.yaml` removes the documents a v1.14 base adds that the cluster + never ran with: workload isolation, the secure `/var` mount and the weekly filesystem trim. +- `patches/controller/00-apiserver.yaml` sets `startupProbes: false`, and + `30-apiserver-oidc.yaml` sets `anonymous.enabled: false`. The v1.14 defaults are probes on and + anonymous access to the health paths. Change the two together or not at all: the probes are + anonymous requests, so probes on with anonymous off gets the apiserver killed. +- `render.sh` appends the `identity` provider to `KubeEtcdEncryptionConfig`, which is what Talos + generated before v1.14. + +To rotate the service-account key again, add a `KubeServiceAccountConfig` patch with +`accepted.publicKeys`; the signing key itself stays in the bundle. ## Tasks ```sh -task talos:diff-node IP=10.100.0.50 # dry run: what would change on one node -task talos:diff-cluster # dry run against every node -task talos:apply-node IP=10.100.0.50 # shows the diff, asks, then applies +task talos:diff-node IP=10.100.0.50 # dry run: summary of what would change on one node +task talos:diff-cluster # the same for every node +task talos:apply-node IP=10.100.0.50 # shows the summary, asks, then applies task talos:render-config IP=10.100.0.50 # print the config (contains secrets) ``` Always run a diff before applying. When the repo and the nodes agree, every node reports -`No changes.` +`no changes`. + +The diff tasks print a summary only: whether a reboot is needed, how many lines change, and which +keys and documents are added or removed. Values are never shown, because a machine config diff +contains CA keys and tokens. That makes the tasks safe in CI logs and agent sessions. +`task talos:diff-node IP=… FULL=true` prints talosctl's raw diff for when you need to see the +values yourself. ## Adding a node 1. Create `nodes//.yaml` (copy a sibling and change the MACs, addresses and disk). - `nodes.sh` finds a node by the first address on link `ethSel0`. + `nodes.sh` finds a node by the first address on link `ethSel0` and reads its install image + from the `UnattendedInstallConfig` document. 2. `task talos:add-node IP=` while the node is in maintenance mode. ## Schematics diff --git a/kubernetes/bootstrap/talos/diff.sh b/kubernetes/bootstrap/talos/diff.sh new file mode 100755 index 0000000000..03291d8786 --- /dev/null +++ b/kubernetes/bootstrap/talos/diff.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Dry-run the rendered config against a node and summarise what would change, without +# printing any value: the diff of a machine config contains CA keys and tokens. +# +# ./diff.sh summary: apply mode, line counts, changed keys and documents +# FULL=true ./diff.sh the raw diff from talosctl (contains secrets; needs a terminal +# you trust, never a log or an agent session) +set -euo pipefail + +dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +host="${1:?usage: diff.sh }" +ip="$("$dir/nodes.sh" "$host" | cut -d' ' -f2)" + +work="$(mktemp -d)" +chmod 700 "$work" +trap 'rm -rf "$work"' EXIT + +# talosctl writes the dry-run report (summary and diff) to stderr. +if ! talosctl --nodes "$ip" apply-config --dry-run --file <("$dir/render.sh" "$host") > "$work/out" 2>&1; then + # A failure message can quote config lines: show only its first line. + echo "$host ($ip): dry run failed: $(grep -vE '^([+ -]|@@|$)' "$work/out" | head -1 | cut -c1-200)" >&2 + exit 1 +fi + +if [ "${FULL:-false}" = "true" ]; then + cat "$work/out" + exit 0 +fi + +if grep -q 'No changes' "$work/out"; then + echo "$host ($ip): no changes" + exit 0 +fi + +mode="$(grep -E '^Applied configuration' "$work/out" | head -1 || true)" +added="$(grep -cE '^\+([^+]|$)' "$work/out" || true)" +removed="$(grep -cE '^-([^-]|$)' "$work/out" || true)" +echo "$host ($ip): ${mode:-changes} +$added -$removed lines" + +# Changed lines reduced to ":". Values, list items and block text are dropped. +keys() { grep -oE "^\\$1 *[A-Za-z0-9_.\$/-]+:" "$work/out" | sed -E "s/^\\$1//; s/:\$//" | sort -u | tr '\n' ' ' | sed -E 's/ +/ /g'; } +docs() { grep -E "^\\$1kind: [A-Za-z0-9]+\$" "$work/out" | sed -E "s/^\\$1kind: //" | sort | uniq -c | awk '{printf "%s%s ", $2, ($1 > 1 ? "×" $1 : "")}'; } +echo " documents added: $(docs +)" +echo " documents removed: $(docs -)" +echo " keys added: $(keys +)" +echo " keys removed: $(keys -)" +echo " (values are hidden; FULL=true shows the raw diff, secrets included)" diff --git a/kubernetes/bootstrap/talos/nodes.sh b/kubernetes/bootstrap/talos/nodes.sh index dfc80a0071..5a78ecb120 100755 --- a/kubernetes/bootstrap/talos/nodes.sh +++ b/kubernetes/bootstrap/talos/nodes.sh @@ -14,7 +14,7 @@ for f in "$dir"/nodes/*/*.yaml; do host="$(basename "$f" .yaml)" role="$(basename "$(dirname "$f")")" ip="$(yq -e 'select(.kind == "LinkConfig" and .name == "ethSel0") | .addresses[0].address' "$f" | cut -d/ -f1)" - image="$(yq -e 'select(document_index == 0) | .machine.install.image' "$f" | envsubst '${TALOS_VERSION}')" + image="$(yq -e 'select(.kind == "UnattendedInstallConfig") | .installer.image' "$f" | envsubst '${TALOS_VERSION}')" if [ $# -eq 0 ] || [ "$1" = "$ip" ] || [ "$1" = "$host" ]; then echo "$host $ip $role $image" found=1 diff --git a/kubernetes/bootstrap/talos/nodes/controlplane/talos-control-1.yaml b/kubernetes/bootstrap/talos/nodes/controlplane/talos-control-1.yaml index 58e9945ff6..c3ee7ab0d7 100644 --- a/kubernetes/bootstrap/talos/nodes/controlplane/talos-control-1.yaml +++ b/kubernetes/bootstrap/talos/nodes/controlplane/talos-control-1.yaml @@ -1,8 +1,12 @@ # talos-control-1 (10.100.0.50). Its role comes from the directory this file is in. -machine: - install: - disk: /dev/vda - image: factory.talos.dev/installer/5cc10e6e83c6e3b32167fe812094d4715c9fa3e1a77bcfe61fa4cf71b4b9fed3:${TALOS_VERSION} +apiVersion: v1alpha1 +kind: UnattendedInstallConfig +installer: + image: factory.talos.dev/installer/5cc10e6e83c6e3b32167fe812094d4715c9fa3e1a77bcfe61fa4cf71b4b9fed3:${TALOS_VERSION} +provisioning: + diskSelector: + match: disk.dev_path == "/dev/vda" + wipe: false --- apiVersion: v1alpha1 kind: HostnameConfig diff --git a/kubernetes/bootstrap/talos/nodes/controlplane/talos-control-2.yaml b/kubernetes/bootstrap/talos/nodes/controlplane/talos-control-2.yaml index 559f584990..de13cc1845 100644 --- a/kubernetes/bootstrap/talos/nodes/controlplane/talos-control-2.yaml +++ b/kubernetes/bootstrap/talos/nodes/controlplane/talos-control-2.yaml @@ -1,8 +1,12 @@ # talos-control-2 (10.100.0.51). Its role comes from the directory this file is in. -machine: - install: - disk: /dev/vda - image: factory.talos.dev/installer/5cc10e6e83c6e3b32167fe812094d4715c9fa3e1a77bcfe61fa4cf71b4b9fed3:${TALOS_VERSION} +apiVersion: v1alpha1 +kind: UnattendedInstallConfig +installer: + image: factory.talos.dev/installer/5cc10e6e83c6e3b32167fe812094d4715c9fa3e1a77bcfe61fa4cf71b4b9fed3:${TALOS_VERSION} +provisioning: + diskSelector: + match: disk.dev_path == "/dev/vda" + wipe: false --- apiVersion: v1alpha1 kind: HostnameConfig diff --git a/kubernetes/bootstrap/talos/nodes/controlplane/talos-control-3.yaml b/kubernetes/bootstrap/talos/nodes/controlplane/talos-control-3.yaml index b4104f9083..5fe8f250f7 100644 --- a/kubernetes/bootstrap/talos/nodes/controlplane/talos-control-3.yaml +++ b/kubernetes/bootstrap/talos/nodes/controlplane/talos-control-3.yaml @@ -1,8 +1,12 @@ # talos-control-3 (10.100.0.52). Its role comes from the directory this file is in. -machine: - install: - disk: /dev/vda - image: factory.talos.dev/installer/5cc10e6e83c6e3b32167fe812094d4715c9fa3e1a77bcfe61fa4cf71b4b9fed3:${TALOS_VERSION} +apiVersion: v1alpha1 +kind: UnattendedInstallConfig +installer: + image: factory.talos.dev/installer/5cc10e6e83c6e3b32167fe812094d4715c9fa3e1a77bcfe61fa4cf71b4b9fed3:${TALOS_VERSION} +provisioning: + diskSelector: + match: disk.dev_path == "/dev/vda" + wipe: false --- apiVersion: v1alpha1 kind: HostnameConfig diff --git a/kubernetes/bootstrap/talos/nodes/worker/talos-node-gpu-1.yaml b/kubernetes/bootstrap/talos/nodes/worker/talos-node-gpu-1.yaml index c3f41bf670..d9c54365b4 100644 --- a/kubernetes/bootstrap/talos/nodes/worker/talos-node-gpu-1.yaml +++ b/kubernetes/bootstrap/talos/nodes/worker/talos-node-gpu-1.yaml @@ -1,20 +1,40 @@ # talos-node-gpu-1 (10.100.0.53). Its role comes from the directory this file is in. -machine: - install: - disk: /dev/sda - image: factory.talos.dev/installer/1f6caf935b35f840ea19acb12d2dcc6d8f9c7d47fc621b1fcebb4a04c5e3130d:${TALOS_VERSION} - nodeLabels: - hardware.cluster.io/gpu: "p100" - nvidia.com/gpu.present: "true" - nvidia.com/mps.capable: "true" - kernel: - modules: - - name: nvidia - - name: nvidia_uvm - - name: nvidia_drm - - name: nvidia_modeset - sysctls: - net.core.bpf_jit_harden: 1 +apiVersion: v1alpha1 +kind: UnattendedInstallConfig +installer: + image: factory.talos.dev/installer/1f6caf935b35f840ea19acb12d2dcc6d8f9c7d47fc621b1fcebb4a04c5e3130d:${TALOS_VERSION} +provisioning: + diskSelector: + match: disk.dev_path == "/dev/sda" + wipe: false +--- +apiVersion: v1alpha1 +kind: KubeNodeConfig +labels: + hardware.cluster.io/gpu: "p100" + nvidia.com/gpu.present: "true" + nvidia.com/mps.capable: "true" +--- +apiVersion: v1alpha1 +kind: SysctlConfig +params: + net.core.bpf_jit_harden: "1" +--- +apiVersion: v1alpha1 +kind: KernelModuleConfig +name: nvidia +--- +apiVersion: v1alpha1 +kind: KernelModuleConfig +name: nvidia_uvm +--- +apiVersion: v1alpha1 +kind: KernelModuleConfig +name: nvidia_drm +--- +apiVersion: v1alpha1 +kind: KernelModuleConfig +name: nvidia_modeset --- apiVersion: v1alpha1 kind: HostnameConfig diff --git a/kubernetes/bootstrap/talos/nodes/worker/talos-node-large-1.yaml b/kubernetes/bootstrap/talos/nodes/worker/talos-node-large-1.yaml index 7661007f91..97783113df 100644 --- a/kubernetes/bootstrap/talos/nodes/worker/talos-node-large-1.yaml +++ b/kubernetes/bootstrap/talos/nodes/worker/talos-node-large-1.yaml @@ -4,14 +4,24 @@ # create user namespaces. Talos defaults this to 0 as hardening, which makes the # sandbox fail with "fork/exec ...: no space left on device". Only enable it on # nodes labelled workload.cluster.io/agent-sandbox, where the worker pool is pinned. -machine: - install: - disk: /dev/sda - image: factory.talos.dev/installer/5cc10e6e83c6e3b32167fe812094d4715c9fa3e1a77bcfe61fa4cf71b4b9fed3:${TALOS_VERSION} - nodeLabels: - workload.cluster.io/agent-sandbox: "true" - sysctls: - user.max_user_namespaces: "11255" +apiVersion: v1alpha1 +kind: UnattendedInstallConfig +installer: + image: factory.talos.dev/installer/5cc10e6e83c6e3b32167fe812094d4715c9fa3e1a77bcfe61fa4cf71b4b9fed3:${TALOS_VERSION} +provisioning: + diskSelector: + match: disk.dev_path == "/dev/sda" + wipe: false +--- +apiVersion: v1alpha1 +kind: KubeNodeConfig +labels: + workload.cluster.io/agent-sandbox: "true" +--- +apiVersion: v1alpha1 +kind: SysctlConfig +params: + user.max_user_namespaces: "11255" --- apiVersion: v1alpha1 kind: HostnameConfig diff --git a/kubernetes/bootstrap/talos/nodes/worker/talos-node-large-2.yaml b/kubernetes/bootstrap/talos/nodes/worker/talos-node-large-2.yaml index 4dacdeb138..a885231a9a 100644 --- a/kubernetes/bootstrap/talos/nodes/worker/talos-node-large-2.yaml +++ b/kubernetes/bootstrap/talos/nodes/worker/talos-node-large-2.yaml @@ -1,8 +1,12 @@ # talos-node-large-2 (10.100.0.55). Its role comes from the directory this file is in. -machine: - install: - disk: /dev/sda - image: factory.talos.dev/installer/5cc10e6e83c6e3b32167fe812094d4715c9fa3e1a77bcfe61fa4cf71b4b9fed3:${TALOS_VERSION} +apiVersion: v1alpha1 +kind: UnattendedInstallConfig +installer: + image: factory.talos.dev/installer/5cc10e6e83c6e3b32167fe812094d4715c9fa3e1a77bcfe61fa4cf71b4b9fed3:${TALOS_VERSION} +provisioning: + diskSelector: + match: disk.dev_path == "/dev/sda" + wipe: false --- apiVersion: v1alpha1 kind: HostnameConfig diff --git a/kubernetes/bootstrap/talos/nodes/worker/talos-node-large-3.yaml b/kubernetes/bootstrap/talos/nodes/worker/talos-node-large-3.yaml index 71d260aada..314f289fe1 100644 --- a/kubernetes/bootstrap/talos/nodes/worker/talos-node-large-3.yaml +++ b/kubernetes/bootstrap/talos/nodes/worker/talos-node-large-3.yaml @@ -1,8 +1,12 @@ # talos-node-large-3 (10.100.0.56). Its role comes from the directory this file is in. -machine: - install: - disk: /dev/sda - image: factory.talos.dev/installer/5cc10e6e83c6e3b32167fe812094d4715c9fa3e1a77bcfe61fa4cf71b4b9fed3:${TALOS_VERSION} +apiVersion: v1alpha1 +kind: UnattendedInstallConfig +installer: + image: factory.talos.dev/installer/5cc10e6e83c6e3b32167fe812094d4715c9fa3e1a77bcfe61fa4cf71b4b9fed3:${TALOS_VERSION} +provisioning: + diskSelector: + match: disk.dev_path == "/dev/sda" + wipe: false --- apiVersion: v1alpha1 kind: HostnameConfig diff --git a/kubernetes/bootstrap/talos/nodes/worker/talos-node-small-1.yaml b/kubernetes/bootstrap/talos/nodes/worker/talos-node-small-1.yaml index 797bd5c92b..9a17fc0215 100644 --- a/kubernetes/bootstrap/talos/nodes/worker/talos-node-small-1.yaml +++ b/kubernetes/bootstrap/talos/nodes/worker/talos-node-small-1.yaml @@ -1,8 +1,12 @@ # talos-node-small-1 (10.100.0.60). Its role comes from the directory this file is in. -machine: - install: - disk: /dev/sda - image: factory.talos.dev/installer/5cc10e6e83c6e3b32167fe812094d4715c9fa3e1a77bcfe61fa4cf71b4b9fed3:${TALOS_VERSION} +apiVersion: v1alpha1 +kind: UnattendedInstallConfig +installer: + image: factory.talos.dev/installer/5cc10e6e83c6e3b32167fe812094d4715c9fa3e1a77bcfe61fa4cf71b4b9fed3:${TALOS_VERSION} +provisioning: + diskSelector: + match: disk.dev_path == "/dev/sda" + wipe: false --- apiVersion: v1alpha1 kind: HostnameConfig diff --git a/kubernetes/bootstrap/talos/nodes/worker/talos-node-small-2.yaml b/kubernetes/bootstrap/talos/nodes/worker/talos-node-small-2.yaml index 04cf5fa4a2..020934a7ba 100644 --- a/kubernetes/bootstrap/talos/nodes/worker/talos-node-small-2.yaml +++ b/kubernetes/bootstrap/talos/nodes/worker/talos-node-small-2.yaml @@ -1,8 +1,12 @@ # talos-node-small-2 (10.100.0.61). Its role comes from the directory this file is in. -machine: - install: - disk: /dev/sda - image: factory.talos.dev/installer/5cc10e6e83c6e3b32167fe812094d4715c9fa3e1a77bcfe61fa4cf71b4b9fed3:${TALOS_VERSION} +apiVersion: v1alpha1 +kind: UnattendedInstallConfig +installer: + image: factory.talos.dev/installer/5cc10e6e83c6e3b32167fe812094d4715c9fa3e1a77bcfe61fa4cf71b4b9fed3:${TALOS_VERSION} +provisioning: + diskSelector: + match: disk.dev_path == "/dev/sda" + wipe: false --- apiVersion: v1alpha1 kind: HostnameConfig diff --git a/kubernetes/bootstrap/talos/nodes/worker/talos-node-small-3.yaml b/kubernetes/bootstrap/talos/nodes/worker/talos-node-small-3.yaml index f33086f21e..eaad4cd20d 100644 --- a/kubernetes/bootstrap/talos/nodes/worker/talos-node-small-3.yaml +++ b/kubernetes/bootstrap/talos/nodes/worker/talos-node-small-3.yaml @@ -1,8 +1,12 @@ # talos-node-small-3 (10.100.0.62). Its role comes from the directory this file is in. -machine: - install: - disk: /dev/sda - image: factory.talos.dev/installer/5cc10e6e83c6e3b32167fe812094d4715c9fa3e1a77bcfe61fa4cf71b4b9fed3:${TALOS_VERSION} +apiVersion: v1alpha1 +kind: UnattendedInstallConfig +installer: + image: factory.talos.dev/installer/5cc10e6e83c6e3b32167fe812094d4715c9fa3e1a77bcfe61fa4cf71b4b9fed3:${TALOS_VERSION} +provisioning: + diskSelector: + match: disk.dev_path == "/dev/sda" + wipe: false --- apiVersion: v1alpha1 kind: HostnameConfig diff --git a/kubernetes/bootstrap/talos/nodes/worker/talos-node-small-4.yaml b/kubernetes/bootstrap/talos/nodes/worker/talos-node-small-4.yaml index 8996a4d468..f87b7b29fa 100644 --- a/kubernetes/bootstrap/talos/nodes/worker/talos-node-small-4.yaml +++ b/kubernetes/bootstrap/talos/nodes/worker/talos-node-small-4.yaml @@ -1,8 +1,12 @@ # talos-node-small-4 (10.100.0.63). Its role comes from the directory this file is in. -machine: - install: - disk: /dev/sda - image: factory.talos.dev/installer/5cc10e6e83c6e3b32167fe812094d4715c9fa3e1a77bcfe61fa4cf71b4b9fed3:${TALOS_VERSION} +apiVersion: v1alpha1 +kind: UnattendedInstallConfig +installer: + image: factory.talos.dev/installer/5cc10e6e83c6e3b32167fe812094d4715c9fa3e1a77bcfe61fa4cf71b4b9fed3:${TALOS_VERSION} +provisioning: + diskSelector: + match: disk.dev_path == "/dev/sda" + wipe: false --- apiVersion: v1alpha1 kind: HostnameConfig diff --git a/kubernetes/bootstrap/talos/patches/controller/00-apiserver-certsans.yaml b/kubernetes/bootstrap/talos/patches/controller/00-apiserver-certsans.yaml deleted file mode 100644 index ea86f400ee..0000000000 --- a/kubernetes/bootstrap/talos/patches/controller/00-apiserver-certsans.yaml +++ /dev/null @@ -1,5 +0,0 @@ -cluster: - apiServer: - certSANs: - - 10.100.0.40 # control-plane VIP - - 127.0.0.1 diff --git a/kubernetes/bootstrap/talos/patches/controller/00-apiserver.yaml b/kubernetes/bootstrap/talos/patches/controller/00-apiserver.yaml new file mode 100644 index 0000000000..56852f91fe --- /dev/null +++ b/kubernetes/bootstrap/talos/patches/controller/00-apiserver.yaml @@ -0,0 +1,35 @@ +apiVersion: v1alpha1 +kind: KubeAPIServerConfig +certExtraSANs: + - 10.100.0.40 # control-plane VIP + - 127.0.0.1 +extraArgs: + # https://kubernetes.io/docs/tasks/extend-kubernetes/configure-aggregation-layer/ + enable-aggregator-routing: "true" + feature-gates: MutatingAdmissionPolicy=true + runtime-config: admissionregistration.k8s.io/v1beta1=true + # Increase timeouts and request handling capacity + request-timeout: "300s" + max-requests-inflight: "800" + max-mutating-requests-inflight: "400" + # Reduce watch cache memory usage + default-watch-cache-size: "100" + # OIDC authentication lives in KubeAuthenticationConfig (30-apiserver-oidc.yaml); + # Talos rejects authentication-config / oidc-* / anonymous-auth here. +# Live heap sits at ~2-3 GiB per apiserver (219 CRDs, ~1,100 watches on the +# busiest one), flat over a week: no leak. At 4Gi, GOMEMLIMIT (95% of the +# limit, set by Talos) left no headroom and control-1 was OOM-killed on +# 2026-11-20. Node has 16 GiB; peak use <50%. +resources: + requests: + cpu: 500m + memory: 3Gi + limits: + cpu: 2000m + memory: 6Gi +# No kubelet probes on the apiserver pod, as before the 1.14 contract (this field +# switches the startup, liveness AND readiness probes; the document default is on). +# The probes are unauthenticated requests to /livez and /readyz, and anonymous access +# is off (30-apiserver-oidc.yaml), so they would get a 401 and the kubelet would kill +# the apiserver. Only turn this on together with anonymous access to the health paths. +startupProbes: false diff --git a/kubernetes/bootstrap/talos/patches/controller/10-admission-control.yaml b/kubernetes/bootstrap/talos/patches/controller/10-admission-control.yaml index e276f35885..431d8c0e6e 100644 --- a/kubernetes/bootstrap/talos/patches/controller/10-admission-control.yaml +++ b/kubernetes/bootstrap/talos/patches/controller/10-admission-control.yaml @@ -1,3 +1,12 @@ -cluster: - apiServer: - admissionControl: [] +# PodSecurity admission. The base document already carries the defaults +# (enforce baseline, audit/warn restricted) and the kube-system exemption; this +# keeps the v1beta1 config version the cluster runs and adds our exemption +# (lists in patches append). +apiVersion: v1alpha1 +kind: KubeAdmissionControlConfig +name: PodSecurity +configuration: + apiVersion: pod-security.admission.config.k8s.io/v1beta1 + exemptions: + namespaces: + - openebs-system diff --git a/kubernetes/bootstrap/talos/patches/controller/20-cluster.yaml b/kubernetes/bootstrap/talos/patches/controller/20-cluster.yaml deleted file mode 100644 index 354b52f344..0000000000 --- a/kubernetes/bootstrap/talos/patches/controller/20-cluster.yaml +++ /dev/null @@ -1,56 +0,0 @@ -cluster: - allowSchedulingOnControlPlanes: false - apiServer: - extraArgs: - # https://kubernetes.io/docs/tasks/extend-kubernetes/configure-aggregation-layer/ - enable-aggregator-routing: true - feature-gates: MutatingAdmissionPolicy=true - runtime-config: admissionregistration.k8s.io/v1beta1=true - # Increase timeouts and request handling capacity - request-timeout: "300s" - max-requests-inflight: "800" - max-mutating-requests-inflight: "400" - # Reduce watch cache memory usage - default-watch-cache-size: "100" - # OIDC authentication (structured multi-issuer) lives in its own patch: - # ./patches/controller/apiserver-oidc.yaml - # Live heap sits at ~2-3 GiB per apiserver (219 CRDs, ~1,100 watches on the - # busiest one), flat over a week: no leak. At 4Gi, GOMEMLIMIT (95% of the - # limit, set by Talos) left no headroom and control-1 was OOM-killed on - # 2026-11-20. Node has 16 GiB; peak use <50%. - resources: - requests: - cpu: 500m - memory: 3Gi - limits: - cpu: 2000m - memory: 6Gi - admissionControl: - - name: PodSecurity - configuration: - apiVersion: pod-security.admission.config.k8s.io/v1beta1 - kind: PodSecurityConfiguration - exemptions: - namespaces: - - openebs-system - controllerManager: - extraArgs: - bind-address: 0.0.0.0 - coreDNS: - disabled: true - etcd: - extraArgs: - listen-metrics-urls: http://0.0.0.0:2381 - quota-backend-bytes: "1073741824" # 1 GiB (double peak usage of ~350MB) - # Widen raft timeouts to tolerate ~20-22ms WAL fsync latency on Proxmox VM - # disks (SSD/hardware migration ruled out). Stops leader flapping when a - # heartbeat is delayed by a slow fsync. election-timeout is 10x heartbeat. - heartbeat-interval: "250" # default 100ms - election-timeout: "2500" # default 1000ms - advertisedSubnets: - - 10.100.0.0/24 - proxy: - disabled: true - scheduler: - extraArgs: - bind-address: 0.0.0.0 diff --git a/kubernetes/bootstrap/talos/patches/controller/20-control-plane.yaml b/kubernetes/bootstrap/talos/patches/controller/20-control-plane.yaml new file mode 100644 index 0000000000..9fbc3a92b9 --- /dev/null +++ b/kubernetes/bootstrap/talos/patches/controller/20-control-plane.yaml @@ -0,0 +1,33 @@ +# etcd has no 1.14 document yet: it stays in the v1alpha1 config. +cluster: + etcd: + extraArgs: + listen-metrics-urls: http://0.0.0.0:2381 + quota-backend-bytes: "1073741824" # 1 GiB (double peak usage of ~350MB) + # Widen raft timeouts to tolerate ~20-22ms WAL fsync latency on Proxmox VM + # disks (SSD/hardware migration ruled out). Stops leader flapping when a + # heartbeat is delayed by a slow fsync. election-timeout is 10x heartbeat. + heartbeat-interval: "250" # default 100ms + election-timeout: "2500" # default 1000ms + advertisedSubnets: + - 10.100.0.0/24 +--- +apiVersion: v1alpha1 +kind: KubeControllerManagerConfig +extraArgs: + bind-address: 0.0.0.0 +--- +apiVersion: v1alpha1 +kind: KubeSchedulerConfig +extraArgs: + bind-address: 0.0.0.0 +--- +# CoreDNS is deployed by Flux, not by Talos. +apiVersion: v1alpha1 +kind: KubeCoreDNSConfig +enabled: false +--- +# Cilium replaces kube-proxy. +apiVersion: v1alpha1 +kind: KubeProxyConfig +enabled: false diff --git a/kubernetes/bootstrap/talos/patches/controller/30-apiserver-oidc.yaml b/kubernetes/bootstrap/talos/patches/controller/30-apiserver-oidc.yaml index c86afc84ba..1c4b973c9e 100644 --- a/kubernetes/bootstrap/talos/patches/controller/30-apiserver-oidc.yaml +++ b/kubernetes/bootstrap/talos/patches/controller/30-apiserver-oidc.yaml @@ -2,20 +2,16 @@ # # Authentik gives every application its OWN issuer URL (…/application/o//), so the # legacy single-issuer --oidc-* flags can only ever trust one app. This uses the k8s -# Structured Authentication Config (apiserver.config.k8s.io/v1, GA in 1.30) to trust -# MULTIPLE issuers — one JWT authenticator per app, each scoped to its own audience. -# Both map preferred_username -> user and groups -> groups with the `oidc:` prefix, so -# the oidc-rbac ClusterRoleBindings apply identically regardless of which app issued the -# token. x509 cert auth (talosconfig / admin kubeconfig) remains the break-glass. +# Structured Authentication Config to trust MULTIPLE issuers — one JWT authenticator per +# app, each scoped to its own audience. Both map preferred_username -> user and +# groups -> groups with the `oidc:` prefix, so the oidc-rbac ClusterRoleBindings apply +# identically regardless of which app issued the token. x509 cert auth (talosconfig / +# admin kubeconfig) remains the break-glass. # -# Delivery: machine.files writes the config to the control-plane nodes; -# apiServer.extraVolumes mounts it read-only into the static pod; --authentication-config -# points the apiserver at it. --authentication-config is mutually exclusive with --oidc-*. -# -# IMPORTANT: machine.files MUST write to a writable host path (/var/...). Writing under -# /etc/kubernetes makes Talos overlay that dir read-only, which breaks the kubelet's own -# PKI writes and takes the node NotReady. So the file lives at /var/lib/... on the host and -# is mounted into the apiserver pod at /etc/kubernetes/authentication (container-only). +# Delivery: Talos renders `configuration` below into the apiserver's +# --authentication-config file itself (as apiserver.config.k8s.io/v1beta1) and restarts +# the apiserver when it changes. Before the 1.14 config contract this was a machine.files +# entry under /var/lib/kubernetes/authentication plus an extraVolume and an extraArg. # # Adding a new OIDC-to-apiserver app (e.g. another dashboard) = add a jwt authenticator # block below with that app's issuer + audience. @@ -31,57 +27,50 @@ # - uid <- sub: audit-stable identity. preferred_username is user-editable in # Authentik; sub (sub_mode: hashed_user_id) is immutable. Never bind kind: User # RBAC to oidc: usernames. -# NOTE: never add an `anonymous:` section to the config — Talos sets -# --anonymous-auth=false and the combination is rejected by the apiserver -# (siderolabs/talos#11324). -cluster: - apiServer: - extraArgs: - authentication-config: /etc/kubernetes/authentication/config.yaml - extraVolumes: - - hostPath: /var/lib/kubernetes/authentication # writable host path (NOT /etc/kubernetes) - mountPath: /etc/kubernetes/authentication # container-only mount point - readonly: true -machine: - files: - - op: create - path: /var/lib/kubernetes/authentication/config.yaml # writable host path - permissions: 0o644 - content: | - apiVersion: apiserver.config.k8s.io/v1 - kind: AuthenticationConfiguration - jwt: - # kubectl CLI (kubelogin) - - issuer: - url: "https://sso.chelonianlabs.com/application/o/kubectl/" - audiences: - - "kubectl" - claimMappings: - username: - claim: "preferred_username" - prefix: "oidc:" - groups: - claim: "groups" - prefix: "oidc:" - uid: - claim: "sub" - userValidationRules: - - expression: 'user.groups.exists(g, g in ["oidc:kubernetes-admins", "oidc:kubernetes-viewers"])' - message: "not a member of a kubernetes access group" - # Headlamp web dashboard - - issuer: - url: "https://sso.chelonianlabs.com/application/o/headlamp/" - audiences: - - "headlamp" - claimMappings: - username: - claim: "preferred_username" - prefix: "oidc:" - groups: - claim: "groups" - prefix: "oidc:" - uid: - claim: "sub" - userValidationRules: - - expression: 'user.groups.exists(g, g in ["oidc:kubernetes-admins", "oidc:kubernetes-viewers"])' - message: "not a member of a kubernetes access group" +# +# anonymous: Talos now always uses this file and no longer passes --anonymous-auth=false, +# and its default document allows anonymous /livez, /readyz and /healthz. Anonymous access +# stays fully off here, as it was. That is tied to `startupProbes: false` in +# 00-apiserver.yaml: the apiserver pod's probes are anonymous requests. +# (`configuration` is replaced as a whole by a patch, not merged, so everything the +# apiserver should use must be in this file.) +apiVersion: v1alpha1 +kind: KubeAuthenticationConfig +configuration: + anonymous: + enabled: false + jwt: + # kubectl CLI (kubelogin) + - issuer: + url: "https://sso.chelonianlabs.com/application/o/kubectl/" + audiences: + - "kubectl" + claimMappings: + username: + claim: "preferred_username" + prefix: "oidc:" + groups: + claim: "groups" + prefix: "oidc:" + uid: + claim: "sub" + userValidationRules: + - expression: 'user.groups.exists(g, g in ["oidc:kubernetes-admins", "oidc:kubernetes-viewers"])' + message: "not a member of a kubernetes access group" + # Headlamp web dashboard + - issuer: + url: "https://sso.chelonianlabs.com/application/o/headlamp/" + audiences: + - "headlamp" + claimMappings: + username: + claim: "preferred_username" + prefix: "oidc:" + groups: + claim: "groups" + prefix: "oidc:" + uid: + claim: "sub" + userValidationRules: + - expression: 'user.groups.exists(g, g in ["oidc:kubernetes-admins", "oidc:kubernetes-viewers"])' + message: "not a member of a kubernetes access group" diff --git a/kubernetes/bootstrap/talos/patches/controller/40-machine-features.yaml b/kubernetes/bootstrap/talos/patches/controller/40-machine-features.yaml deleted file mode 100644 index 72bee09615..0000000000 --- a/kubernetes/bootstrap/talos/patches/controller/40-machine-features.yaml +++ /dev/null @@ -1,14 +0,0 @@ -machine: - features: - kubernetesTalosAPIAccess: - enabled: true - allowedRoles: ["os:admin"] - allowedKubernetesNamespaces: ["actions-runner-system", "kube-system"] - diskQuotaSupport: true - kubePrism: - enabled: true - port: 7445 - hostDNS: - enabled: true - resolveMemberNames: true - forwardKubeDNSToHost: false diff --git a/kubernetes/bootstrap/talos/patches/controller/40-talos-api-access.yaml b/kubernetes/bootstrap/talos/patches/controller/40-talos-api-access.yaml new file mode 100644 index 0000000000..9caeaa9a2e --- /dev/null +++ b/kubernetes/bootstrap/talos/patches/controller/40-talos-api-access.yaml @@ -0,0 +1,19 @@ +# Talos API access from Kubernetes pods (ServiceAccount-based): the GitHub runners, +# kube-system and tuppr (system-upgrade). +apiVersion: v1alpha1 +kind: KubeTalosAPIAccessConfig +allowedRoles: + - os:admin +allowedKubernetesNamespaces: + - actions-runner-system + - kube-system + - system-upgrade +--- +# Control planes resolve cluster member names through host DNS and do not forward +# kube-dns to it. Workers keep the generated defaults (forwardKubeDNSToHost: true). +apiVersion: v1alpha1 +kind: ResolverConfig +hostDNS: + enabled: true + forwardKubeDNSToHost: false + resolveMemberNames: true diff --git a/kubernetes/bootstrap/talos/patches/controller/50-machine-talos-api.yaml b/kubernetes/bootstrap/talos/patches/controller/50-machine-talos-api.yaml deleted file mode 100644 index 0c150e4574..0000000000 --- a/kubernetes/bootstrap/talos/patches/controller/50-machine-talos-api.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -# Enable Talos API access from Kubernetes -# This allows pods to interact with the Talos API using ServiceAccounts -machine: - features: - kubernetesTalosAPIAccess: - enabled: true - allowedRoles: - - os:admin - allowedKubernetesNamespaces: - - system-upgrade diff --git a/kubernetes/bootstrap/talos/patches/controller/60-service-account.yaml b/kubernetes/bootstrap/talos/patches/controller/60-service-account.yaml deleted file mode 100644 index 6f2f7f290b..0000000000 --- a/kubernetes/bootstrap/talos/patches/controller/60-service-account.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Service-account signing key. -# -# The key lives in the Talos 1.14 KubeServiceAccountConfig document instead of -# v1alpha1 .cluster.serviceAccount (Talos rejects both at once), because only -# the new document has accepted.publicKeys: extra keys the apiserver trusts, -# which is what makes a zero-downtime key rotation possible. -# -# issuer.privateKey is filled in by render.sh from talsecret.sops.yaml -# (certs.k8sserviceaccount.key); only public keys belong in this file. -# -# Rotation 2026-11-21 (old key exposed in a session transcript): -# stage A sign with OLD, accept NEW (done) -# stage B sign with NEW, accept OLD (bundle key swapped) (done) -# stage C sign with NEW, accept nothing <- current -# -# To rotate again: add the next public key under accepted.publicKeys (stage A), -# swap the bundle key and list the previous public key (stage B), then remove it (stage C). -cluster: - serviceAccount: - $patch: delete ---- -apiVersion: v1alpha1 -kind: KubeServiceAccountConfig -issuer: - privateKey: SET_BY_RENDER_SH - issuerURL: https://10.100.0.40:6443 diff --git a/kubernetes/bootstrap/talos/patches/global/00-cluster-network.yaml b/kubernetes/bootstrap/talos/patches/global/00-cluster-network.yaml deleted file mode 100644 index 098a6b367e..0000000000 --- a/kubernetes/bootstrap/talos/patches/global/00-cluster-network.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# Cluster-wide settings talhelper used to inject from talconfig.yaml. -machine: - certSANs: - - 10.100.0.40 # control-plane VIP - - 127.0.0.1 -cluster: - network: - cni: - name: none # Cilium is installed by the bootstrap helmfile - podSubnets: - - 10.69.0.0/16 - serviceSubnets: - - 10.96.0.0/16 diff --git a/kubernetes/bootstrap/talos/patches/global/00-network.yaml b/kubernetes/bootstrap/talos/patches/global/00-network.yaml new file mode 100644 index 0000000000..bae8f89d44 --- /dev/null +++ b/kubernetes/bootstrap/talos/patches/global/00-network.yaml @@ -0,0 +1,18 @@ +# Talos API certificate SANs (still a v1alpha1 field). +machine: + certSANs: + - 10.100.0.40 # control-plane VIP + - 127.0.0.1 +--- +apiVersion: v1alpha1 +kind: KubeNetworkConfig +podSubnets: + - 10.69.0.0/16 +serviceSubnets: + - 10.96.0.0/16 +--- +# No Talos-managed CNI: Cilium is installed by the bootstrap helmfile. +# (This was `cluster.network.cni.name: none`.) +apiVersion: v1alpha1 +kind: KubeFlannelCNIConfig +$patch: delete diff --git a/kubernetes/bootstrap/talos/patches/global/10-cluster-discovery.yaml b/kubernetes/bootstrap/talos/patches/global/10-cluster-discovery.yaml deleted file mode 100644 index 00d5d83525..0000000000 --- a/kubernetes/bootstrap/talos/patches/global/10-cluster-discovery.yaml +++ /dev/null @@ -1,7 +0,0 @@ -cluster: - discovery: - registries: - kubernetes: - disabled: true - service: - disabled: false diff --git a/kubernetes/bootstrap/talos/patches/global/20-machine-files.yaml b/kubernetes/bootstrap/talos/patches/global/20-machine-files.yaml index 35ad3f3ddb..39275a2790 100644 --- a/kubernetes/bootstrap/talos/patches/global/20-machine-files.yaml +++ b/kubernetes/bootstrap/talos/patches/global/20-machine-files.yaml @@ -1,3 +1,8 @@ +# Still the deprecated v1alpha1 `machine.files` on purpose (Talos warns about it). +# The 1.14 replacement is a CRICustomizationConfig document, but the name +# "customization" is reserved for this legacy file, and any other name changes the +# header of the merged containerd config, which makes Talos restart containerd on +# every node. Convert it together with a planned reboot (e.g. a Talos upgrade). machine: files: - op: create diff --git a/kubernetes/bootstrap/talos/patches/global/30-kubelet.yaml b/kubernetes/bootstrap/talos/patches/global/30-kubelet.yaml new file mode 100644 index 0000000000..37edac1bfc --- /dev/null +++ b/kubernetes/bootstrap/talos/patches/global/30-kubelet.yaml @@ -0,0 +1,14 @@ +apiVersion: v1alpha1 +kind: KubeletConfig +extraArgs: + # Conservative but safe kubelet settings + max-pods: "250" + # Aggressive image GC to prevent disk pressure on image-heavy nodes + image-gc-high-threshold: "75" + image-gc-low-threshold: "65" +--- +apiVersion: v1alpha1 +kind: KubeNodeConfig +nodeIP: + validSubnets: + - 10.100.0.0/24 diff --git a/kubernetes/bootstrap/talos/patches/global/30-machine-kubelet.yaml b/kubernetes/bootstrap/talos/patches/global/30-machine-kubelet.yaml deleted file mode 100644 index 466892a9ce..0000000000 --- a/kubernetes/bootstrap/talos/patches/global/30-machine-kubelet.yaml +++ /dev/null @@ -1,11 +0,0 @@ -machine: - kubelet: - nodeIP: - validSubnets: - - 10.100.0.0/24 - extraArgs: - # Conservative but safe kubelet settings - max-pods: "250" - # Aggressive image GC to prevent disk pressure on image-heavy nodes - image-gc-high-threshold: "75" - image-gc-low-threshold: "65" diff --git a/kubernetes/bootstrap/talos/patches/global/40-machine-network.yaml b/kubernetes/bootstrap/talos/patches/global/40-machine-network.yaml deleted file mode 100644 index bf9b79cf02..0000000000 --- a/kubernetes/bootstrap/talos/patches/global/40-machine-network.yaml +++ /dev/null @@ -1,6 +0,0 @@ -machine: - network: - disableSearchDomain: true - nameservers: - - 10.100.0.1 # OPNsense (house, primary) - - 10.100.0.2 # garage resolver (fallback) diff --git a/kubernetes/bootstrap/talos/patches/global/40-resolver.yaml b/kubernetes/bootstrap/talos/patches/global/40-resolver.yaml new file mode 100644 index 0000000000..bd05b35ddd --- /dev/null +++ b/kubernetes/bootstrap/talos/patches/global/40-resolver.yaml @@ -0,0 +1,7 @@ +apiVersion: v1alpha1 +kind: ResolverConfig +nameservers: + - address: 10.100.0.1 # OPNsense (house, primary) + - address: 10.100.0.2 # garage resolver (fallback) +searchDomains: + disableDefault: true diff --git a/kubernetes/bootstrap/talos/patches/global/50-machine-sysctls.yaml b/kubernetes/bootstrap/talos/patches/global/50-machine-sysctls.yaml deleted file mode 100644 index 75c71e700f..0000000000 --- a/kubernetes/bootstrap/talos/patches/global/50-machine-sysctls.yaml +++ /dev/null @@ -1,25 +0,0 @@ -machine: - sysctls: - # File system watches - fs.inotify.max_user_watches: "1048576" # Watchdog - fs.inotify.max_user_instances: "8192" # Watchdog - # File handle limits - fs.file-max: "2097152" - fs.aio-max-nr: "1048576" - # Network buffers for Cloudflared/QUIC - net.core.rmem_max: "7500000" # Cloudflared | QUIC - net.core.wmem_max: "7500000" # Cloudflared | QUIC - # Network performance - net.core.netdev_max_backlog: "5000" - # BBR + fq: loss-tolerant TCP for remote access over Starlink (WireGuard wg-home) - net.ipv4.tcp_congestion_control: "bbr" - net.core.default_qdisc: "fq" - # Connection tracking - net.netfilter.nf_conntrack_max: "131072" - # TCP keepalive - net.ipv4.tcp_keepalive_time: "600" - net.ipv4.tcp_keepalive_intvl: "60" - net.ipv4.tcp_keepalive_probes: "6" - # VM tuning - vm.swappiness: "10" - vm.max_map_count: "262144" # OpenSearch/Elasticsearch requirement diff --git a/kubernetes/bootstrap/talos/patches/global/50-sysctls.yaml b/kubernetes/bootstrap/talos/patches/global/50-sysctls.yaml new file mode 100644 index 0000000000..c816fbb99e --- /dev/null +++ b/kubernetes/bootstrap/talos/patches/global/50-sysctls.yaml @@ -0,0 +1,26 @@ +apiVersion: v1alpha1 +kind: SysctlConfig +params: + # File system watches + fs.inotify.max_user_watches: "1048576" # Watchdog + fs.inotify.max_user_instances: "8192" # Watchdog + # File handle limits + fs.file-max: "2097152" + fs.aio-max-nr: "1048576" + # Network buffers for Cloudflared/QUIC + net.core.rmem_max: "7500000" # Cloudflared | QUIC + net.core.wmem_max: "7500000" # Cloudflared | QUIC + # Network performance + net.core.netdev_max_backlog: "5000" + # BBR + fq: loss-tolerant TCP for remote access over Starlink (WireGuard wg-home) + net.ipv4.tcp_congestion_control: "bbr" + net.core.default_qdisc: "fq" + # Connection tracking + net.netfilter.nf_conntrack_max: "131072" + # TCP keepalive + net.ipv4.tcp_keepalive_time: "600" + net.ipv4.tcp_keepalive_intvl: "60" + net.ipv4.tcp_keepalive_probes: "6" + # VM tuning + vm.swappiness: "10" + vm.max_map_count: "262144" # OpenSearch/Elasticsearch requirement diff --git a/kubernetes/bootstrap/talos/patches/global/60-machine-time.yaml b/kubernetes/bootstrap/talos/patches/global/60-machine-time.yaml deleted file mode 100644 index b7d65948ac..0000000000 --- a/kubernetes/bootstrap/talos/patches/global/60-machine-time.yaml +++ /dev/null @@ -1,6 +0,0 @@ -machine: - time: - disabled: false - servers: - - 162.159.200.1 - - 162.159.200.123 diff --git a/kubernetes/bootstrap/talos/patches/global/60-time.yaml b/kubernetes/bootstrap/talos/patches/global/60-time.yaml new file mode 100644 index 0000000000..77696e28be --- /dev/null +++ b/kubernetes/bootstrap/talos/patches/global/60-time.yaml @@ -0,0 +1,9 @@ +apiVersion: v1alpha1 +kind: TimeSyncConfig +enabled: true +ntp: + servers: + - 162.159.200.1 + - 162.159.200.123 + # Plain NTP, as before. NTS needs hostnames, and these are IPs. + useNTS: false diff --git a/kubernetes/bootstrap/talos/patches/global/70-pinned-defaults.yaml b/kubernetes/bootstrap/talos/patches/global/70-pinned-defaults.yaml new file mode 100644 index 0000000000..2088ffc174 --- /dev/null +++ b/kubernetes/bootstrap/talos/patches/global/70-pinned-defaults.yaml @@ -0,0 +1,23 @@ +# A v1.14 `talosctl gen config` base turns on things the cluster has never run with. +# The config-contract move is meant to change nothing, so each of those documents is +# removed here, which is exactly the state the nodes were in under the v1.13 contract. +# To adopt one later, delete its block below (and test on one worker first). +# +# Workload isolation: runs the container plane in the sandbox namespace. +# Absent document = off. +apiVersion: v1alpha1 +kind: SecurityProfileConfig +$patch: delete +--- +# Secure mount (nosuid,nodev) for EPHEMERAL (/var). Absent document = not secure, +# as today. /var holds containerd, kubelet and hostPath data, so check workloads +# before turning this on. +apiVersion: v1alpha1 +kind: VolumeConfig +name: EPHEMERAL +$patch: delete +--- +# Weekly fstrim of the system volumes. Absent document = no scheduled trim. +apiVersion: v1alpha1 +kind: FilesystemTrimConfig +$patch: delete diff --git a/kubernetes/bootstrap/talos/render.sh b/kubernetes/bootstrap/talos/render.sh index 822458d226..9637d8b2ec 100755 --- a/kubernetes/bootstrap/talos/render.sh +++ b/kubernetes/bootstrap/talos/render.sh @@ -8,8 +8,8 @@ # 2. patches/controller/*.yaml (control-plane nodes only) # 3. patches/global/*.yaml (every node) # 4. nodes//.yaml -# Role patches go before global ones to keep list order identical to what talhelper -# produced (e.g. machine.files, kubernetesTalosAPIAccess namespaces). +# The base already holds every secret (CAs, tokens, the service-account signing key) in +# its own document, so patches never carry secret material. # # ${TALOS_VERSION} and ${KUBERNETES_VERSION} in patches are substituted from versions.yaml. # Output contains cluster secrets: never write it inside the repo. @@ -61,15 +61,18 @@ talosctl gen config "$CLUSTER_NAME" "$CLUSTER_ENDPOINT" \ --with-docs=false --with-examples=false \ --output "$work/base.yaml" >/dev/null 2>&1 +# Up to the v1.13 contract Talos wrote the secrets-at-rest EncryptionConfig itself: +# secretbox first, then the identity (plaintext) provider as a read fallback. The v1.14 +# base carries the config as a KubeEtcdEncryptionConfig document with secretbox only. +# Keep the fallback so the apiserver reads exactly what it could read before; a patch +# can't do this (lists append, which would add a second "secrets" entry). +# No-op on workers: the document is control-plane only. +yq -i '(select(.kind == "KubeEtcdEncryptionConfig") | .config.resources[0].providers) += [{"identity": {}}]' "$work/base.yaml" + patch_args=() i=0 for p in "${role_patches[@]}" "$dir"/patches/global/*.yaml "$node_file"; do envsubst '${TALOS_VERSION} ${KUBERNETES_VERSION}' < "$p" > "$work/patch-$i.yaml" - # The service-account signing key comes from the secrets bundle, never the repo. - if yq -e 'select(.kind == "KubeServiceAccountConfig")' "$work/patch-$i.yaml" >/dev/null 2>&1; then - SA_KEY="$(yq -e '.certs.k8sserviceaccount.key' "$work/secrets.yaml" | base64 -d)" \ - yq -i '(select(.kind == "KubeServiceAccountConfig") | .issuer.privateKey) = strenv(SA_KEY)' "$work/patch-$i.yaml" - fi patch_args+=(--patch "@$work/patch-$i.yaml") i=$((i + 1)) done diff --git a/kubernetes/bootstrap/talos/versions.yaml b/kubernetes/bootstrap/talos/versions.yaml index 2350fc3f1b..9cad3789aa 100644 --- a/kubernetes/bootstrap/talos/versions.yaml +++ b/kubernetes/bootstrap/talos/versions.yaml @@ -5,6 +5,7 @@ talosVersion: v1.14.5 # renovate: datasource=docker depName=ghcr.io/siderolabs/kubelet kubernetesVersion: v1.37.2 -# Config contract passed to `talosctl gen config --talos-version`. Pinned to -# v1.13 until the patches move to the v1.14 multi-document format. -configContract: v1.13 +# Config contract passed to `talosctl gen config --talos-version`: it decides which +# documents and defaults the generated base has. Raise it deliberately, never with a +# Talos bump; see README.md ("Config contract"). +configContract: v1.14