From dbb5568777e1b450c719e2ddc93302f425e3965e Mon Sep 17 00:00:00 2001 From: Lee Chapman Date: Sun, 4 Oct 2026 18:35:27 -0700 Subject: [PATCH] fix(envoy-gateway): make module safe for upgrades, pin v1.9.2 The deploy script deleted every Gateway API CRD (and so every Gateway and Route) when the gateways CRD lacked the string "gateway.envoyproxy.io", which is true for all current clusters. The null_resource had no triggers, so version bumps never re-ran it, and a grep in the pipeline masked kubectl apply failures. - never delete CRDs; apply CRDs first (SSA), wait Established, then the controller (EG 1.9 needs Gateway API 1.6 CRDs before the controller) - delete the finished certgen Job before re-apply (immutable template; certgen keeps existing cert secrets) - pipefail; wait for controller rollout; warn on bundle mismatch - triggers on versions + script hash; gateway_api_version now checked - defaults/example: EG v1.9.2, Gateway API v1.6.1 - GATEWAY_API_SETUP: drop standard-install v1.0.0 step (would downgrade the experimental CRDs) Co-authored-by: Cursor --- docs/GATEWAY_API_SETUP.md | 33 ++--- .../envoy_gateway/deploy-envoy-gateway.sh | 127 ++++++------------ terraform/modules/envoy_gateway/main.tf | 20 ++- terraform/terraform.tfvars.example | 14 +- terraform/variables.tf | 22 +-- 5 files changed, 89 insertions(+), 127 deletions(-) diff --git a/docs/GATEWAY_API_SETUP.md b/docs/GATEWAY_API_SETUP.md index 5f9a682..bc59c96 100644 --- a/docs/GATEWAY_API_SETUP.md +++ b/docs/GATEWAY_API_SETUP.md @@ -22,17 +22,16 @@ This guide covers setting up Gateway API on RKE2 clusters managed by Rancher. Ga Envoy Gateway can be installed using the official installation manifest: ```bash -# Install Gateway API CRDs first (if not already installed) -kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/standard-install.yaml - -# Install Envoy Gateway using official manifest (version 1.6.1) +# install.yaml bundles the Gateway API CRDs (experimental channel); don't install +# standard-install.yaml separately - it would downgrade/strip the experimental CRDs. +# Install Envoy Gateway using official manifest (version 1.9.2, Gateway API v1.6.1) # On manager cluster export KUBECONFIG=~/.kube/rancher-manager.yaml -kubectl apply -f https://github.com/envoyproxy/gateway/releases/download/v1.6.1/install.yaml +kubectl apply --server-side -f https://github.com/envoyproxy/gateway/releases/download/v1.9.2/install.yaml # On nprd-apps cluster export KUBECONFIG=~/.kube/nprd-apps.yaml -kubectl apply -f https://github.com/envoyproxy/gateway/releases/download/v1.6.1/install.yaml +kubectl apply --server-side -f https://github.com/envoyproxy/gateway/releases/download/v1.9.2/install.yaml # Wait for deployment to be ready kubectl wait --for=condition=available deployment/envoy-gateway -n envoy-gateway-system --timeout=5m @@ -44,7 +43,7 @@ kubectl get gatewayclass **Note:** The Helm repository method is not currently available. Use the manifest installation method shown above. -**Note:** Envoy Gateway v1.6.1 uses Gateway API v1.4.1 internally, but Gateway API v1.0.0 CRDs are fully compatible. +**Note:** Envoy Gateway v1.9.x ships and requires Gateway API v1.6.x CRDs (TCPRoute/UDPRoute `v1`). Terraform (`modules/envoy_gateway`) applies the CRDs first, then the controller; upgrade one minor version at a time and never delete the Gateway API CRDs (that deletes every Gateway and Route). **Basic Gateway Configuration:** @@ -144,24 +143,14 @@ helm install traefik traefik/traefik \ ### Prerequisites ```bash -# Ensure Gateway API CRDs are installed -kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/standard-install.yaml - -# Note: Envoy Gateway v1.6.1 supports Gateway API v1.0.0 - -# Verify CRDs +# Gateway API CRDs come with the Envoy Gateway install.yaml (Step 2). +# Verify CRDs (after install) kubectl get crd | grep gateway ``` -### Step 1: Install Gateway API CRDs (if not already installed) +### Step 1: Gateway API CRDs -```bash -# Install Gateway API CRDs -kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/standard-install.yaml - -# Verify CRDs are installed -kubectl get crd | grep gateway -``` +Bundled in the Envoy Gateway `install.yaml` (experimental channel, `v1.6.1` for Envoy Gateway `v1.9.x`). Don't apply `standard-install.yaml` on top. ### Step 2: Install Envoy Gateway @@ -170,7 +159,7 @@ kubectl get crd | grep gateway export KUBECONFIG=~/.kube/rancher-manager.yaml # Install Envoy Gateway using official manifest -kubectl apply -f https://github.com/envoyproxy/gateway/releases/download/v1.6.1/install.yaml +kubectl apply --server-side -f https://github.com/envoyproxy/gateway/releases/download/v1.9.2/install.yaml # Wait for deployment to be ready kubectl wait --for=condition=available deployment/envoy-gateway -n envoy-gateway-system --timeout=5m diff --git a/terraform/modules/envoy_gateway/deploy-envoy-gateway.sh b/terraform/modules/envoy_gateway/deploy-envoy-gateway.sh index 2743e95..5b4fc11 100755 --- a/terraform/modules/envoy_gateway/deploy-envoy-gateway.sh +++ b/terraform/modules/envoy_gateway/deploy-envoy-gateway.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -eo pipefail # Script to deploy Envoy Gateway and Gateway API CRDs to a Kubernetes cluster # Usage: deploy-envoy-gateway.sh @@ -19,7 +19,7 @@ echo "Kubeconfig: $KUBECONFIG" echo "Cluster: $CLUSTER_NAME" echo "Envoy Gateway Version: $ENVOY_GATEWAY_VERSION" echo "Namespace: $NAMESPACE" -echo "Note: Envoy Gateway install.yaml includes Gateway API CRDs automatically" +echo "Expected Gateway API bundle: $GATEWAY_API_VERSION (shipped in Envoy Gateway install.yaml)" echo "" # Verify cluster is accessible @@ -72,99 +72,60 @@ sleep 5 echo "✓ Cluster is stable, proceeding with installation" echo "" -# Step 1: Check for and handle existing Gateway API CRDs -# Envoy Gateway install.yaml includes Gateway API CRDs. If CRDs from a previous install exist, -# we need to handle them to avoid annotation size conflicts. -echo "[1/2] Checking for existing Gateway API CRDs..." -GATEWAY_CRDS_EXIST=false +# Step 1: Download the release manifest and split CRDs from the rest. +# install.yaml bundles the Gateway API (experimental channel) and Envoy Gateway CRDs. +# Existing CRDs are only ever updated in place: deleting a CRD deletes every +# Gateway/Route/Policy object of that kind cluster-wide. +echo "[1/3] Downloading Envoy Gateway $ENVOY_GATEWAY_VERSION manifest..." +MANIFEST_URL="https://github.com/envoyproxy/gateway/releases/download/${ENVOY_GATEWAY_VERSION}/install.yaml" +WORKDIR=$(mktemp -d) +trap 'rm -rf "$WORKDIR"' EXIT +curl -fsSL -o "$WORKDIR/install.yaml" "$MANIFEST_URL" +awk -v crds="$WORKDIR/crds.yaml" -v rest="$WORKDIR/rest.yaml" ' + function flush() { if (doc != "") { print "---\n" doc > (is_crd ? crds : rest) } doc = ""; is_crd = 0 } + /^---/ { flush(); next } + /^kind: CustomResourceDefinition$/ { is_crd = 1 } + { doc = doc $0 "\n" } + END { flush() } +' "$WORKDIR/install.yaml" +echo " ✓ $(grep -c '^kind: CustomResourceDefinition$' "$WORKDIR/crds.yaml") CRDs, $(grep -c '^kind:' "$WORKDIR/rest.yaml") other resources" +echo "" + +# Step 2: CRDs first. Envoy Gateway >= v1.9 requires the matching Gateway API CRDs +# (TCPRoute/UDPRoute v1) before the controller is upgraded, otherwise those routes are skipped. +echo "[2/3] Applying CRDs (server-side)..." if kubectl get crd gateways.gateway.networking.k8s.io &>/dev/null; then - GATEWAY_CRDS_EXIST=true - echo " ⚠ Gateway API CRDs already exist" - # Check if these are from Envoy Gateway (will have envoy-specific labels) or from a separate install - if kubectl get crd gateways.gateway.networking.k8s.io -o yaml | grep -q "gateway.envoyproxy.io" 2>/dev/null; then - echo " CRDs appear to be from Envoy Gateway - will update via server-side apply" - else - echo " CRDs appear to be from a separate Gateway API installation" - echo " Will delete and recreate to ensure compatibility with Envoy Gateway" - echo " Deleting existing Gateway API CRDs..." - kubectl delete crd \ - gateways.gateway.networking.k8s.io \ - httproutes.gateway.networking.k8s.io \ - gatewayclasses.gateway.networking.k8s.io \ - grpcroutes.gateway.networking.k8s.io \ - tcproutes.gateway.networking.k8s.io \ - tlsroutes.gateway.networking.k8s.io \ - udproutes.gateway.networking.k8s.io \ - referencegrants.gateway.networking.k8s.io \ - backendtlspolicies.gateway.networking.k8s.io \ - --ignore-not-found=true --wait=true --timeout=60s || { - echo " ⚠ Some CRDs may still exist, continuing with installation..." - } - echo " ✓ Existing Gateway API CRDs removed" - GATEWAY_CRDS_EXIST=false - fi + CURRENT_BUNDLE=$(kubectl get crd gateways.gateway.networking.k8s.io \ + -o jsonpath='{.metadata.annotations.gateway\.networking\.k8s\.io/bundle-version}') + echo " Existing Gateway API CRDs: bundle ${CURRENT_BUNDLE:-unknown}" +fi +kubectl apply --server-side --force-conflicts --field-manager=envoy-gateway-installer -f "$WORKDIR/crds.yaml" +kubectl wait --for=condition=Established --timeout=120s -f "$WORKDIR/crds.yaml" +NEW_BUNDLE=$(kubectl get crd gateways.gateway.networking.k8s.io \ + -o jsonpath='{.metadata.annotations.gateway\.networking\.k8s\.io/bundle-version}') +echo " ✓ CRDs established (Gateway API bundle $NEW_BUNDLE)" +if [ -n "$GATEWAY_API_VERSION" ] && [ "$NEW_BUNDLE" != "$GATEWAY_API_VERSION" ]; then + echo " ⚠ gateway_api_version is $GATEWAY_API_VERSION but Envoy Gateway $ENVOY_GATEWAY_VERSION ships $NEW_BUNDLE" fi echo "" -# Step 2: Install Envoy Gateway using official installation manifest -echo "[2/2] Installing Envoy Gateway (version $ENVOY_GATEWAY_VERSION)..." -MANIFEST_URL="https://github.com/envoyproxy/gateway/releases/download/${ENVOY_GATEWAY_VERSION}/install.yaml" +# Step 3: Envoy Gateway controller, RBAC, webhook and certgen job +echo "[3/3] Installing Envoy Gateway (version $ENVOY_GATEWAY_VERSION)..." # Check if Envoy Gateway is already installed if kubectl get namespace "$NAMESPACE" &>/dev/null && kubectl get deployment envoy-gateway -n "$NAMESPACE" &>/dev/null; then echo " Envoy Gateway already installed, applying updated manifest with server-side apply..." - # Use server-side apply for updates to handle CRD conflicts properly - kubectl apply --server-side --force-conflicts --field-manager=envoy-gateway-installer -f "$MANIFEST_URL" 2>&1 | grep -v "Too long: may not be more than" || { - echo " ⚠ Server-side apply had some conflicts, checking if resources were applied..." - # Check if deployment was updated despite conflicts - kubectl get deployment envoy-gateway -n "$NAMESPACE" || { - echo "ERROR: Deployment not found after apply attempt." - exit 1 - } - } + # The certgen Job template is immutable; a finished Job from a previous version must go first. + # certgen does not overwrite existing control-plane cert secrets. + kubectl delete job eg-gateway-helm-certgen -n "$NAMESPACE" --ignore-not-found=true --wait=true + kubectl apply --server-side --force-conflicts --field-manager=envoy-gateway-installer -f "$WORKDIR/rest.yaml" + kubectl rollout status deployment/envoy-gateway -n "$NAMESPACE" --timeout=5m echo " ✓ Envoy Gateway resources updated" else echo " Installing Envoy Gateway from official manifest (first time installation)..." - # For new installations, use server-side apply which handles CRDs better - kubectl apply --server-side --field-manager=envoy-gateway-installer -f "$MANIFEST_URL" || { - echo " ⚠ Server-side apply failed, trying regular apply..." - # Regular apply, but filter out CRD annotation size errors (they're warnings, not fatal) - kubectl apply -f "$MANIFEST_URL" 2>&1 | grep -v "Too long: may not be more than" || { - echo "ERROR: Failed to install Envoy Gateway from manifest." - echo "" - echo "If you see CRD annotation errors above, the existing Gateway API CRDs conflict with Envoy Gateway's CRDs." - echo "To resolve:" - echo " 1. Delete existing Gateway API CRDs:" - echo " kubectl delete crd -l gateway.networking.k8s.io/bundle-version" - echo " 2. Then re-run terraform apply" - exit 1 - } - } + kubectl apply --server-side --force-conflicts --field-manager=envoy-gateway-installer -f "$WORKDIR/rest.yaml" echo " ✓ Envoy Gateway manifest applied" - - # Wait for Gateway API CRDs to be established - echo " Waiting for Gateway API CRDs to be established..." - CRD_ESTABLISHED=0 - for i in {1..30}; do - if kubectl wait --for=condition=Established crd/gateways.gateway.networking.k8s.io \ - crd/httproutes.gateway.networking.k8s.io \ - crd/gatewayclasses.gateway.networking.k8s.io \ - --timeout=60s &>/dev/null 2>&1; then - CRD_ESTABLISHED=1 - break - fi - if [ $((i % 5)) -eq 0 ]; then - echo " Waiting for CRDs... attempt $i/30" - fi - sleep 2 - done - - if [ "$CRD_ESTABLISHED" -eq 1 ]; then - echo " ✓ Gateway API CRDs established" - else - echo " ⚠ Gateway API CRDs may not be fully established yet (installation continues)" - fi - + # Wait for deployment to be available echo " Waiting for Envoy Gateway deployment to be ready..." kubectl wait --for=condition=available deployment/envoy-gateway -n "$NAMESPACE" --timeout=5m || { diff --git a/terraform/modules/envoy_gateway/main.tf b/terraform/modules/envoy_gateway/main.tf index 732d865..790b53e 100644 --- a/terraform/modules/envoy_gateway/main.tf +++ b/terraform/modules/envoy_gateway/main.tf @@ -24,15 +24,15 @@ variable "install_envoy_gateway" { } variable "gateway_api_version" { - description = "Gateway API CRDs version (not used - Envoy Gateway install.yaml includes CRDs)" + description = "Gateway API CRD bundle version shipped by envoy_gateway_version's install.yaml (checked after apply, not installed separately)" type = string - default = "v1.1.0" + default = "v1.6.1" } variable "envoy_gateway_version" { - description = "Envoy Gateway Helm chart version" + description = "Envoy Gateway release (install.yaml). Upgrade one minor version at a time." type = string - default = "v1.6.1" + default = "v1.9.2" } variable "namespace" { @@ -54,6 +54,14 @@ output "cluster_name" { resource "null_resource" "deploy_envoy_gateway" { count = var.install_envoy_gateway ? 1 : 0 + # Re-run the (idempotent, server-side apply) install script on version changes. + # No destroy provisioner: replacing this resource must never uninstall Envoy Gateway. + triggers = { + envoy_gateway_version = var.envoy_gateway_version + gateway_api_version = var.gateway_api_version + script_sha = filesha256("${path.module}/deploy-envoy-gateway.sh") + } + provisioner "local-exec" { command = <<-EOT chmod +x ${path.module}/deploy-envoy-gateway.sh @@ -71,6 +79,10 @@ resource "null_resource" "deploy_envoy_gateway" { resource "null_resource" "verify_envoy_gateway" { count = var.install_envoy_gateway ? 1 : 0 + triggers = { + deploy_id = null_resource.deploy_envoy_gateway[0].id + } + provisioner "local-exec" { command = <<-EOT echo "" diff --git a/terraform/terraform.tfvars.example b/terraform/terraform.tfvars.example index 38ed375..7d36618 100644 --- a/terraform/terraform.tfvars.example +++ b/terraform/terraform.tfvars.example @@ -316,15 +316,15 @@ rke2_registries_yaml_file = "config/rke2-registries.yaml" # Install Envoy Gateway on downstream clusters (recommended for Gateway API) install_envoy_gateway = true -# Gateway API CRDs version (default: v1.1.0) -# Envoy Gateway 1.6.1 uses Gateway API 1.4.1 internally -# v1.1.0 is the latest stable Gateway API CRDs and is compatible -# Alternatively, set to "helm" to let Envoy Gateway Helm chart install CRDs automatically -gateway_api_version = "v1.1.0" +# Gateway API CRD bundle shipped by the Envoy Gateway release below (experimental channel). +# Not installed separately; the deploy script warns if the installed bundle differs. +# Envoy Gateway v1.7.x -> v1.4.1, v1.8.x -> v1.5.1, v1.9.x -> v1.6.1 +gateway_api_version = "v1.6.1" -# Envoy Gateway Helm chart version (default: v1.6.1) +# Envoy Gateway release (install.yaml). Upgrade one minor version at a time +# (e.g. v1.6 -> v1.7 -> v1.8 -> v1.9); never delete the CRDs to "reinstall". # Check latest version: https://github.com/envoyproxy/gateway/releases -envoy_gateway_version = "v1.6.1" +envoy_gateway_version = "v1.9.2" # ============================================================================ # KUBE-VIP CONFIGURATION diff --git a/terraform/variables.tf b/terraform/variables.tf index 2b1307a..122379c 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -55,13 +55,13 @@ variable "clusters" { large_worker_memory_mb = optional(number, null) large_worker_disk_size_gb = optional(number, null) large_worker_proxmox_node = optional(string, null) # Override node for large workers (e.g. after migrate) - domain = string - ip_subnet = string - ip_start_octet = number # Starting IP octet (e.g., 100 for 192.168.1.100) - gateway = string - dns_servers = list(string) - storage = string - vlan_id = number # VLAN ID for network interface + domain = string + ip_subnet = string + ip_start_octet = number # Starting IP octet (e.g., 100 for 192.168.1.100) + gateway = string + dns_servers = list(string) + storage = string + vlan_id = number # VLAN ID for network interface })) } @@ -378,15 +378,15 @@ variable "install_envoy_gateway" { } variable "gateway_api_version" { - description = "Gateway API CRDs version (deprecated - Envoy Gateway install.yaml includes CRDs automatically). Kept for compatibility but not used." + description = "Gateway API CRD bundle version shipped by envoy_gateway_version's install.yaml (checked after apply, not installed separately)" type = string - default = "v1.1.0" + default = "v1.6.1" } variable "envoy_gateway_version" { - description = "Envoy Gateway Helm chart version" + description = "Envoy Gateway release (install.yaml). Upgrade one minor version at a time." type = string - default = "v1.6.1" + default = "v1.9.2" } variable "opensearch_operator_version" {