fix(envoy-gateway): make module safe for upgrades, pin v1.9.2

docs/GATEWAY_API_SETUP.md

@@ -22,17 +22,16 @@ This guide covers setting up Gateway API on RKE2 clusters managed by Rancher. Ga
 Envoy Gateway can be installed using the official installation manifest:
 
 ```bash
-# Install Gateway API CRDs first (if not already installed)
-kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/standard-install.yaml
-
-# Install Envoy Gateway using official manifest (version 1.6.1)
+# install.yaml bundles the Gateway API CRDs (experimental channel); don't install
+# standard-install.yaml separately - it would downgrade/strip the experimental CRDs.
+# Install Envoy Gateway using official manifest (version 1.9.2, Gateway API v1.6.1)
 # On manager cluster
 export KUBECONFIG=~/.kube/rancher-manager.yaml
-kubectl apply -f https://github.com/envoyproxy/gateway/releases/download/v1.6.1/install.yaml
+kubectl apply --server-side -f https://github.com/envoyproxy/gateway/releases/download/v1.9.2/install.yaml
 
 # On nprd-apps cluster
 export KUBECONFIG=~/.kube/nprd-apps.yaml
-kubectl apply -f https://github.com/envoyproxy/gateway/releases/download/v1.6.1/install.yaml
+kubectl apply --server-side -f https://github.com/envoyproxy/gateway/releases/download/v1.9.2/install.yaml
 
 # Wait for deployment to be ready
 kubectl wait --for=condition=available deployment/envoy-gateway -n envoy-gateway-system --timeout=5m
@@ -44,7 +43,7 @@ kubectl get gatewayclass
 
 **Note:** The Helm repository method is not currently available. Use the manifest installation method shown above.
 
-**Note:** Envoy Gateway v1.6.1 uses Gateway API v1.4.1 internally, but Gateway API v1.0.0 CRDs are fully compatible.
+**Note:** Envoy Gateway v1.9.x ships and requires Gateway API v1.6.x CRDs (TCPRoute/UDPRoute `v1`). Terraform (`modules/envoy_gateway`) applies the CRDs first, then the controller; upgrade one minor version at a time and never delete the Gateway API CRDs (that deletes every Gateway and Route).
 
 **Basic Gateway Configuration:**
 
@@ -144,24 +143,14 @@ helm install traefik traefik/traefik \
 ### Prerequisites
 
 ```bash
-# Ensure Gateway API CRDs are installed
-kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/standard-install.yaml
-
-# Note: Envoy Gateway v1.6.1 supports Gateway API v1.0.0
-
-# Verify CRDs
+# Gateway API CRDs come with the Envoy Gateway install.yaml (Step 2).
+# Verify CRDs (after install)
 kubectl get crd | grep gateway
 ```
 
-### Step 1: Install Gateway API CRDs (if not already installed)
+### Step 1: Gateway API CRDs
 
-```bash
-# Install Gateway API CRDs
-kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/standard-install.yaml
-
-# Verify CRDs are installed
-kubectl get crd | grep gateway
-```
+Bundled in the Envoy Gateway `install.yaml` (experimental channel, `v1.6.1` for Envoy Gateway `v1.9.x`). Don't apply `standard-install.yaml` on top.
 
 ### Step 2: Install Envoy Gateway
 
@@ -170,7 +159,7 @@ kubectl get crd | grep gateway
 export KUBECONFIG=~/.kube/rancher-manager.yaml
 
 # Install Envoy Gateway using official manifest
-kubectl apply -f https://github.com/envoyproxy/gateway/releases/download/v1.6.1/install.yaml
+kubectl apply --server-side -f https://github.com/envoyproxy/gateway/releases/download/v1.9.2/install.yaml
 
 # Wait for deployment to be ready
 kubectl wait --for=condition=available deployment/envoy-gateway -n envoy-gateway-system --timeout=5m

terraform/modules/envoy_gateway/deploy-envoy-gateway.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -e
+set -eo pipefail
 
 # Script to deploy Envoy Gateway and Gateway API CRDs to a Kubernetes cluster
 # Usage: deploy-envoy-gateway.sh <kubeconfig_path> <gateway_api_version> <envoy_gateway_version> <namespace> <cluster_name>
@@ -19,7 +19,7 @@ echo "Kubeconfig: $KUBECONFIG"
 echo "Cluster: $CLUSTER_NAME"
 echo "Envoy Gateway Version: $ENVOY_GATEWAY_VERSION"
 echo "Namespace: $NAMESPACE"
-echo "Note: Envoy Gateway install.yaml includes Gateway API CRDs automatically"
+echo "Expected Gateway API bundle: $GATEWAY_API_VERSION (shipped in Envoy Gateway install.yaml)"
 echo ""
 
 # Verify cluster is accessible
@@ -72,99 +72,60 @@ sleep 5
 echo "✓ Cluster is stable, proceeding with installation"
 echo ""
 
-# Step 1: Check for and handle existing Gateway API CRDs
-# Envoy Gateway install.yaml includes Gateway API CRDs. If CRDs from a previous install exist,
-# we need to handle them to avoid annotation size conflicts.
-echo "[1/2] Checking for existing Gateway API CRDs..."
-GATEWAY_CRDS_EXIST=false
+# Step 1: Download the release manifest and split CRDs from the rest.
+# install.yaml bundles the Gateway API (experimental channel) and Envoy Gateway CRDs.
+# Existing CRDs are only ever updated in place: deleting a CRD deletes every
+# Gateway/Route/Policy object of that kind cluster-wide.
+echo "[1/3] Downloading Envoy Gateway $ENVOY_GATEWAY_VERSION manifest..."
+MANIFEST_URL="https://github.com/envoyproxy/gateway/releases/download/${ENVOY_GATEWAY_VERSION}/install.yaml"
+WORKDIR=$(mktemp -d)
+trap 'rm -rf "$WORKDIR"' EXIT
+curl -fsSL -o "$WORKDIR/install.yaml" "$MANIFEST_URL"
+awk -v crds="$WORKDIR/crds.yaml" -v rest="$WORKDIR/rest.yaml" '
+  function flush() { if (doc != "") { print "---\n" doc > (is_crd ? crds : rest) } doc = ""; is_crd = 0 }
+  /^---/ { flush(); next }
+  /^kind: CustomResourceDefinition$/ { is_crd = 1 }
+  { doc = doc $0 "\n" }
+  END { flush() }
+' "$WORKDIR/install.yaml"
+echo "  ✓ $(grep -c '^kind: CustomResourceDefinition$' "$WORKDIR/crds.yaml") CRDs, $(grep -c '^kind:' "$WORKDIR/rest.yaml") other resources"
+echo ""
+
+# Step 2: CRDs first. Envoy Gateway >= v1.9 requires the matching Gateway API CRDs
+# (TCPRoute/UDPRoute v1) before the controller is upgraded, otherwise those routes are skipped.
+echo "[2/3] Applying CRDs (server-side)..."
 if kubectl get crd gateways.gateway.networking.k8s.io &>/dev/null; then
-  GATEWAY_CRDS_EXIST=true
-  echo "  ⚠ Gateway API CRDs already exist"
-  # Check if these are from Envoy Gateway (will have envoy-specific labels) or from a separate install
-  if kubectl get crd gateways.gateway.networking.k8s.io -o yaml | grep -q "gateway.envoyproxy.io" 2>/dev/null; then
-    echo "  CRDs appear to be from Envoy Gateway - will update via server-side apply"
-  else
-    echo "  CRDs appear to be from a separate Gateway API installation"
-    echo "  Will delete and recreate to ensure compatibility with Envoy Gateway"
-    echo "  Deleting existing Gateway API CRDs..."
-    kubectl delete crd \
-      gateways.gateway.networking.k8s.io \
-      httproutes.gateway.networking.k8s.io \
-      gatewayclasses.gateway.networking.k8s.io \
-      grpcroutes.gateway.networking.k8s.io \
-      tcproutes.gateway.networking.k8s.io \
-      tlsroutes.gateway.networking.k8s.io \
-      udproutes.gateway.networking.k8s.io \
-      referencegrants.gateway.networking.k8s.io \
-      backendtlspolicies.gateway.networking.k8s.io \
-      --ignore-not-found=true --wait=true --timeout=60s || {
-      echo "  ⚠ Some CRDs may still exist, continuing with installation..."
-    }
-    echo "  ✓ Existing Gateway API CRDs removed"
-    GATEWAY_CRDS_EXIST=false
-  fi
+  CURRENT_BUNDLE=$(kubectl get crd gateways.gateway.networking.k8s.io \
+    -o jsonpath='{.metadata.annotations.gateway\.networking\.k8s\.io/bundle-version}')
+  echo "  Existing Gateway API CRDs: bundle ${CURRENT_BUNDLE:-unknown}"
+fi
+kubectl apply --server-side --force-conflicts --field-manager=envoy-gateway-installer -f "$WORKDIR/crds.yaml"
+kubectl wait --for=condition=Established --timeout=120s -f "$WORKDIR/crds.yaml"
+NEW_BUNDLE=$(kubectl get crd gateways.gateway.networking.k8s.io \
+  -o jsonpath='{.metadata.annotations.gateway\.networking\.k8s\.io/bundle-version}')
+echo "  ✓ CRDs established (Gateway API bundle $NEW_BUNDLE)"
+if [ -n "$GATEWAY_API_VERSION" ] && [ "$NEW_BUNDLE" != "$GATEWAY_API_VERSION" ]; then
+  echo "  ⚠ gateway_api_version is $GATEWAY_API_VERSION but Envoy Gateway $ENVOY_GATEWAY_VERSION ships $NEW_BUNDLE"
 fi
 echo ""
 
-# Step 2: Install Envoy Gateway using official installation manifest
-echo "[2/2] Installing Envoy Gateway (version $ENVOY_GATEWAY_VERSION)..."
-MANIFEST_URL="https://github.com/envoyproxy/gateway/releases/download/${ENVOY_GATEWAY_VERSION}/install.yaml"
+# Step 3: Envoy Gateway controller, RBAC, webhook and certgen job
+echo "[3/3] Installing Envoy Gateway (version $ENVOY_GATEWAY_VERSION)..."
 
 # Check if Envoy Gateway is already installed
 if kubectl get namespace "$NAMESPACE" &>/dev/null && kubectl get deployment envoy-gateway -n "$NAMESPACE" &>/dev/null; then
   echo "  Envoy Gateway already installed, applying updated manifest with server-side apply..."
-  # Use server-side apply for updates to handle CRD conflicts properly
-  kubectl apply --server-side --force-conflicts --field-manager=envoy-gateway-installer -f "$MANIFEST_URL" 2>&1 | grep -v "Too long: may not be more than" || {
-    echo "  ⚠ Server-side apply had some conflicts, checking if resources were applied..."
-    # Check if deployment was updated despite conflicts
-    kubectl get deployment envoy-gateway -n "$NAMESPACE" || {
-      echo "ERROR: Deployment not found after apply attempt."
-      exit 1
-    }
-  }
+  # The certgen Job template is immutable; a finished Job from a previous version must go first.
+  # certgen does not overwrite existing control-plane cert secrets.
+  kubectl delete job eg-gateway-helm-certgen -n "$NAMESPACE" --ignore-not-found=true --wait=true
+  kubectl apply --server-side --force-conflicts --field-manager=envoy-gateway-installer -f "$WORKDIR/rest.yaml"
+  kubectl rollout status deployment/envoy-gateway -n "$NAMESPACE" --timeout=5m
   echo "  ✓ Envoy Gateway resources updated"
 else
   echo "  Installing Envoy Gateway from official manifest (first time installation)..."
-  # For new installations, use server-side apply which handles CRDs better
-  kubectl apply --server-side --field-manager=envoy-gateway-installer -f "$MANIFEST_URL" || {
-    echo "  ⚠ Server-side apply failed, trying regular apply..."
-    # Regular apply, but filter out CRD annotation size errors (they're warnings, not fatal)
-    kubectl apply -f "$MANIFEST_URL" 2>&1 | grep -v "Too long: may not be more than" || {
-      echo "ERROR: Failed to install Envoy Gateway from manifest."
-      echo ""
-      echo "If you see CRD annotation errors above, the existing Gateway API CRDs conflict with Envoy Gateway's CRDs."
-      echo "To resolve:"
-      echo "  1. Delete existing Gateway API CRDs:"
-      echo "     kubectl delete crd -l gateway.networking.k8s.io/bundle-version"
-      echo "  2. Then re-run terraform apply"
-      exit 1
-    }
-  }
+  kubectl apply --server-side --force-conflicts --field-manager=envoy-gateway-installer -f "$WORKDIR/rest.yaml"
   echo "  ✓ Envoy Gateway manifest applied"
-
-  # Wait for Gateway API CRDs to be established
-  echo "  Waiting for Gateway API CRDs to be established..."
-  CRD_ESTABLISHED=0
-  for i in {1..30}; do
-    if kubectl wait --for=condition=Established crd/gateways.gateway.networking.k8s.io \
-       crd/httproutes.gateway.networking.k8s.io \
-       crd/gatewayclasses.gateway.networking.k8s.io \
-       --timeout=60s &>/dev/null 2>&1; then
-      CRD_ESTABLISHED=1
-      break
-    fi
-    if [ $((i % 5)) -eq 0 ]; then
-      echo "    Waiting for CRDs... attempt $i/30"
-    fi
-    sleep 2
-  done
-
-  if [ "$CRD_ESTABLISHED" -eq 1 ]; then
-    echo "  ✓ Gateway API CRDs established"
-  else
-    echo "  ⚠ Gateway API CRDs may not be fully established yet (installation continues)"
-  fi
-
+
   # Wait for deployment to be available
   echo "  Waiting for Envoy Gateway deployment to be ready..."
   kubectl wait --for=condition=available deployment/envoy-gateway -n "$NAMESPACE" --timeout=5m || {

terraform/modules/envoy_gateway/main.tf

@@ -24,15 +24,15 @@ variable "install_envoy_gateway" {
 }
 
 variable "gateway_api_version" {
-  description = "Gateway API CRDs version (not used - Envoy Gateway install.yaml includes CRDs)"
+  description = "Gateway API CRD bundle version shipped by envoy_gateway_version's install.yaml (checked after apply, not installed separately)"
   type        = string
-  default     = "v1.1.0"
+  default     = "v1.6.1"
 }
 
 variable "envoy_gateway_version" {
-  description = "Envoy Gateway Helm chart version"
+  description = "Envoy Gateway release (install.yaml). Upgrade one minor version at a time."
   type        = string
-  default     = "v1.6.1"
+  default     = "v1.9.2"
 }
 
 variable "namespace" {
@@ -54,6 +54,14 @@ output "cluster_name" {
 resource "null_resource" "deploy_envoy_gateway" {
   count = var.install_envoy_gateway ? 1 : 0
 
+  # Re-run the (idempotent, server-side apply) install script on version changes.
+  # No destroy provisioner: replacing this resource must never uninstall Envoy Gateway.
+  triggers = {
+    envoy_gateway_version = var.envoy_gateway_version
+    gateway_api_version   = var.gateway_api_version
+    script_sha            = filesha256("${path.module}/deploy-envoy-gateway.sh")
+  }
+
   provisioner "local-exec" {
     command = <<-EOT
       chmod +x ${path.module}/deploy-envoy-gateway.sh
@@ -71,6 +79,10 @@ resource "null_resource" "deploy_envoy_gateway" {
 resource "null_resource" "verify_envoy_gateway" {
   count = var.install_envoy_gateway ? 1 : 0
 
+  triggers = {
+    deploy_id = null_resource.deploy_envoy_gateway[0].id
+  }
+
   provisioner "local-exec" {
     command = <<-EOT
       echo ""

terraform/terraform.tfvars.example

@@ -316,15 +316,15 @@ rke2_registries_yaml_file = "config/rke2-registries.yaml"
 # Install Envoy Gateway on downstream clusters (recommended for Gateway API)
 install_envoy_gateway = true
 
-# Gateway API CRDs version (default: v1.1.0)
-# Envoy Gateway 1.6.1 uses Gateway API 1.4.1 internally
-# v1.1.0 is the latest stable Gateway API CRDs and is compatible
-# Alternatively, set to "helm" to let Envoy Gateway Helm chart install CRDs automatically
-gateway_api_version = "v1.1.0"
+# Gateway API CRD bundle shipped by the Envoy Gateway release below (experimental channel).
+# Not installed separately; the deploy script warns if the installed bundle differs.
+# Envoy Gateway v1.7.x -> v1.4.1, v1.8.x -> v1.5.1, v1.9.x -> v1.6.1
+gateway_api_version = "v1.6.1"
 
-# Envoy Gateway Helm chart version (default: v1.6.1)
+# Envoy Gateway release (install.yaml). Upgrade one minor version at a time
+# (e.g. v1.6 -> v1.7 -> v1.8 -> v1.9); never delete the CRDs to "reinstall".
 # Check latest version: https://github.com/envoyproxy/gateway/releases
-envoy_gateway_version = "v1.6.1"
+envoy_gateway_version = "v1.9.2"
 
 # ============================================================================
 # KUBE-VIP CONFIGURATION

terraform/variables.tf

@@ -55,13 +55,13 @@ variable "clusters" {
     large_worker_memory_mb    = optional(number, null)
     large_worker_disk_size_gb = optional(number, null)
     large_worker_proxmox_node = optional(string, null) # Override node for large workers (e.g. after migrate)
-    domain         = string
-    ip_subnet      = string
-    ip_start_octet = number # Starting IP octet (e.g., 100 for 192.168.1.100)
-    gateway        = string
-    dns_servers    = list(string)
-    storage        = string
-    vlan_id        = number # VLAN ID for network interface
+    domain                    = string
+    ip_subnet                 = string
+    ip_start_octet            = number # Starting IP octet (e.g., 100 for 192.168.1.100)
+    gateway                   = string
+    dns_servers               = list(string)
+    storage                   = string
+    vlan_id                   = number # VLAN ID for network interface
   }))
 }
 
@@ -378,15 +378,15 @@ variable "install_envoy_gateway" {
 }
 
 variable "gateway_api_version" {
-  description = "Gateway API CRDs version (deprecated - Envoy Gateway install.yaml includes CRDs automatically). Kept for compatibility but not used."
+  description = "Gateway API CRD bundle version shipped by envoy_gateway_version's install.yaml (checked after apply, not installed separately)"
   type        = string
-  default     = "v1.1.0"
+  default     = "v1.6.1"
 }
 
 variable "envoy_gateway_version" {
-  description = "Envoy Gateway Helm chart version"
+  description = "Envoy Gateway release (install.yaml). Upgrade one minor version at a time."
   type        = string
-  default     = "v1.6.1"
+  default     = "v1.9.2"
 }
 
 variable "opensearch_operator_version" {