diff --git a/docs/GATEWAY_API_SETUP.md b/docs/GATEWAY_API_SETUP.md index 5f9a682..bc59c96 100644 --- a/docs/GATEWAY_API_SETUP.md +++ b/docs/GATEWAY_API_SETUP.md @@ -22,17 +22,16 @@ This guide covers setting up Gateway API on RKE2 clusters managed by Rancher. Ga Envoy Gateway can be installed using the official installation manifest: ```bash -# Install Gateway API CRDs first (if not already installed) -kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/standard-install.yaml - -# Install Envoy Gateway using official manifest (version 1.6.1) +# install.yaml bundles the Gateway API CRDs (experimental channel); don't install +# standard-install.yaml separately - it would downgrade/strip the experimental CRDs. +# Install Envoy Gateway using official manifest (version 1.9.2, Gateway API v1.6.1) # On manager cluster export KUBECONFIG=~/.kube/rancher-manager.yaml -kubectl apply -f https://github.com/envoyproxy/gateway/releases/download/v1.6.1/install.yaml +kubectl apply --server-side -f https://github.com/envoyproxy/gateway/releases/download/v1.9.2/install.yaml # On nprd-apps cluster export KUBECONFIG=~/.kube/nprd-apps.yaml -kubectl apply -f https://github.com/envoyproxy/gateway/releases/download/v1.6.1/install.yaml +kubectl apply --server-side -f https://github.com/envoyproxy/gateway/releases/download/v1.9.2/install.yaml # Wait for deployment to be ready kubectl wait --for=condition=available deployment/envoy-gateway -n envoy-gateway-system --timeout=5m @@ -44,7 +43,7 @@ kubectl get gatewayclass **Note:** The Helm repository method is not currently available. Use the manifest installation method shown above. -**Note:** Envoy Gateway v1.6.1 uses Gateway API v1.4.1 internally, but Gateway API v1.0.0 CRDs are fully compatible. +**Note:** Envoy Gateway v1.9.x ships and requires Gateway API v1.6.x CRDs (TCPRoute/UDPRoute `v1`). Terraform (`modules/envoy_gateway`) applies the CRDs first, then the controller; upgrade one minor version at a time and never delete the Gateway API CRDs (that deletes every Gateway and Route). **Basic Gateway Configuration:** @@ -144,24 +143,14 @@ helm install traefik traefik/traefik \ ### Prerequisites ```bash -# Ensure Gateway API CRDs are installed -kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/standard-install.yaml - -# Note: Envoy Gateway v1.6.1 supports Gateway API v1.0.0 - -# Verify CRDs +# Gateway API CRDs come with the Envoy Gateway install.yaml (Step 2). +# Verify CRDs (after install) kubectl get crd | grep gateway ``` -### Step 1: Install Gateway API CRDs (if not already installed) +### Step 1: Gateway API CRDs -```bash -# Install Gateway API CRDs -kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/standard-install.yaml - -# Verify CRDs are installed -kubectl get crd | grep gateway -``` +Bundled in the Envoy Gateway `install.yaml` (experimental channel, `v1.6.1` for Envoy Gateway `v1.9.x`). Don't apply `standard-install.yaml` on top. ### Step 2: Install Envoy Gateway @@ -170,7 +159,7 @@ kubectl get crd | grep gateway export KUBECONFIG=~/.kube/rancher-manager.yaml # Install Envoy Gateway using official manifest -kubectl apply -f https://github.com/envoyproxy/gateway/releases/download/v1.6.1/install.yaml +kubectl apply --server-side -f https://github.com/envoyproxy/gateway/releases/download/v1.9.2/install.yaml # Wait for deployment to be ready kubectl wait --for=condition=available deployment/envoy-gateway -n envoy-gateway-system --timeout=5m diff --git a/terraform/modules/envoy_gateway/deploy-envoy-gateway.sh b/terraform/modules/envoy_gateway/deploy-envoy-gateway.sh index 2743e95..5b4fc11 100755 --- a/terraform/modules/envoy_gateway/deploy-envoy-gateway.sh +++ b/terraform/modules/envoy_gateway/deploy-envoy-gateway.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -eo pipefail # Script to deploy Envoy Gateway and Gateway API CRDs to a Kubernetes cluster # Usage: deploy-envoy-gateway.sh @@ -19,7 +19,7 @@ echo "Kubeconfig: $KUBECONFIG" echo "Cluster: $CLUSTER_NAME" echo "Envoy Gateway Version: $ENVOY_GATEWAY_VERSION" echo "Namespace: $NAMESPACE" -echo "Note: Envoy Gateway install.yaml includes Gateway API CRDs automatically" +echo "Expected Gateway API bundle: $GATEWAY_API_VERSION (shipped in Envoy Gateway install.yaml)" echo "" # Verify cluster is accessible @@ -72,99 +72,60 @@ sleep 5 echo "✓ Cluster is stable, proceeding with installation" echo "" -# Step 1: Check for and handle existing Gateway API CRDs -# Envoy Gateway install.yaml includes Gateway API CRDs. If CRDs from a previous install exist, -# we need to handle them to avoid annotation size conflicts. -echo "[1/2] Checking for existing Gateway API CRDs..." -GATEWAY_CRDS_EXIST=false +# Step 1: Download the release manifest and split CRDs from the rest. +# install.yaml bundles the Gateway API (experimental channel) and Envoy Gateway CRDs. +# Existing CRDs are only ever updated in place: deleting a CRD deletes every +# Gateway/Route/Policy object of that kind cluster-wide. +echo "[1/3] Downloading Envoy Gateway $ENVOY_GATEWAY_VERSION manifest..." +MANIFEST_URL="https://github.com/envoyproxy/gateway/releases/download/${ENVOY_GATEWAY_VERSION}/install.yaml" +WORKDIR=$(mktemp -d) +trap 'rm -rf "$WORKDIR"' EXIT +curl -fsSL -o "$WORKDIR/install.yaml" "$MANIFEST_URL" +awk -v crds="$WORKDIR/crds.yaml" -v rest="$WORKDIR/rest.yaml" ' + function flush() { if (doc != "") { print "---\n" doc > (is_crd ? crds : rest) } doc = ""; is_crd = 0 } + /^---/ { flush(); next } + /^kind: CustomResourceDefinition$/ { is_crd = 1 } + { doc = doc $0 "\n" } + END { flush() } +' "$WORKDIR/install.yaml" +echo " ✓ $(grep -c '^kind: CustomResourceDefinition$' "$WORKDIR/crds.yaml") CRDs, $(grep -c '^kind:' "$WORKDIR/rest.yaml") other resources" +echo "" + +# Step 2: CRDs first. Envoy Gateway >= v1.9 requires the matching Gateway API CRDs +# (TCPRoute/UDPRoute v1) before the controller is upgraded, otherwise those routes are skipped. +echo "[2/3] Applying CRDs (server-side)..." if kubectl get crd gateways.gateway.networking.k8s.io &>/dev/null; then - GATEWAY_CRDS_EXIST=true - echo " ⚠ Gateway API CRDs already exist" - # Check if these are from Envoy Gateway (will have envoy-specific labels) or from a separate install - if kubectl get crd gateways.gateway.networking.k8s.io -o yaml | grep -q "gateway.envoyproxy.io" 2>/dev/null; then - echo " CRDs appear to be from Envoy Gateway - will update via server-side apply" - else - echo " CRDs appear to be from a separate Gateway API installation" - echo " Will delete and recreate to ensure compatibility with Envoy Gateway" - echo " Deleting existing Gateway API CRDs..." - kubectl delete crd \ - gateways.gateway.networking.k8s.io \ - httproutes.gateway.networking.k8s.io \ - gatewayclasses.gateway.networking.k8s.io \ - grpcroutes.gateway.networking.k8s.io \ - tcproutes.gateway.networking.k8s.io \ - tlsroutes.gateway.networking.k8s.io \ - udproutes.gateway.networking.k8s.io \ - referencegrants.gateway.networking.k8s.io \ - backendtlspolicies.gateway.networking.k8s.io \ - --ignore-not-found=true --wait=true --timeout=60s || { - echo " ⚠ Some CRDs may still exist, continuing with installation..." - } - echo " ✓ Existing Gateway API CRDs removed" - GATEWAY_CRDS_EXIST=false - fi + CURRENT_BUNDLE=$(kubectl get crd gateways.gateway.networking.k8s.io \ + -o jsonpath='{.metadata.annotations.gateway\.networking\.k8s\.io/bundle-version}') + echo " Existing Gateway API CRDs: bundle ${CURRENT_BUNDLE:-unknown}" +fi +kubectl apply --server-side --force-conflicts --field-manager=envoy-gateway-installer -f "$WORKDIR/crds.yaml" +kubectl wait --for=condition=Established --timeout=120s -f "$WORKDIR/crds.yaml" +NEW_BUNDLE=$(kubectl get crd gateways.gateway.networking.k8s.io \ + -o jsonpath='{.metadata.annotations.gateway\.networking\.k8s\.io/bundle-version}') +echo " ✓ CRDs established (Gateway API bundle $NEW_BUNDLE)" +if [ -n "$GATEWAY_API_VERSION" ] && [ "$NEW_BUNDLE" != "$GATEWAY_API_VERSION" ]; then + echo " ⚠ gateway_api_version is $GATEWAY_API_VERSION but Envoy Gateway $ENVOY_GATEWAY_VERSION ships $NEW_BUNDLE" fi echo "" -# Step 2: Install Envoy Gateway using official installation manifest -echo "[2/2] Installing Envoy Gateway (version $ENVOY_GATEWAY_VERSION)..." -MANIFEST_URL="https://github.com/envoyproxy/gateway/releases/download/${ENVOY_GATEWAY_VERSION}/install.yaml" +# Step 3: Envoy Gateway controller, RBAC, webhook and certgen job +echo "[3/3] Installing Envoy Gateway (version $ENVOY_GATEWAY_VERSION)..." # Check if Envoy Gateway is already installed if kubectl get namespace "$NAMESPACE" &>/dev/null && kubectl get deployment envoy-gateway -n "$NAMESPACE" &>/dev/null; then echo " Envoy Gateway already installed, applying updated manifest with server-side apply..." - # Use server-side apply for updates to handle CRD conflicts properly - kubectl apply --server-side --force-conflicts --field-manager=envoy-gateway-installer -f "$MANIFEST_URL" 2>&1 | grep -v "Too long: may not be more than" || { - echo " ⚠ Server-side apply had some conflicts, checking if resources were applied..." - # Check if deployment was updated despite conflicts - kubectl get deployment envoy-gateway -n "$NAMESPACE" || { - echo "ERROR: Deployment not found after apply attempt." - exit 1 - } - } + # The certgen Job template is immutable; a finished Job from a previous version must go first. + # certgen does not overwrite existing control-plane cert secrets. + kubectl delete job eg-gateway-helm-certgen -n "$NAMESPACE" --ignore-not-found=true --wait=true + kubectl apply --server-side --force-conflicts --field-manager=envoy-gateway-installer -f "$WORKDIR/rest.yaml" + kubectl rollout status deployment/envoy-gateway -n "$NAMESPACE" --timeout=5m echo " ✓ Envoy Gateway resources updated" else echo " Installing Envoy Gateway from official manifest (first time installation)..." - # For new installations, use server-side apply which handles CRDs better - kubectl apply --server-side --field-manager=envoy-gateway-installer -f "$MANIFEST_URL" || { - echo " ⚠ Server-side apply failed, trying regular apply..." - # Regular apply, but filter out CRD annotation size errors (they're warnings, not fatal) - kubectl apply -f "$MANIFEST_URL" 2>&1 | grep -v "Too long: may not be more than" || { - echo "ERROR: Failed to install Envoy Gateway from manifest." - echo "" - echo "If you see CRD annotation errors above, the existing Gateway API CRDs conflict with Envoy Gateway's CRDs." - echo "To resolve:" - echo " 1. Delete existing Gateway API CRDs:" - echo " kubectl delete crd -l gateway.networking.k8s.io/bundle-version" - echo " 2. Then re-run terraform apply" - exit 1 - } - } + kubectl apply --server-side --force-conflicts --field-manager=envoy-gateway-installer -f "$WORKDIR/rest.yaml" echo " ✓ Envoy Gateway manifest applied" - - # Wait for Gateway API CRDs to be established - echo " Waiting for Gateway API CRDs to be established..." - CRD_ESTABLISHED=0 - for i in {1..30}; do - if kubectl wait --for=condition=Established crd/gateways.gateway.networking.k8s.io \ - crd/httproutes.gateway.networking.k8s.io \ - crd/gatewayclasses.gateway.networking.k8s.io \ - --timeout=60s &>/dev/null 2>&1; then - CRD_ESTABLISHED=1 - break - fi - if [ $((i % 5)) -eq 0 ]; then - echo " Waiting for CRDs... attempt $i/30" - fi - sleep 2 - done - - if [ "$CRD_ESTABLISHED" -eq 1 ]; then - echo " ✓ Gateway API CRDs established" - else - echo " ⚠ Gateway API CRDs may not be fully established yet (installation continues)" - fi - + # Wait for deployment to be available echo " Waiting for Envoy Gateway deployment to be ready..." kubectl wait --for=condition=available deployment/envoy-gateway -n "$NAMESPACE" --timeout=5m || { diff --git a/terraform/modules/envoy_gateway/main.tf b/terraform/modules/envoy_gateway/main.tf index 732d865..790b53e 100644 --- a/terraform/modules/envoy_gateway/main.tf +++ b/terraform/modules/envoy_gateway/main.tf @@ -24,15 +24,15 @@ variable "install_envoy_gateway" { } variable "gateway_api_version" { - description = "Gateway API CRDs version (not used - Envoy Gateway install.yaml includes CRDs)" + description = "Gateway API CRD bundle version shipped by envoy_gateway_version's install.yaml (checked after apply, not installed separately)" type = string - default = "v1.1.0" + default = "v1.6.1" } variable "envoy_gateway_version" { - description = "Envoy Gateway Helm chart version" + description = "Envoy Gateway release (install.yaml). Upgrade one minor version at a time." type = string - default = "v1.6.1" + default = "v1.9.2" } variable "namespace" { @@ -54,6 +54,14 @@ output "cluster_name" { resource "null_resource" "deploy_envoy_gateway" { count = var.install_envoy_gateway ? 1 : 0 + # Re-run the (idempotent, server-side apply) install script on version changes. + # No destroy provisioner: replacing this resource must never uninstall Envoy Gateway. + triggers = { + envoy_gateway_version = var.envoy_gateway_version + gateway_api_version = var.gateway_api_version + script_sha = filesha256("${path.module}/deploy-envoy-gateway.sh") + } + provisioner "local-exec" { command = <<-EOT chmod +x ${path.module}/deploy-envoy-gateway.sh @@ -71,6 +79,10 @@ resource "null_resource" "deploy_envoy_gateway" { resource "null_resource" "verify_envoy_gateway" { count = var.install_envoy_gateway ? 1 : 0 + triggers = { + deploy_id = null_resource.deploy_envoy_gateway[0].id + } + provisioner "local-exec" { command = <<-EOT echo "" diff --git a/terraform/terraform.tfvars.example b/terraform/terraform.tfvars.example index 38ed375..7d36618 100644 --- a/terraform/terraform.tfvars.example +++ b/terraform/terraform.tfvars.example @@ -316,15 +316,15 @@ rke2_registries_yaml_file = "config/rke2-registries.yaml" # Install Envoy Gateway on downstream clusters (recommended for Gateway API) install_envoy_gateway = true -# Gateway API CRDs version (default: v1.1.0) -# Envoy Gateway 1.6.1 uses Gateway API 1.4.1 internally -# v1.1.0 is the latest stable Gateway API CRDs and is compatible -# Alternatively, set to "helm" to let Envoy Gateway Helm chart install CRDs automatically -gateway_api_version = "v1.1.0" +# Gateway API CRD bundle shipped by the Envoy Gateway release below (experimental channel). +# Not installed separately; the deploy script warns if the installed bundle differs. +# Envoy Gateway v1.7.x -> v1.4.1, v1.8.x -> v1.5.1, v1.9.x -> v1.6.1 +gateway_api_version = "v1.6.1" -# Envoy Gateway Helm chart version (default: v1.6.1) +# Envoy Gateway release (install.yaml). Upgrade one minor version at a time +# (e.g. v1.6 -> v1.7 -> v1.8 -> v1.9); never delete the CRDs to "reinstall". # Check latest version: https://github.com/envoyproxy/gateway/releases -envoy_gateway_version = "v1.6.1" +envoy_gateway_version = "v1.9.2" # ============================================================================ # KUBE-VIP CONFIGURATION diff --git a/terraform/variables.tf b/terraform/variables.tf index 2b1307a..122379c 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -55,13 +55,13 @@ variable "clusters" { large_worker_memory_mb = optional(number, null) large_worker_disk_size_gb = optional(number, null) large_worker_proxmox_node = optional(string, null) # Override node for large workers (e.g. after migrate) - domain = string - ip_subnet = string - ip_start_octet = number # Starting IP octet (e.g., 100 for 192.168.1.100) - gateway = string - dns_servers = list(string) - storage = string - vlan_id = number # VLAN ID for network interface + domain = string + ip_subnet = string + ip_start_octet = number # Starting IP octet (e.g., 100 for 192.168.1.100) + gateway = string + dns_servers = list(string) + storage = string + vlan_id = number # VLAN ID for network interface })) } @@ -378,15 +378,15 @@ variable "install_envoy_gateway" { } variable "gateway_api_version" { - description = "Gateway API CRDs version (deprecated - Envoy Gateway install.yaml includes CRDs automatically). Kept for compatibility but not used." + description = "Gateway API CRD bundle version shipped by envoy_gateway_version's install.yaml (checked after apply, not installed separately)" type = string - default = "v1.1.0" + default = "v1.6.1" } variable "envoy_gateway_version" { - description = "Envoy Gateway Helm chart version" + description = "Envoy Gateway release (install.yaml). Upgrade one minor version at a time." type = string - default = "v1.6.1" + default = "v1.9.2" } variable "opensearch_operator_version" {