diff --git a/api/v1alpha1/shared_types.go b/api/v1alpha1/shared_types.go index d33f6544d3..3944e275b3 100644 --- a/api/v1alpha1/shared_types.go +++ b/api/v1alpha1/shared_types.go @@ -105,6 +105,16 @@ type KubernetesDeploymentSpec struct { // +optional Replicas *int32 `json:"replicas,omitempty"` + // ReplicasManagedByExternalAutoscaler, when set to true, tells Envoy Gateway not to + // render the replicas field on the generated Deployment, so an external autoscaler + // (an external HorizontalPodAutoscaler, a KEDA ScaledObject, etc.) can own the replica + // count without Envoy Gateway reverting it on the next reconcile. The Replicas field is + // ignored when this is true. This has no additional effect when the built-in EnvoyHpa is + // configured, which already omits the replicas field. + // + // +optional + ReplicasManagedByExternalAutoscaler *bool `json:"replicasManagedByExternalAutoscaler,omitempty"` + // The deployment strategy to use to replace existing pods with new ones. // +optional Strategy *appsv1.DeploymentStrategy `json:"strategy,omitempty"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index b966f945df..b08a9e670b 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -5513,6 +5513,11 @@ func (in *KubernetesDeploymentSpec) DeepCopyInto(out *KubernetesDeploymentSpec) *out = new(int32) **out = **in } + if in.ReplicasManagedByExternalAutoscaler != nil { + in, out := &in.ReplicasManagedByExternalAutoscaler, &out.ReplicasManagedByExternalAutoscaler + *out = new(bool) + **out = **in + } if in.Strategy != nil { in, out := &in.Strategy, &out.Strategy *out = new(appsv1.DeploymentStrategy) diff --git a/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml index 649f37b137..c1a62156e7 100644 --- a/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -10350,6 +10350,15 @@ spec: to 1. format: int32 type: integer + replicasManagedByExternalAutoscaler: + description: |- + ReplicasManagedByExternalAutoscaler, when set to true, tells Envoy Gateway not to + render the replicas field on the generated Deployment, so an external autoscaler + (an external HorizontalPodAutoscaler, a KEDA ScaledObject, etc.) can own the replica + count without Envoy Gateway reverting it on the next reconcile. The Replicas field is + ignored when this is true. This has no additional effect when the built-in EnvoyHpa is + configured, which already omits the replicas field. + type: boolean strategy: description: The deployment strategy to use to replace existing pods with new ones. diff --git a/charts/gateway-helm/charts/crds/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/charts/crds/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index c213f6d051..ebd2bfb4e4 100644 --- a/charts/gateway-helm/charts/crds/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/charts/crds/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -10349,6 +10349,15 @@ spec: to 1. format: int32 type: integer + replicasManagedByExternalAutoscaler: + description: |- + ReplicasManagedByExternalAutoscaler, when set to true, tells Envoy Gateway not to + render the replicas field on the generated Deployment, so an external autoscaler + (an external HorizontalPodAutoscaler, a KEDA ScaledObject, etc.) can own the replica + count without Envoy Gateway reverting it on the next reconcile. The Replicas field is + ignored when this is true. This has no additional effect when the built-in EnvoyHpa is + configured, which already omits the replicas field. + type: boolean strategy: description: The deployment strategy to use to replace existing pods with new ones. diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider.go b/internal/infrastructure/kubernetes/proxy/resource_provider.go index 3ce6421fae..9f3445c535 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_provider.go +++ b/internal/infrastructure/kubernetes/proxy/resource_provider.go @@ -393,12 +393,14 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { return nil, err } - // When an HPA is configured, the replica count is owned by the HPA, so the replicas - // field is left unset here. Since the field is omitted from the server-side apply - // patch, Envoy Gateway doesn't take ownership of it and won't revert the replica - // count computed by the HPA on subsequent reconciliations. + // When an autoscaler owns the replica count, the replicas field is left unset here. + // Since the field is omitted from the server-side apply patch, Envoy Gateway doesn't + // take ownership of it and won't revert the count on subsequent reconciliations. This + // applies both to the built-in EnvoyHpa and to an external autoscaler (external HPA, + // KEDA ScaledObject, etc.) opted into via ReplicasManagedByExternalAutoscaler. replicas := deploymentConfig.Replicas - if provider.GetEnvoyProxyKubeProvider().EnvoyHpa != nil { + if provider.GetEnvoyProxyKubeProvider().EnvoyHpa != nil || + ptr.Deref(deploymentConfig.ReplicasManagedByExternalAutoscaler, false) { replicas = nil } diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go index 33740c04a7..435df808c0 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go +++ b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go @@ -690,6 +690,18 @@ func TestDeployment(t *testing.T) { MaxReplicas: new(int32(10)), }, }, + { + // The replicas field must not be rendered when it is managed by an external + // autoscaler (external HPA, KEDA, etc.), even though no built-in EnvoyHpa is set, + // so that Envoy Gateway doesn't own spec.replicas and revert the external count. + caseName: "with-external-autoscaler", + infra: newTestInfra(), + deploy: &egv1a1.KubernetesDeploymentSpec{ + Replicas: new(int32(2)), + ReplicasManagedByExternalAutoscaler: new(true), + Strategy: egv1a1.DefaultKubernetesDeploymentStrategy(), + }, + }, { caseName: "gateway-namespace-mode", infra: newTestInfraWithNamespacedName(types.NamespacedName{Namespace: "ns1", Name: "gateway-1"}), diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-external-autoscaler.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-external-autoscaler.yaml new file mode 100644 index 0000000000..4024d79708 --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-external-autoscaler.yaml @@ -0,0 +1,404 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + name: envoy-default-37a8eec1 + namespace: envoy-gateway-system + ownerReferences: + - apiVersion: gateway.networking.k8s.io/v1 + kind: GatewayClass + name: envoy-gateway-class + uid: test-owner-reference-uid-for-gatewayclass +spec: + progressDeadlineSeconds: 600 + revisionHistoryLimit: 10 + selector: + matchLabels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + strategy: + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/path: /stats/prometheus + prometheus.io/port: "19001" + prometheus.io/scrape: "true" + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + spec: + automountServiceAccountToken: false + containers: + - args: + - --service-cluster + - default + - --service-node + - $(ENVOY_POD_NAME) + - --config-yaml + - | + admin: + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + cluster_manager: + local_cluster_name: default + node: + locality: + zone: $(ENVOY_SERVICE_ZONE) + deferredStatOptions: + enableDeferredCreationStats: true + stats_config: + use_all_default_tags: true + stats_tags: + - regex: \.zone(\.(([^\.]+)\.)) + tag_name: from_zone + - regex: \.zone\.[^\.]+\.(([^\.]+)\.) + tag_name: to_zone + - regex: "^cluster(\\..+\\.(.+))\\.total_match_count$" + tag_name: socket_match_name + - regex: "circuit_breakers\\.((.+?)\\.).+" + tag_name: priority + layered_runtime: + layers: + - name: global_config + static_layer: + envoy.restart_features.use_eds_cache_for_ads: true + re2.max_program_size.error_level: 4294967295 + re2.max_program_size.warn_level: 1000 + dynamic_resources: + ads_config: + api_type: DELTA_GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + set_node_on_first_message_only: true + lds_config: + ads: {} + initial_fetch_timeout: 0s + resource_api_version: V3 + cds_config: + ads: {} + initial_fetch_timeout: 0s + resource_api_version: V3 + static_resources: + listeners: + - name: envoy-gateway-proxy-stats-0.0.0.0-19001 + address: + socket_address: + address: '0.0.0.0' + port_value: 19001 + protocol: TCP + bypass_overload_manager: true + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: eg-stats-http + normalize_path: true + route_config: + name: local_route + virtual_hosts: + - name: prometheus_stats + domains: + - "*" + routes: + - match: + path: /stats/prometheus + headers: + - name: ":method" + string_match: + exact: GET + route: + cluster: prometheus_stats + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: prometheus_stats + connect_timeout: 0.250s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: prometheus_stats + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + - connect_timeout: 10s + eds_cluster_config: + eds_config: + ads: {} + resource_api_version: 'V3' + service_name: default + load_balancing_policy: + policies: + - typed_extension_config: + name: 'envoy.load_balancing_policies.least_request' + typed_config: + '@type': 'type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest' + locality_lb_config: + zone_aware_lb_config: + min_cluster_size: '1' + name: default + type: EDS + - connect_timeout: 10s + load_assignment: + cluster_name: xds_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway.envoy-gateway-system.svc.cluster.local. + port_value: 18000 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: + connection_keepalive: + interval: 30s + timeout: 5s + name: xds_cluster + type: STRICT_DNS + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: /sds/xds-certificate.json + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: /sds/xds-trusted-ca.json + resource_api_version: V3 + overload_manager: + refresh_interval: 0.25s + resource_monitors: + - name: "envoy.resource_monitors.global_downstream_max_connections" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 50000 + - --log-level + - warn + - --cpuset-threads + - --drain-strategy + - immediate + - --component-log-level + - misc:error + - --drain-time-s + - "60" + command: + - envoy + env: + - name: ENVOY_POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: ENVOY_SERVICE_ZONE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.annotations['topology.kubernetes.io/zone'] + image: docker.io/envoyproxy/envoy:distroless-dev + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + httpGet: + path: /shutdown/ready + port: 19002 + scheme: HTTP + livenessProbe: + failureThreshold: 3 + httpGet: + path: /ready + port: 19003 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: envoy + ports: + - containerPort: 19001 + name: metrics + protocol: TCP + - containerPort: 19003 + name: readiness + protocol: TCP + readinessProbe: + failureThreshold: 1 + httpGet: + path: /ready + port: 19003 + scheme: HTTP + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 100m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + startupProbe: + failureThreshold: 30 + httpGet: + path: /ready + port: 19003 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - mountPath: /sds + name: sds + - args: + - envoy + - shutdown-manager + command: + - envoy-gateway + env: + - name: ENVOY_POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: ENVOY_SERVICE_ZONE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.annotations['topology.kubernetes.io/zone'] + image: docker.io/envoyproxy/gateway-dev:latest + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - envoy-gateway + - envoy + - shutdown + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: shutdown-manager + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 10m + memory: 32Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + startupProbe: + failureThreshold: 30 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: envoy-default-37a8eec1 + terminationGracePeriodSeconds: 360 + volumes: + - name: certs + secret: + defaultMode: 420 + secretName: envoy + - configMap: + defaultMode: 420 + items: + - key: xds-trusted-ca.json + path: xds-trusted-ca.json + - key: xds-certificate.json + path: xds-certificate.json + name: envoy-default-37a8eec1 + optional: false + name: sds +status: {} diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go index e71d206d72..16f55361f5 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go @@ -235,12 +235,14 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { } } - // When an HPA is configured, the replica count is owned by the HPA, so the replicas - // field is left unset here. Since the field is omitted from the server-side apply - // patch, Envoy Gateway doesn't take ownership of it and won't revert the replica - // count computed by the HPA on subsequent reconciliations. + // When an autoscaler owns the replica count, the replicas field is left unset here. + // Since the field is omitted from the server-side apply patch, Envoy Gateway doesn't + // take ownership of it and won't revert the count on subsequent reconciliations. This + // applies both to the built-in RateLimitHpa and to an external autoscaler (external HPA, + // KEDA ScaledObject, etc.) opted into via ReplicasManagedByExternalAutoscaler. replicas := r.rateLimitDeployment.Replicas - if r.rateLimitHpa != nil { + if r.rateLimitHpa != nil || + ptr.Deref(r.rateLimitDeployment.ReplicasManagedByExternalAutoscaler, false) { replicas = nil } diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go index d39b7c9edf..f259ffe5ec 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go @@ -818,6 +818,18 @@ func TestDeployment(t *testing.T) { MaxReplicas: new(int32(10)), }, }, + { + // The replicas field must not be rendered when it is managed by an external + // autoscaler, even though no built-in RateLimitHpa is set, so that Envoy Gateway + // doesn't own spec.replicas and revert the externally computed count. + caseName: "with-external-autoscaler", + rateLimit: rateLimit, + deploy: &egv1a1.KubernetesDeploymentSpec{ + Replicas: new(int32(2)), + ReplicasManagedByExternalAutoscaler: new(true), + Strategy: egv1a1.DefaultKubernetesDeploymentStrategy(), + }, + }, } for _, tc := range cases { t.Run(tc.caseName, func(t *testing.T) { diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/with-external-autoscaler.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/with-external-autoscaler.yaml new file mode 100644 index 0000000000..0a31f4994b --- /dev/null +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/with-external-autoscaler.yaml @@ -0,0 +1,164 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: ratelimit + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy-ratelimit + name: envoy-ratelimit + namespace: envoy-gateway-system + ownerReferences: + - apiVersion: apps/v1 + kind: Deployment + name: envoy-gateway + uid: test-owner-reference-uid-for-deployment +spec: + progressDeadlineSeconds: 600 + revisionHistoryLimit: 10 + selector: + matchLabels: + app.kubernetes.io/component: ratelimit + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy-ratelimit + strategy: + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "19001" + prometheus.io/scrape: "true" + labels: + app.kubernetes.io/component: ratelimit + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy-ratelimit + spec: + automountServiceAccountToken: false + containers: + - command: + - /bin/ratelimit + env: + - name: RUNTIME_ROOT + value: /data + - name: RUNTIME_SUBDIRECTORY + value: ratelimit + - name: RUNTIME_IGNOREDOTFILES + value: "true" + - name: RUNTIME_WATCH_ROOT + value: "false" + - name: LOG_LEVEL + value: info + - name: USE_STATSD + value: "false" + - name: CONFIG_TYPE + value: GRPC_XDS_SOTW + - name: CONFIG_GRPC_XDS_SERVER_URL + value: envoy-gateway:18001 + - name: CONFIG_GRPC_XDS_NODE_ID + value: envoy-ratelimit + - name: GRPC_SERVER_USE_TLS + value: "true" + - name: GRPC_SERVER_TLS_CERT + value: /certs/tls.crt + - name: GRPC_SERVER_TLS_KEY + value: /certs/tls.key + - name: GRPC_SERVER_TLS_CA_CERT + value: /certs/ca.crt + - name: CONFIG_GRPC_XDS_SERVER_USE_TLS + value: "true" + - name: CONFIG_GRPC_XDS_CLIENT_TLS_CERT + value: /certs/tls.crt + - name: CONFIG_GRPC_XDS_CLIENT_TLS_KEY + value: /certs/tls.key + - name: CONFIG_GRPC_XDS_SERVER_TLS_CACERT + value: /certs/ca.crt + - name: FORCE_START_WITHOUT_INITIAL_CONFIG + value: "true" + - name: REDIS_SOCKET_TYPE + value: tcp + - name: REDIS_URL + value: redis.redis.svc:6379 + - name: USE_PROMETHEUS + value: "true" + - name: PROMETHEUS_ADDR + value: :19001 + - name: PROMETHEUS_MAPPER_YAML + value: /etc/statsd-exporter/conf.yaml + image: docker.io/envoyproxy/ratelimit:master + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthcheck + port: 8080 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: envoy-ratelimit + ports: + - containerPort: 8081 + name: grpc + protocol: TCP + - containerPort: 19001 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 1 + httpGet: + path: /healthcheck + port: 8080 + scheme: HTTP + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 100m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + startupProbe: + failureThreshold: 30 + httpGet: + path: /healthcheck + port: 8080 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - mountPath: /etc/statsd-exporter + name: statsd-exporter-config + readOnly: true + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: envoy-ratelimit + terminationGracePeriodSeconds: 300 + volumes: + - name: certs + secret: + defaultMode: 420 + secretName: envoy-rate-limit + - configMap: + defaultMode: 420 + name: statsd-exporter-config + optional: true + name: statsd-exporter-config +status: {} diff --git a/release-notes/current/new_features/9627-replicas-managed-by-external-autoscaler.md b/release-notes/current/new_features/9627-replicas-managed-by-external-autoscaler.md new file mode 100644 index 0000000000..ba2e179848 --- /dev/null +++ b/release-notes/current/new_features/9627-replicas-managed-by-external-autoscaler.md @@ -0,0 +1 @@ +Added `replicasManagedByExternalAutoscaler` to the Envoy proxy `KubernetesDeploymentSpec`. When set to true, Envoy Gateway omits the `replicas` field from the generated Deployment so an external autoscaler (an external HorizontalPodAutoscaler, a KEDA ScaledObject, etc.) can own the replica count without Envoy Gateway reverting it on the next reconcile. Previously this was only possible when the built-in `envoyHpa` was configured. diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 12e60a35cc..c12c3171f7 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -3869,6 +3869,7 @@ _Appears in:_ | --- | --- | --- | --- | --- | | `patch` | _[KubernetesPatchSpec](#kubernetespatchspec)_ | false | | Patch defines how to perform the patch operation to deployment | | `replicas` | _integer_ | false | | Replicas is the number of desired pods. Defaults to 1. | +| `replicasManagedByExternalAutoscaler` | _boolean_ | false | | ReplicasManagedByExternalAutoscaler, when set to true, tells Envoy Gateway not to
render the replicas field on the generated Deployment, so an external autoscaler
(an external HorizontalPodAutoscaler, a KEDA ScaledObject, etc.) can own the replica
count without Envoy Gateway reverting it on the next reconcile. The Replicas field is
ignored when this is true. This has no additional effect when the built-in EnvoyHpa is
configured, which already omits the replicas field. | | `strategy` | _[DeploymentStrategy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#deploymentstrategy-v1-apps)_ | false | | The deployment strategy to use to replace existing pods with new ones. | | `pod` | _[KubernetesPodSpec](#kubernetespodspec)_ | false | | Pod defines the desired specification of pod. | | `container` | _[KubernetesContainerSpec](#kubernetescontainerspec)_ | false | | Container defines the desired specification of main container. | diff --git a/site/content/en/latest/tasks/operations/customize-envoyproxy.md b/site/content/en/latest/tasks/operations/customize-envoyproxy.md index 6548c32cc2..48ed34da4f 100644 --- a/site/content/en/latest/tasks/operations/customize-envoyproxy.md +++ b/site/content/en/latest/tasks/operations/customize-envoyproxy.md @@ -890,6 +890,79 @@ spec: After applying the config, the EnvoyProxy HPA (Horizontal Pod Autoscaler) is generated. However, upon activating the EnvoyProxy's HPA, the Envoy Gateway will no longer reference the `replicas` field specified in the `envoyDeployment`, as outlined [here](#customize-envoyproxy-deployment-replicas). The `replicas` field is omitted from the generated Deployment entirely, so that Envoy Gateway does not take ownership of it and revert the replica count computed by the HPA. Use `minReplicas` to control the lower bound of the replica count instead. +## Scale EnvoyProxy with an External Autoscaler + +The built-in `envoyHpa` only supports what a native `HorizontalPodAutoscaler` metrics spec can express. If you scale the proxy fleet with an external autoscaler that Envoy Gateway does not manage — for example a user-managed `HorizontalPodAutoscaler`, or a [KEDA](https://keda.sh/) `ScaledObject` scaling on a Prometheus query or another event source — set `replicasManagedByExternalAutoscaler: true` on the `envoyDeployment`. Envoy Gateway then omits the `replicas` field from the generated Deployment (exactly as it does for the built-in `envoyHpa`), so it does not take ownership of `spec.replicas` and will not revert the count computed by your autoscaler on the next reconcile. The `replicas` field is ignored while this is set. + +{{< tabpane text=true >}} +{{% tab header="Apply from stdin" %}} + +```shell +cat <}} + +### Targeting the generated Deployment + +An external autoscaler needs to reference the proxy Deployment. By default the Deployment name is autogenerated (a hashed name derived from the owning Gateway). You have two stable ways to target it: + +- **Pin the name**: set `envoyDeployment.name` to a fixed value and reference that from your autoscaler's `scaleTargetRef.name`, instead of recomputing the autogenerated name. +- **Select by label**: the generated Deployment always carries the stable owning-gateway labels `gateway.envoyproxy.io/owning-gateway-name`, `gateway.envoyproxy.io/owning-gateway-namespace`, and `gateway.envoyproxy.io/owning-gatewayclass`, which you can use where a label selector is accepted. + +A minimal KEDA example targeting a pinned Deployment name: + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: envoy-proxy + namespace: envoy-gateway-system +spec: + scaleTargetRef: + name: my-envoy-proxy # matches envoyDeployment.name + minReplicaCount: 2 + maxReplicaCount: 10 + triggers: + - type: prometheus + metadata: + serverAddress: http://prometheus.monitoring.svc:9090 + query: sum(rate(envoy_http_downstream_rq_total[1m])) + threshold: "100" +``` + ## Customize EnvoyProxy Command line options You can customize the EnvoyProxy Command line options via `spec.extraArgs` in EnvoyProxy Config. diff --git a/test/helm/gateway-crds-helm/all.out.yaml b/test/helm/gateway-crds-helm/all.out.yaml index 62f7f1e130..d5603b1d18 100644 --- a/test/helm/gateway-crds-helm/all.out.yaml +++ b/test/helm/gateway-crds-helm/all.out.yaml @@ -44178,6 +44178,15 @@ spec: to 1. format: int32 type: integer + replicasManagedByExternalAutoscaler: + description: |- + ReplicasManagedByExternalAutoscaler, when set to true, tells Envoy Gateway not to + render the replicas field on the generated Deployment, so an external autoscaler + (an external HorizontalPodAutoscaler, a KEDA ScaledObject, etc.) can own the replica + count without Envoy Gateway reverting it on the next reconcile. The Replicas field is + ignored when this is true. This has no additional effect when the built-in EnvoyHpa is + configured, which already omits the replicas field. + type: boolean strategy: description: The deployment strategy to use to replace existing pods with new ones. diff --git a/test/helm/gateway-crds-helm/e2e.out.yaml b/test/helm/gateway-crds-helm/e2e.out.yaml index 0120595e5c..ed029ba40d 100644 --- a/test/helm/gateway-crds-helm/e2e.out.yaml +++ b/test/helm/gateway-crds-helm/e2e.out.yaml @@ -20116,6 +20116,15 @@ spec: to 1. format: int32 type: integer + replicasManagedByExternalAutoscaler: + description: |- + ReplicasManagedByExternalAutoscaler, when set to true, tells Envoy Gateway not to + render the replicas field on the generated Deployment, so an external autoscaler + (an external HorizontalPodAutoscaler, a KEDA ScaledObject, etc.) can own the replica + count without Envoy Gateway reverting it on the next reconcile. The Replicas field is + ignored when this is true. This has no additional effect when the built-in EnvoyHpa is + configured, which already omits the replicas field. + type: boolean strategy: description: The deployment strategy to use to replace existing pods with new ones. diff --git a/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml b/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml index babc7aa51c..1fdaa81e05 100644 --- a/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml +++ b/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml @@ -20116,6 +20116,15 @@ spec: to 1. format: int32 type: integer + replicasManagedByExternalAutoscaler: + description: |- + ReplicasManagedByExternalAutoscaler, when set to true, tells Envoy Gateway not to + render the replicas field on the generated Deployment, so an external autoscaler + (an external HorizontalPodAutoscaler, a KEDA ScaledObject, etc.) can own the replica + count without Envoy Gateway reverting it on the next reconcile. The Replicas field is + ignored when this is true. This has no additional effect when the built-in EnvoyHpa is + configured, which already omits the replicas field. + type: boolean strategy: description: The deployment strategy to use to replace existing pods with new ones.