From 495f1af5141ea15acb40e9dee74d1e8a4c1e9f69 Mon Sep 17 00:00:00 2001 From: Michael Wain Date: Thu, 11 Jun 2026 09:21:42 +0100 Subject: [PATCH 1/2] api: add drainDelay to ShutdownConfig drainDelay defines how long the shutdown manager waits before failing healthchecks (delaying call to healthcheck/fail). The delay allows external load balancers time to deregister a terminating pod while still serving traffic. Signed-off-by: Michael Wain --- api/v1alpha1/envoyproxy_types.go | 5 +++++ api/v1alpha1/zz_generated.deepcopy.go | 5 +++++ .../generated/gateway.envoyproxy.io_envoyproxies.yaml | 6 ++++++ .../crds/generated/gateway.envoyproxy.io_envoyproxies.yaml | 6 ++++++ site/content/en/latest/api/extension_types.md | 1 + test/helm/gateway-crds-helm/all.out.yaml | 6 ++++++ test/helm/gateway-crds-helm/e2e.out.yaml | 6 ++++++ test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml | 6 ++++++ 8 files changed, 41 insertions(+) diff --git a/api/v1alpha1/envoyproxy_types.go b/api/v1alpha1/envoyproxy_types.go index a4ef2ee57a..27e20b93e5 100644 --- a/api/v1alpha1/envoyproxy_types.go +++ b/api/v1alpha1/envoyproxy_types.go @@ -489,6 +489,11 @@ type EnvoyProxyProvider struct { // ShutdownConfig defines configuration for graceful envoy shutdown process. type ShutdownConfig struct { + // DrainDelay defines the delay before starting the drain process. + // If unspecified, defaults to 0 seconds. + // + // +optional + DrainDelay *gwapiv1.Duration `json:"drainDelay,omitempty"` // DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds. // If unspecified, defaults to 60 seconds. // diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index ca269771b8..4f119ab08f 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -7935,6 +7935,11 @@ func (in *SessionResumption) DeepCopy() *SessionResumption { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ShutdownConfig) DeepCopyInto(out *ShutdownConfig) { *out = *in + if in.DrainDelay != nil { + in, out := &in.DrainDelay, &out.DrainDelay + *out = new(v1.Duration) + **out = **in + } if in.DrainTimeout != nil { in, out := &in.DrainTimeout, &out.DrainTimeout *out = new(v1.Duration) diff --git a/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml index 93db35107b..323e15b246 100644 --- a/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -11283,6 +11283,12 @@ spec: description: Shutdown defines configuration for graceful envoy shutdown process. properties: + drainDelay: + description: |- + DrainDelay defines the delay before starting the drain process. + If unspecified, defaults to 0 seconds. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string drainTimeout: description: |- DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds. diff --git a/charts/gateway-helm/charts/crds/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/charts/crds/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index eafaa9486a..c0aa103339 100644 --- a/charts/gateway-helm/charts/crds/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/charts/crds/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -11282,6 +11282,12 @@ spec: description: Shutdown defines configuration for graceful envoy shutdown process. properties: + drainDelay: + description: |- + DrainDelay defines the delay before starting the drain process. + If unspecified, defaults to 0 seconds. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string drainTimeout: description: |- DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds. diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 95dc07d4b2..f8eba64d33 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -5849,6 +5849,7 @@ _Appears in:_ | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | +| `drainDelay` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ | false | | DrainDelay defines the delay before starting the drain process.
If unspecified, defaults to 0 seconds. | | `drainTimeout` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ | false | | DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds.
If unspecified, defaults to 60 seconds. | | `minDrainDuration` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ | false | | MinDrainDuration defines the minimum drain duration allowing time for endpoint deprogramming to complete.
If unspecified, defaults to 10 seconds. | diff --git a/test/helm/gateway-crds-helm/all.out.yaml b/test/helm/gateway-crds-helm/all.out.yaml index b1c406e7af..da9e6c0a8c 100644 --- a/test/helm/gateway-crds-helm/all.out.yaml +++ b/test/helm/gateway-crds-helm/all.out.yaml @@ -42788,6 +42788,12 @@ spec: description: Shutdown defines configuration for graceful envoy shutdown process. properties: + drainDelay: + description: |- + DrainDelay defines the delay before starting the drain process. + If unspecified, defaults to 0 seconds. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string drainTimeout: description: |- DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds. diff --git a/test/helm/gateway-crds-helm/e2e.out.yaml b/test/helm/gateway-crds-helm/e2e.out.yaml index bd13c17ee3..be2776da20 100644 --- a/test/helm/gateway-crds-helm/e2e.out.yaml +++ b/test/helm/gateway-crds-helm/e2e.out.yaml @@ -20761,6 +20761,12 @@ spec: description: Shutdown defines configuration for graceful envoy shutdown process. properties: + drainDelay: + description: |- + DrainDelay defines the delay before starting the drain process. + If unspecified, defaults to 0 seconds. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string drainTimeout: description: |- DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds. diff --git a/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml b/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml index 8db76ab5eb..9da81ca740 100644 --- a/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml +++ b/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml @@ -20761,6 +20761,12 @@ spec: description: Shutdown defines configuration for graceful envoy shutdown process. properties: + drainDelay: + description: |- + DrainDelay defines the delay before starting the drain process. + If unspecified, defaults to 0 seconds. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string drainTimeout: description: |- DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds. From 78a96baaddfea543214ba396e079453c8fd25ae5 Mon Sep 17 00:00:00 2001 From: Michael Wain Date: Thu, 11 Jun 2026 12:18:55 +0100 Subject: [PATCH 2/2] feat: add drain delay to graceful shutdown process When a pod terminates, the shutdown-manager immediately calls `healthcheck/fail` to fail health checks. This causes the readiness probe to fail and removes the pod from service endpoints. External load balancers that use health checks for deregistration need several failed probes before they stop sending new connections to the pod. On GKE, the L4 passthrough load balancer probes every 3 seconds and needs 2 failures (at the time of writing), so deregistration takes 6 seconds or more. The pod readiness probe fails faster than that, so for a few seconds the load balancer keeps sending connections that kube-proxy no longer routes. Kubernetes has a mechanism for this window. When a service has no other ready endpoints, kube-proxy keeps routing to terminating pods that still pass their readiness probe (KEP-1669). This only works if the pod keeps passing its probe while the load balancer deregisters it. Calling `healthcheck/fail` at the start of shutdown breaks that. This adds a drainDelay field to ShutdownConfig which delays the `healthcheck/fail` call. During the delay the pod stays ready and keeps serving, so the load balancer has time to deregister it. The ready-timeout and termination grace period are extended by the delay. Signed-off-by: Michael Wain --- api/v1alpha1/envoyproxy_helpers.go | 5 ++ internal/cmd/envoy.go | 9 +++- internal/cmd/envoy/shutdown_manager.go | 10 +++- internal/infrastructure/common/proxy_args.go | 2 +- .../kubernetes/proxy/resource.go | 33 ++++++++++-- .../kubernetes/proxy/resource_provider.go | 9 ++++ .../proxy/resource_provider_test.go | 50 +++++++++++++++++++ .../kubernetes/proxy/resource_test.go | 48 ++++++++++++++++++ .../deployments/shutdown-manager.yaml | 5 +- release-notes/current.yaml | 1 + 10 files changed, 161 insertions(+), 11 deletions(-) diff --git a/api/v1alpha1/envoyproxy_helpers.go b/api/v1alpha1/envoyproxy_helpers.go index 33111e1229..585092da18 100644 --- a/api/v1alpha1/envoyproxy_helpers.go +++ b/api/v1alpha1/envoyproxy_helpers.go @@ -9,12 +9,17 @@ import ( "fmt" "sort" "strings" + "time" autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" ) +// DefaultDrainTimeout is the default drain timeout for the graceful drain +// sequence, used when ShutdownConfig.DrainTimeout is not specified. +const DefaultDrainTimeout = 60 * time.Second + // DefaultEnvoyProxyProvider returns a new EnvoyProxyProvider with default settings. func DefaultEnvoyProxyProvider() *EnvoyProxyProvider { return &EnvoyProxyProvider{ diff --git a/internal/cmd/envoy.go b/internal/cmd/envoy.go index 8c2d0f46ac..53602313a3 100644 --- a/internal/cmd/envoy.go +++ b/internal/cmd/envoy.go @@ -10,6 +10,7 @@ import ( "github.com/spf13/cobra" + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/cmd/envoy" ) @@ -28,6 +29,7 @@ func GetEnvoyCommand() *cobra.Command { // getShutdownCommand returns the shutdown cobra command to be executed. func getShutdownCommand() *cobra.Command { + var drainDelay time.Duration var drainTimeout time.Duration var minDrainDuration time.Duration var exitAtConnections int @@ -36,11 +38,14 @@ func getShutdownCommand() *cobra.Command { Use: "shutdown", Short: "Gracefully drain open connections prior to pod shutdown.", RunE: func(_ *cobra.Command, _ []string) error { - return envoy.Shutdown(drainTimeout, minDrainDuration, exitAtConnections) + return envoy.Shutdown(drainDelay, drainTimeout, minDrainDuration, exitAtConnections) }, } - cmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", 60*time.Second, + cmd.PersistentFlags().DurationVar(&drainDelay, "drain-delay", 0*time.Second, + "Delay before starting the drain process.") + + cmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", egv1a1.DefaultDrainTimeout, "Graceful shutdown timeout. This should be less than the pod's terminationGracePeriodSeconds.") cmd.PersistentFlags().DurationVar(&minDrainDuration, "min-drain-duration", 10*time.Second, diff --git a/internal/cmd/envoy/shutdown_manager.go b/internal/cmd/envoy/shutdown_manager.go index cd56853392..e50533f530 100644 --- a/internal/cmd/envoy/shutdown_manager.go +++ b/internal/cmd/envoy/shutdown_manager.go @@ -118,8 +118,7 @@ func shutdownReadyHandler(w http.ResponseWriter, readyTimeout time.Duration, rea // Shutdown is called from a preStop hook on the shutdown-manager container where // it will initiate a drain sequence on the Envoy proxy and block until // connections are drained or a timeout is exceeded. -func Shutdown(drainTimeout, minDrainDuration time.Duration, exitAtConnections int) error { - startTime := time.Now() +func Shutdown(drainDelay, drainTimeout, minDrainDuration time.Duration, exitAtConnections int) error { allowedToExit := false // Reconfigure logger to write to stdout of main process if running in Kubernetes @@ -127,6 +126,13 @@ func Shutdown(drainTimeout, minDrainDuration time.Duration, exitAtConnections in logger = logging.FileLogger("/proc/1/fd/1", "shutdown-manager", egv1a1.LogLevelInfo) } + if drainDelay > 0 { + logger.Info(fmt.Sprintf("waiting %.0f seconds before starting drain", drainDelay.Seconds())) + time.Sleep(drainDelay) + } + + startTime := time.Now() + logger.Info(fmt.Sprintf("initiating drain with %.0f second minimum drain period and %.0f second timeout", minDrainDuration.Seconds(), drainTimeout.Seconds())) diff --git a/internal/infrastructure/common/proxy_args.go b/internal/infrastructure/common/proxy_args.go index 6c3ec688ca..5e9a11a1d8 100644 --- a/internal/infrastructure/common/proxy_args.go +++ b/internal/infrastructure/common/proxy_args.go @@ -84,7 +84,7 @@ func BuildProxyArgs( } // Default drain timeout. - drainTimeout := 60.0 + drainTimeout := egv1a1.DefaultDrainTimeout.Seconds() if shutdownConfig != nil && shutdownConfig.DrainTimeout != nil { d, err := time.ParseDuration(string(*shutdownConfig.DrainTimeout)) if err != nil { diff --git a/internal/infrastructure/kubernetes/proxy/resource.go b/internal/infrastructure/kubernetes/proxy/resource.go index 40a39d4e91..b6940059b7 100644 --- a/internal/infrastructure/kubernetes/proxy/resource.go +++ b/internal/infrastructure/kubernetes/proxy/resource.go @@ -266,15 +266,32 @@ func expectedShutdownManagerImage(shutdownManager *egv1a1.ShutdownManager) strin } func expectedShutdownManagerArgs(cfg *egv1a1.ShutdownConfig) []string { - args := []string{"envoy", "shutdown-manager"} - if cfg != nil && cfg.DrainTimeout != nil { + args := make([]string, 0, 3) + args = append(args, "envoy", "shutdown-manager") + + // Use default --ready-timeout when neither drain field is configured. + if cfg == nil || (cfg.DrainTimeout == nil && cfg.DrainDelay == nil) { + return args + } + + readyTimeout := egv1a1.DefaultDrainTimeout + if cfg.DrainTimeout != nil { d, err := time.ParseDuration(string(*cfg.DrainTimeout)) if err != nil { return nil } - args = append(args, fmt.Sprintf("--ready-timeout=%.0fs", d.Seconds()+10)) + readyTimeout = d } - return args + + if cfg.DrainDelay != nil { + delay, err := time.ParseDuration(string(*cfg.DrainDelay)) + if err != nil { + return nil + } + readyTimeout += delay + } + + return append(args, fmt.Sprintf("--ready-timeout=%.0fs", readyTimeout.Seconds()+10)) } func expectedShutdownPreStopCommand(cfg *egv1a1.ShutdownConfig) []string { @@ -284,6 +301,14 @@ func expectedShutdownPreStopCommand(cfg *egv1a1.ShutdownConfig) []string { return command } + if cfg.DrainDelay != nil { + d, err := time.ParseDuration(string(*cfg.DrainDelay)) + if err != nil { + return nil + } + command = append(command, fmt.Sprintf("--drain-delay=%.0fs", d.Seconds())) + } + if cfg.DrainTimeout != nil { d, err := time.ParseDuration(string(*cfg.DrainTimeout)) if err != nil { diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider.go b/internal/infrastructure/kubernetes/proxy/resource_provider.go index b8635d89b3..2a5aa85ea7 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_provider.go +++ b/internal/infrastructure/kubernetes/proxy/resource_provider.go @@ -620,6 +620,15 @@ func expectedTerminationGracePeriodSeconds(cfg *egv1a1.ShutdownConfig) *int64 { } s = int(d.Seconds() + 300) // 5 minutes longer than drain timeout } + + if cfg != nil && cfg.DrainDelay != nil { + d, err := time.ParseDuration(string(*cfg.DrainDelay)) + if err != nil { + return nil + } + s += int(d.Seconds()) + } + return new(int64(s)) } diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go index 9b8275bcda..f1e248f7b6 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go +++ b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go @@ -292,6 +292,7 @@ func TestDeployment(t *testing.T) { }, }, shutdown: &egv1a1.ShutdownConfig{ + DrainDelay: new(gwapiv1.Duration("15s")), DrainTimeout: new(gwapiv1.Duration("30s")), MinDrainDuration: new(gwapiv1.Duration("15s")), }, @@ -2120,3 +2121,52 @@ func writeTestDataToFile(filename string, resources []any) error { return os.WriteFile(filename, combinedYAML, 0o600) } + +func TestExpectedTerminationGracePeriodSeconds(t *testing.T) { + tests := []struct { + name string + cfg *egv1a1.ShutdownConfig + expected int64 + }{ + { + name: "nil config", + cfg: nil, + expected: 360, + }, + { + name: "empty config", + cfg: &egv1a1.ShutdownConfig{}, + expected: 360, + }, + { + name: "only drainTimeout", + cfg: &egv1a1.ShutdownConfig{ + DrainTimeout: new(gwapiv1.Duration("30s")), + }, + expected: 330, + }, + { + name: "only drainDelay", + cfg: &egv1a1.ShutdownConfig{ + DrainDelay: new(gwapiv1.Duration("15s")), + }, + expected: 375, + }, + { + name: "drainDelay and drainTimeout", + cfg: &egv1a1.ShutdownConfig{ + DrainDelay: new(gwapiv1.Duration("15s")), + DrainTimeout: new(gwapiv1.Duration("30s")), + }, + expected: 345, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := expectedTerminationGracePeriodSeconds(tt.cfg) + require.NotNil(t, got) + require.Equal(t, tt.expected, *got) + }) + } +} diff --git a/internal/infrastructure/kubernetes/proxy/resource_test.go b/internal/infrastructure/kubernetes/proxy/resource_test.go index 2428311495..2742ce272b 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_test.go +++ b/internal/infrastructure/kubernetes/proxy/resource_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/resource" @@ -182,3 +183,50 @@ func TestGetImageTag(t *testing.T) { }) } } + +func TestExpectedShutdownManagerArgs(t *testing.T) { + tests := []struct { + name string + cfg *egv1a1.ShutdownConfig + expected []string + }{ + { + name: "nil config", + cfg: nil, + expected: []string{"envoy", "shutdown-manager"}, + }, + { + name: "empty config", + cfg: &egv1a1.ShutdownConfig{}, + expected: []string{"envoy", "shutdown-manager"}, + }, + { + name: "only drainTimeout", + cfg: &egv1a1.ShutdownConfig{ + DrainTimeout: new(gwapiv1.Duration("30s")), + }, + expected: []string{"envoy", "shutdown-manager", "--ready-timeout=40s"}, + }, + { + name: "only drainDelay", + cfg: &egv1a1.ShutdownConfig{ + DrainDelay: new(gwapiv1.Duration("15s")), + }, + expected: []string{"envoy", "shutdown-manager", "--ready-timeout=85s"}, + }, + { + name: "drainDelay and drainTimeout", + cfg: &egv1a1.ShutdownConfig{ + DrainDelay: new(gwapiv1.Duration("15s")), + DrainTimeout: new(gwapiv1.Duration("30s")), + }, + expected: []string{"envoy", "shutdown-manager", "--ready-timeout=55s"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.expected, expectedShutdownManagerArgs(tt.cfg)) + }) + } +} diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/shutdown-manager.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/shutdown-manager.yaml index be75b9f274..89d928a885 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/shutdown-manager.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/shutdown-manager.yaml @@ -309,7 +309,7 @@ spec: - args: - envoy - shutdown-manager - - --ready-timeout=40s + - --ready-timeout=55s command: - envoy-gateway env: @@ -341,6 +341,7 @@ spec: - envoy-gateway - envoy - shutdown + - --drain-delay=15s - --drain-timeout=30s - --min-drain-duration=15s livenessProbe: @@ -395,7 +396,7 @@ spec: restartPolicy: Always schedulerName: default-scheduler serviceAccountName: envoy-default-37a8eec1 - terminationGracePeriodSeconds: 330 + terminationGracePeriodSeconds: 345 volumes: - name: certs secret: diff --git a/release-notes/current.yaml b/release-notes/current.yaml index 98a85836b4..41ce35d9c7 100644 --- a/release-notes/current.yaml +++ b/release-notes/current.yaml @@ -15,6 +15,7 @@ new features: | Added a `requestBody` field to the HTTP active health checker in `BackendTrafficPolicy`, allowing a request body payload to be sent during HTTP health checking. The field requires the health check `method` to be `POST` or `PUT`. Added a `xdsNACKTotal` metric to track the number of NACKs received from Envoy, labeled by node ID and resource type URL. A NACK is a DiscoveryRequest carrying an ErrorDetail, indicating that Envoy rejected the last config update. This metric can be used to alert on config issues causing xDS rejections. Added the `autoSNIFromEndpointHostname` TLS setting to Backends, allowing the SNI value sent to the backend to be automatically derived from the backend endpoint hostname instead of using a fixed SNI value. + Added a `drainDelay` field to `ShutdownConfig`, delaying the start of the Envoy proxy drain sequence on pod termination. During the delay the proxy continues to accept and serve new connections, allowing external load balancers that rely on health-check based de-registration (e.g. L4 passthrough load balancers) to stop sending traffic to the terminating pod before it stops serving. bug fixes: | Fixed HTTPRoute, GRPCRoute, TLSRoute, TCPRoute, and UDPRoute Accepted condition being set to False when an attached listener is not programmed due to a missing TLS certificate ref; listener programmed state is now correctly separated from route acceptance.