vmoperator: set default -http.shutdownDelay for HTTP-serving components

internal/controller/operator/factory/build/container.go

@@ -216,6 +216,28 @@ func AddExtraArgsOverrideDefaults(args []string, extraArgs map[string]string, da
 	return args
 }
 
+const (
+	// DefaultTerminationGracePeriodSeconds is the default termination grace period for all components
+	DefaultTerminationGracePeriodSeconds int64 = 30
+)
+
+// AddHTTPShutdownDelayArg adds default -http.shutdownDelay flag if user didn't override it in extraArgs.
+// The delay is derived from terminationGracePeriodSeconds.
+// It is added only for new resources
+func AddHTTPShutdownDelayArg(args []string, extraArgs map[string]string, terminationGracePeriodSeconds *int64) []string {
+	if _, ok := extraArgs["http.shutdownDelay"]; ok {
+		return args
+	}
+
+	delaySeconds := DefaultTerminationGracePeriodSeconds
+	if terminationGracePeriodSeconds != nil {
+		delaySeconds = *terminationGracePeriodSeconds
+	}
+
+	args = append(args, fmt.Sprintf("-http.shutdownDelay=%ds", delaySeconds))
+	return args
+}
+
 // formatContainerImage returns container image with registry prefix if needed.
 func formatContainerImage(registry string, containerImage string) string {
 	if registry == "" {

internal/controller/operator/factory/build/container_test.go

@@ -9,6 +9,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/utils/ptr"
 
 	vmv1 "github.com/VictoriaMetrics/operator/api/operator/v1"
 	vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1"
@@ -205,6 +206,45 @@ func Test_addExtraArgsOverrideDefaults(t *testing.T) {
 	})
 }
 
+func TestAddHTTPShutdownDelayArg(t *testing.T) {
+	type opts struct {
+		extraArgs                     map[string]string
+		terminationGracePeriodSeconds *int64
+		wantArgs                      []string
+	}
+	f := func(opts opts) {
+		t.Helper()
+		assert.Equal(
+			t,
+			AddHTTPShutdownDelayArg(nil, opts.extraArgs, opts.terminationGracePeriodSeconds),
+			opts.wantArgs,
+		)
+	}
+	// new resource, no explicit settings
+	f(opts{
+		extraArgs:                     nil,
+		terminationGracePeriodSeconds: nil,
+		wantArgs:                      []string{"-http.shutdownDelay=30s"},
+	})
+	// if extraArg already exists, no more args should be added, it will be added later in the process of adding extra args
+	f(opts{
+		extraArgs:                     map[string]string{"http.shutdownDelay": "5s"},
+		terminationGracePeriodSeconds: nil,
+		wantArgs:                      nil,
+	})
+	// new resource with explicit terminationGracePeriodSeconds
+	f(opts{
+		extraArgs:                     nil,
+		terminationGracePeriodSeconds: ptr.To[int64](60),
+		wantArgs:                      []string{"-http.shutdownDelay=60s"},
+	})
+	f(opts{
+		extraArgs:                     map[string]string{"http.shutdownDelay": "20s"},
+		terminationGracePeriodSeconds: ptr.To[int64](120),
+		wantArgs:                      nil,
+	})
+}
+
 func TestFormatContainerImage(t *testing.T) {
 	f := func(globalRepo, image, wantImage string) {
 		t.Helper()

internal/controller/operator/factory/build/daemonset.go

@@ -23,10 +23,12 @@ func DaemonSetAddCommonParams(dst *appsv1.DaemonSet, useStrictSecurity bool, par
 	dst.Spec.Template.Spec.DNSConfig = params.DNSConfig
 	dst.Spec.Template.Spec.NodeSelector = params.NodeSelector
 	dst.Spec.Template.Spec.SecurityContext = AddStrictSecuritySettingsWithRootToPod(params.SecurityContext, useStrictSecurity)
+	if params.TerminationGracePeriodSeconds == nil {
+		params.TerminationGracePeriodSeconds = ptr.To(DefaultTerminationGracePeriodSeconds)
+	}
 	dst.Spec.Template.Spec.TerminationGracePeriodSeconds = params.TerminationGracePeriodSeconds
 	dst.Spec.Template.Spec.TopologySpreadConstraints = params.TopologySpreadConstraints
 	dst.Spec.Template.Spec.ImagePullSecrets = params.ImagePullSecrets
-	dst.Spec.Template.Spec.TerminationGracePeriodSeconds = params.TerminationGracePeriodSeconds
 	dst.Spec.Template.Spec.ReadinessGates = params.ReadinessGates
 	dst.Spec.MinReadySeconds = params.MinReadySeconds
 	dst.Spec.RevisionHistoryLimit = params.RevisionHistoryLimitCount

internal/controller/operator/factory/build/defaults.go

@@ -347,8 +347,6 @@ const (
 	vmStorageDefaultDBPath = "vmstorage-data"
 )
 
-var defaultTerminationGracePeriod = int64(30)
-
 func addVMClusterSpecDefaults(spec *vmv1beta1.VMClusterSpec) {
 	c := getCfg()
 
@@ -409,9 +407,6 @@ func addVMClusterSpecDefaults(spec *vmv1beta1.VMClusterSpec) {
 		if spec.VMStorage.StorageDataPath == "" {
 			spec.VMStorage.StorageDataPath = vmStorageDefaultDBPath
 		}
-		if spec.VMStorage.TerminationGracePeriodSeconds == nil {
-			spec.VMStorage.TerminationGracePeriodSeconds = &defaultTerminationGracePeriod
-		}
 		if spec.VMStorage.UseDefaultResources == nil {
 			spec.VMStorage.UseDefaultResources = &c.VMCluster.UseDefaultResources
 		}

internal/controller/operator/factory/build/deployment.go

@@ -23,10 +23,12 @@ func DeploymentAddCommonParams(dst *appsv1.Deployment, useStrictSecurity bool, p
 	dst.Spec.Template.Spec.DNSConfig = params.DNSConfig
 	dst.Spec.Template.Spec.NodeSelector = params.NodeSelector
 	dst.Spec.Template.Spec.SecurityContext = AddStrictSecuritySettingsToPod(params.SecurityContext, useStrictSecurity)
+	if params.TerminationGracePeriodSeconds == nil {
+		params.TerminationGracePeriodSeconds = ptr.To(DefaultTerminationGracePeriodSeconds)
+	}
 	dst.Spec.Template.Spec.TerminationGracePeriodSeconds = params.TerminationGracePeriodSeconds
 	dst.Spec.Template.Spec.TopologySpreadConstraints = params.TopologySpreadConstraints
 	dst.Spec.Template.Spec.ImagePullSecrets = params.ImagePullSecrets
-	dst.Spec.Template.Spec.TerminationGracePeriodSeconds = params.TerminationGracePeriodSeconds
 	dst.Spec.Template.Spec.ReadinessGates = params.ReadinessGates
 	dst.Spec.MinReadySeconds = params.MinReadySeconds
 	dst.Spec.Replicas = params.ReplicaCount

internal/controller/operator/factory/build/statefulset.go

@@ -23,10 +23,12 @@ func StatefulSetAddCommonParams(dst *appsv1.StatefulSet, useStrictSecurity bool,
 	dst.Spec.Template.Spec.DNSConfig = params.DNSConfig
 	dst.Spec.Template.Spec.NodeSelector = params.NodeSelector
 	dst.Spec.Template.Spec.SecurityContext = AddStrictSecuritySettingsToPod(params.SecurityContext, useStrictSecurity)
+	if params.TerminationGracePeriodSeconds == nil {
+		params.TerminationGracePeriodSeconds = ptr.To(DefaultTerminationGracePeriodSeconds)
+	}
 	dst.Spec.Template.Spec.TerminationGracePeriodSeconds = params.TerminationGracePeriodSeconds
 	dst.Spec.Template.Spec.TopologySpreadConstraints = params.TopologySpreadConstraints
 	dst.Spec.Template.Spec.ImagePullSecrets = params.ImagePullSecrets
-	dst.Spec.Template.Spec.TerminationGracePeriodSeconds = params.TerminationGracePeriodSeconds
 	dst.Spec.Template.Spec.ReadinessGates = params.ReadinessGates
 	dst.Spec.MinReadySeconds = params.MinReadySeconds
 	dst.Spec.Replicas = params.ReplicaCount

internal/controller/operator/factory/vlagent/vlagent.go

@@ -425,6 +425,7 @@ func newPodSpec(cr *vmv1.VLAgent) (*corev1.PodSpec, error) {
 		agentVolumeMounts = append(agentVolumeMounts, cvm)
 	}
 	volumes, agentVolumeMounts = addRemoteWriteAssetsToVolumes(volumes, agentVolumeMounts, cr)
+	args = build.AddHTTPShutdownDelayArg(args, cr.Spec.ExtraArgs, cr.Spec.TerminationGracePeriodSeconds)
 	args = build.AddExtraArgsOverrideDefaults(args, cr.Spec.ExtraArgs, "-")
 	sort.Strings(args)
 

internal/controller/operator/factory/vlagent/vlagent_test.go

@@ -294,6 +294,34 @@ func TestCreateOrUpdate(t *testing.T) {
 			},
 		},
 	})
+
+	// test custom terminationGracePeriodSeconds is propagated to pod spec and shutdownDelay
+	f(opts{
+		cr: &vmv1.VLAgent{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "example-agent-grace",
+				Namespace: "default",
+			},
+			Spec: vmv1.VLAgentSpec{
+				RemoteWrite: []vmv1.VLAgentRemoteWriteSpec{
+					{URL: "http://remote-write"},
+				},
+				CommonApplicationDeploymentParams: vmv1beta1.CommonApplicationDeploymentParams{
+					ReplicaCount:                  ptr.To(int32(1)),
+					TerminationGracePeriodSeconds: ptr.To[int64](60),
+				},
+			},
+		},
+		validate: func(got *appsv1.StatefulSet) {
+			// terminationGracePeriodSeconds should be set on the pod spec
+			assert.NotNil(t, got.Spec.Template.Spec.TerminationGracePeriodSeconds)
+			assert.Equal(t, int64(60), *got.Spec.Template.Spec.TerminationGracePeriodSeconds)
+			// http.shutdownDelay should inherit from terminationGracePeriodSeconds
+			cnt := got.Spec.Template.Spec.Containers[0]
+			assert.Contains(t, cnt.Args, "-http.shutdownDelay=60s")
+		},
+	})
+
 }
 
 func TestBuildRemoteWriteArgs(t *testing.T) {
@@ -775,6 +803,7 @@ containers:
   - name: vlagent
     image: vm-repo:v1.97.1
     args:
+      - -http.shutdownDelay=30s
       - -httpListenAddr=:9425
       - -tmpDataPath=/vlagent-data
     ports:
@@ -839,6 +868,7 @@ containers:
   - name: vlagent
     image: victoriametrics/vlagent:v1.97.1
     args:
+      - -http.shutdownDelay=30s
       - -httpListenAddr=:9429
       - -tmpDataPath=/vlagent-data
     ports:
@@ -905,6 +935,7 @@ containers:
   - name: vlagent
     image: victoriametrics/vlagent:v1.97.1
     args:
+      - -http.shutdownDelay=30s
       - -httpListenAddr=:9425
       - -remoteWrite.maxDiskUsagePerURL=10GB,10GB,
       - -remoteWrite.url=http://some-url/api/v1/write,http://some-url-2/api/v1/write,http://some-url-3/api/v1/write
@@ -978,6 +1009,7 @@ containers:
   - name: vlagent
     image: victoriametrics/vlagent:v1.47.0
     args:
+      - -http.shutdownDelay=30s
       - -httpListenAddr=:9425
       - -kubernetesCollector
       - -kubernetesCollector.includePodLabels
@@ -1072,6 +1104,7 @@ containers:
   - name: vlagent
     image: victoriametrics/vlagent:v1.97.1
     args:
+      - -http.shutdownDelay=30s
       - -httpListenAddr=:9425
       - -remoteWrite.maxDiskUsagePerURL=10GB,20MB,10GB
       - -remoteWrite.url=http://some-url/api/v1/write,http://some-url-2/api/v1/write,http://some-url-3/api/v1/write
@@ -1148,6 +1181,7 @@ containers:
   - name: vlagent
     image: victoriametrics/vlagent:v0.0.1
     args:
+      - -http.shutdownDelay=30s
       - -httpListenAddr=:9425
       - -remoteWrite.maxDiskUsagePerURL=35GiB
       - -remoteWrite.url=http://some-url/api/v1/write,http://some-url-2/api/v1/write,http://some-url-3/api/v1/write
@@ -1186,5 +1220,61 @@ containers:
 serviceaccountname: vlagent-agent
 
     `)
+	// test custom terminationGrace probe affects both readiness and shutdownDelay
+	f(&vmv1.VLAgent{
+		ObjectMeta: metav1.ObjectMeta{Name: "agent", Namespace: "default"},
+		Spec: vmv1.VLAgentSpec{
+			CommonDefaultableParams: vmv1beta1.CommonDefaultableParams{
+				Image: vmv1beta1.Image{
+					Tag: "v1.97.1",
+				},
+				UseDefaultResources: ptr.To(false),
+				Port:                "9429",
+			},
+			CommonApplicationDeploymentParams: vmv1beta1.CommonApplicationDeploymentParams{
+				TerminationGracePeriodSeconds: ptr.To[int64](40),
+			},
+		},
+	}, []runtime.Object{}, `
+containers:
+  - name: vlagent
+    image: victoriametrics/vlagent:v1.97.1
+    args:
+      - -http.shutdownDelay=40s
+      - -httpListenAddr=:9429
+      - -tmpDataPath=/vlagent-data
+    ports:
+      - name: http
+        containerport: 9429
+        protocol: TCP
+    volumemounts:
+      - name: tmp-data
+        mountpath: /vlagent-data
+    livenessprobe:
+      probehandler:
+        httpget:
+          path: /health
+          port:
+            intval: 9429
+          scheme: HTTP
+      timeoutseconds: 5
+      periodseconds: 5
+      successthreshold: 1
+      failurethreshold: 10
+    readinessprobe:
+      probehandler:
+        httpget:
+          path: /health
+          port:
+            intval: 9429
+          scheme: HTTP
+      timeoutseconds: 5
+      periodseconds: 5
+      successthreshold: 1
+      failurethreshold: 10
+    terminationmessagepolicy: FallbackToLogsOnError
+    imagepullpolicy: IfNotPresent
+serviceaccountname: vlagent-agent
+`)
 
 }

internal/controller/operator/factory/vlcluster/vlcluster_test.go

@@ -100,15 +100,15 @@ func TestCreateOrUpdate(t *testing.T) {
 			assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentInsert), Namespace: cr.Namespace}, &dep))
 			assert.Len(t, dep.Spec.Template.Spec.Containers, 1)
 			cnt := dep.Spec.Template.Spec.Containers[0]
-			assert.Equal(t, cnt.Args, []string{"-httpListenAddr=:9481", "-internalselect.disable=true", "-storageNode=vlstorage-base-0.vlstorage-base.default:9491,vlstorage-base-1.vlstorage-base.default:9491"})
+			assert.Equal(t, cnt.Args, []string{"-http.shutdownDelay=30s", "-httpListenAddr=:9481", "-internalselect.disable=true", "-storageNode=vlstorage-base-0.vlstorage-base.default:9491,vlstorage-base-1.vlstorage-base.default:9491"})
 			assert.Nil(t, dep.Annotations)
 			assert.Equal(t, dep.Labels, cr.FinalLabels(vmv1beta1.ClusterComponentInsert))
 
 			// check select
 			assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect), Namespace: cr.Namespace}, &dep))
 			assert.Len(t, dep.Spec.Template.Spec.Containers, 1)
 			cnt = dep.Spec.Template.Spec.Containers[0]
-			assert.Equal(t, cnt.Args, []string{"-httpListenAddr=:9471", "-internalinsert.disable=true", "-storageNode=vlstorage-base-0.vlstorage-base.default:9491,vlstorage-base-1.vlstorage-base.default:9491"})
+			assert.Equal(t, cnt.Args, []string{"-http.shutdownDelay=30s", "-httpListenAddr=:9471", "-internalinsert.disable=true", "-storageNode=vlstorage-base-0.vlstorage-base.default:9491,vlstorage-base-1.vlstorage-base.default:9491"})
 			assert.Nil(t, dep.Annotations)
 			assert.Equal(t, dep.Labels, cr.FinalLabels(vmv1beta1.ClusterComponentSelect))
 
@@ -117,7 +117,7 @@ func TestCreateOrUpdate(t *testing.T) {
 			assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage), Namespace: cr.Namespace}, &sts))
 			assert.Len(t, sts.Spec.Template.Spec.Containers, 1)
 			cnt = sts.Spec.Template.Spec.Containers[0]
-			assert.Equal(t, cnt.Args, []string{"-httpListenAddr=:9491", "-storageDataPath=/vlstorage-data"})
+			assert.Equal(t, cnt.Args, []string{"-http.shutdownDelay=30s", "-httpListenAddr=:9491", "-storageDataPath=/vlstorage-data"})
 			assert.Nil(t, sts.Annotations)
 			assert.Equal(t, sts.Labels, cr.FinalLabels(vmv1beta1.ClusterComponentStorage))
 		},
@@ -147,7 +147,7 @@ func TestCreateOrUpdate(t *testing.T) {
 			assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage), Namespace: cr.Namespace}, &sts))
 			assert.Len(t, sts.Spec.Template.Spec.Containers, 1)
 			cnt := sts.Spec.Template.Spec.Containers[0]
-			assert.Equal(t, cnt.Args, []string{"-futureRetention=2d", "-httpListenAddr=:9491", "-retention.maxDiskSpaceUsageBytes=5GB", "-retentionPeriod=1w", "-storageDataPath=/vlstorage-data"})
+			assert.Equal(t, cnt.Args, []string{"-futureRetention=2d", "-http.shutdownDelay=30s", "-httpListenAddr=:9491", "-retention.maxDiskSpaceUsageBytes=5GB", "-retentionPeriod=1w", "-storageDataPath=/vlstorage-data"})
 		},
 	})
 
@@ -184,6 +184,7 @@ func TestCreateOrUpdate(t *testing.T) {
 			assert.Len(t, d.Spec.Template.Spec.Containers, 1)
 			cnt := d.Spec.Template.Spec.Containers[0]
 			assert.Equal(t, cnt.Args, []string{
+				"-http.shutdownDelay=30s",
 				"-httpListenAddr=:9471",
 				"-internalinsert.disable=true",
 				"-storageNode=vlstorage-read-only-0.vlstorage-read-only.default:9491,localhost:10101",

internal/controller/operator/factory/vlcluster/vlinsert.go

@@ -206,6 +206,7 @@ func buildVLInsertPodSpec(cr *vmv1.VLCluster) (*corev1.PodTemplateSpec, error) {
 		})
 	}
 
+	args = build.AddHTTPShutdownDelayArg(args, cr.Spec.VLInsert.ExtraArgs, cr.Spec.VLInsert.TerminationGracePeriodSeconds)
 	args = build.AddExtraArgsOverrideDefaults(args, cr.Spec.VLInsert.ExtraArgs, "-")
 	sort.Strings(args)
 

internal/controller/operator/factory/vlcluster/vlselect.go

@@ -307,6 +307,7 @@ func buildVLSelectPodSpec(cr *vmv1.VLCluster) (*corev1.PodTemplateSpec, error) {
 		})
 	}
 
+	args = build.AddHTTPShutdownDelayArg(args, cr.Spec.VLSelect.ExtraArgs, cr.Spec.VLSelect.TerminationGracePeriodSeconds)
 	args = build.AddExtraArgsOverrideDefaults(args, cr.Spec.VLSelect.ExtraArgs, "-")
 	sort.Strings(args)
 	selectContainers := corev1.Container{

internal/controller/operator/factory/vlcluster/vlstorage.go

@@ -309,6 +309,7 @@ func buildVLStoragePodSpec(cr *vmv1.VLCluster) (*corev1.PodTemplateSpec, error)
 		})
 	}
 
+	args = build.AddHTTPShutdownDelayArg(args, cr.Spec.VLStorage.ExtraArgs, cr.Spec.VLStorage.TerminationGracePeriodSeconds)
 	args = build.AddExtraArgsOverrideDefaults(args, cr.Spec.VLStorage.ExtraArgs, "-")
 	sort.Strings(args)
 	vmstorageContainer := corev1.Container{

internal/controller/operator/factory/vlcluster/vmauth_lb.go

@@ -161,6 +161,7 @@ func buildVMauthLBDeployment(cr *vmv1.VLCluster) (*appsv1.Deployment, error) {
 	volumes, vmMounts = build.LicenseVolumeTo(volumes, vmMounts, cr.Spec.License, vmv1beta1.SecretsDir)
 	args = build.LicenseArgsTo(args, cr.Spec.License, vmv1beta1.SecretsDir)
 
+	args = build.AddHTTPShutdownDelayArg(args, spec.ExtraArgs, spec.TerminationGracePeriodSeconds)
 	args = build.AddExtraArgsOverrideDefaults(args, spec.ExtraArgs, "-")
 	sort.Strings(args)
 	vmauthLBCnt := corev1.Container{

internal/controller/operator/factory/vlsingle/vlsingle.go

@@ -235,6 +235,7 @@ func makePodSpec(r *vmv1.VLSingle) (*corev1.PodTemplateSpec, error) {
 	volumes, vmMounts = build.LicenseVolumeTo(volumes, vmMounts, r.Spec.License, vmv1beta1.SecretsDir)
 	args = build.LicenseArgsTo(args, r.Spec.License, vmv1beta1.SecretsDir)
 
+	args = build.AddHTTPShutdownDelayArg(args, r.Spec.ExtraArgs, r.Spec.TerminationGracePeriodSeconds)
 	args = build.AddExtraArgsOverrideDefaults(args, r.Spec.ExtraArgs, "-")
 	sort.Strings(args)
 	vlsingleContainer := corev1.Container{

internal/controller/operator/factory/vmagent/vmagent.go

@@ -656,6 +656,7 @@ func newPodSpec(cr *vmv1beta1.VMAgent, ac *build.AssetsCache) (*corev1.PodSpec,
 	args = build.StreamAggrArgsTo(args, "streamAggr", streamAggrKeys, streamAggrConfigs...)
 
 	args = build.AppendArgsForInsertPorts(args, cr.Spec.InsertPorts)
+	args = build.AddHTTPShutdownDelayArg(args, cr.Spec.ExtraArgs, cr.Spec.TerminationGracePeriodSeconds)
 	args = build.AddExtraArgsOverrideDefaults(args, cr.Spec.ExtraArgs, "-")
 	sort.Strings(args)