diff --git a/internal/controller/operator/factory/build/container.go b/internal/controller/operator/factory/build/container.go index 18b65e0a7..0de9bd42b 100644 --- a/internal/controller/operator/factory/build/container.go +++ b/internal/controller/operator/factory/build/container.go @@ -216,6 +216,28 @@ func AddExtraArgsOverrideDefaults(args []string, extraArgs map[string]string, da return args } +const ( + // DefaultTerminationGracePeriodSeconds is the default termination grace period for all components + DefaultTerminationGracePeriodSeconds int64 = 30 +) + +// AddHTTPShutdownDelayArg adds default -http.shutdownDelay flag if user didn't override it in extraArgs. +// The delay is derived from terminationGracePeriodSeconds. +// It is added only for new resources +func AddHTTPShutdownDelayArg(args []string, extraArgs map[string]string, terminationGracePeriodSeconds *int64) []string { + if _, ok := extraArgs["http.shutdownDelay"]; ok { + return args + } + + delaySeconds := DefaultTerminationGracePeriodSeconds + if terminationGracePeriodSeconds != nil { + delaySeconds = *terminationGracePeriodSeconds + } + + args = append(args, fmt.Sprintf("-http.shutdownDelay=%ds", delaySeconds)) + return args +} + // formatContainerImage returns container image with registry prefix if needed. func formatContainerImage(registry string, containerImage string) string { if registry == "" { diff --git a/internal/controller/operator/factory/build/container_test.go b/internal/controller/operator/factory/build/container_test.go index e2e763d8d..7eec66f9b 100644 --- a/internal/controller/operator/factory/build/container_test.go +++ b/internal/controller/operator/factory/build/container_test.go @@ -9,6 +9,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" vmv1 "github.com/VictoriaMetrics/operator/api/operator/v1" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" @@ -205,6 +206,45 @@ func Test_addExtraArgsOverrideDefaults(t *testing.T) { }) } +func TestAddHTTPShutdownDelayArg(t *testing.T) { + type opts struct { + extraArgs map[string]string + terminationGracePeriodSeconds *int64 + wantArgs []string + } + f := func(opts opts) { + t.Helper() + assert.Equal( + t, + AddHTTPShutdownDelayArg(nil, opts.extraArgs, opts.terminationGracePeriodSeconds), + opts.wantArgs, + ) + } + // new resource, no explicit settings + f(opts{ + extraArgs: nil, + terminationGracePeriodSeconds: nil, + wantArgs: []string{"-http.shutdownDelay=30s"}, + }) + // if extraArg already exists, no more args should be added, it will be added later in the process of adding extra args + f(opts{ + extraArgs: map[string]string{"http.shutdownDelay": "5s"}, + terminationGracePeriodSeconds: nil, + wantArgs: nil, + }) + // new resource with explicit terminationGracePeriodSeconds + f(opts{ + extraArgs: nil, + terminationGracePeriodSeconds: ptr.To[int64](60), + wantArgs: []string{"-http.shutdownDelay=60s"}, + }) + f(opts{ + extraArgs: map[string]string{"http.shutdownDelay": "20s"}, + terminationGracePeriodSeconds: ptr.To[int64](120), + wantArgs: nil, + }) +} + func TestFormatContainerImage(t *testing.T) { f := func(globalRepo, image, wantImage string) { t.Helper() diff --git a/internal/controller/operator/factory/build/daemonset.go b/internal/controller/operator/factory/build/daemonset.go index c82f22dd0..e7f45894f 100644 --- a/internal/controller/operator/factory/build/daemonset.go +++ b/internal/controller/operator/factory/build/daemonset.go @@ -23,10 +23,12 @@ func DaemonSetAddCommonParams(dst *appsv1.DaemonSet, useStrictSecurity bool, par dst.Spec.Template.Spec.DNSConfig = params.DNSConfig dst.Spec.Template.Spec.NodeSelector = params.NodeSelector dst.Spec.Template.Spec.SecurityContext = AddStrictSecuritySettingsWithRootToPod(params.SecurityContext, useStrictSecurity) + if params.TerminationGracePeriodSeconds == nil { + params.TerminationGracePeriodSeconds = ptr.To(DefaultTerminationGracePeriodSeconds) + } dst.Spec.Template.Spec.TerminationGracePeriodSeconds = params.TerminationGracePeriodSeconds dst.Spec.Template.Spec.TopologySpreadConstraints = params.TopologySpreadConstraints dst.Spec.Template.Spec.ImagePullSecrets = params.ImagePullSecrets - dst.Spec.Template.Spec.TerminationGracePeriodSeconds = params.TerminationGracePeriodSeconds dst.Spec.Template.Spec.ReadinessGates = params.ReadinessGates dst.Spec.MinReadySeconds = params.MinReadySeconds dst.Spec.RevisionHistoryLimit = params.RevisionHistoryLimitCount diff --git a/internal/controller/operator/factory/build/defaults.go b/internal/controller/operator/factory/build/defaults.go index d6075dd6a..db920ca2e 100644 --- a/internal/controller/operator/factory/build/defaults.go +++ b/internal/controller/operator/factory/build/defaults.go @@ -347,8 +347,6 @@ const ( vmStorageDefaultDBPath = "vmstorage-data" ) -var defaultTerminationGracePeriod = int64(30) - func addVMClusterSpecDefaults(spec *vmv1beta1.VMClusterSpec) { c := getCfg() @@ -409,9 +407,6 @@ func addVMClusterSpecDefaults(spec *vmv1beta1.VMClusterSpec) { if spec.VMStorage.StorageDataPath == "" { spec.VMStorage.StorageDataPath = vmStorageDefaultDBPath } - if spec.VMStorage.TerminationGracePeriodSeconds == nil { - spec.VMStorage.TerminationGracePeriodSeconds = &defaultTerminationGracePeriod - } if spec.VMStorage.UseDefaultResources == nil { spec.VMStorage.UseDefaultResources = &c.VMCluster.UseDefaultResources } diff --git a/internal/controller/operator/factory/build/deployment.go b/internal/controller/operator/factory/build/deployment.go index 215e10fd1..a95d89a94 100644 --- a/internal/controller/operator/factory/build/deployment.go +++ b/internal/controller/operator/factory/build/deployment.go @@ -23,10 +23,12 @@ func DeploymentAddCommonParams(dst *appsv1.Deployment, useStrictSecurity bool, p dst.Spec.Template.Spec.DNSConfig = params.DNSConfig dst.Spec.Template.Spec.NodeSelector = params.NodeSelector dst.Spec.Template.Spec.SecurityContext = AddStrictSecuritySettingsToPod(params.SecurityContext, useStrictSecurity) + if params.TerminationGracePeriodSeconds == nil { + params.TerminationGracePeriodSeconds = ptr.To(DefaultTerminationGracePeriodSeconds) + } dst.Spec.Template.Spec.TerminationGracePeriodSeconds = params.TerminationGracePeriodSeconds dst.Spec.Template.Spec.TopologySpreadConstraints = params.TopologySpreadConstraints dst.Spec.Template.Spec.ImagePullSecrets = params.ImagePullSecrets - dst.Spec.Template.Spec.TerminationGracePeriodSeconds = params.TerminationGracePeriodSeconds dst.Spec.Template.Spec.ReadinessGates = params.ReadinessGates dst.Spec.MinReadySeconds = params.MinReadySeconds dst.Spec.Replicas = params.ReplicaCount diff --git a/internal/controller/operator/factory/build/statefulset.go b/internal/controller/operator/factory/build/statefulset.go index 172fa55f7..dd7c03e54 100644 --- a/internal/controller/operator/factory/build/statefulset.go +++ b/internal/controller/operator/factory/build/statefulset.go @@ -23,10 +23,12 @@ func StatefulSetAddCommonParams(dst *appsv1.StatefulSet, useStrictSecurity bool, dst.Spec.Template.Spec.DNSConfig = params.DNSConfig dst.Spec.Template.Spec.NodeSelector = params.NodeSelector dst.Spec.Template.Spec.SecurityContext = AddStrictSecuritySettingsToPod(params.SecurityContext, useStrictSecurity) + if params.TerminationGracePeriodSeconds == nil { + params.TerminationGracePeriodSeconds = ptr.To(DefaultTerminationGracePeriodSeconds) + } dst.Spec.Template.Spec.TerminationGracePeriodSeconds = params.TerminationGracePeriodSeconds dst.Spec.Template.Spec.TopologySpreadConstraints = params.TopologySpreadConstraints dst.Spec.Template.Spec.ImagePullSecrets = params.ImagePullSecrets - dst.Spec.Template.Spec.TerminationGracePeriodSeconds = params.TerminationGracePeriodSeconds dst.Spec.Template.Spec.ReadinessGates = params.ReadinessGates dst.Spec.MinReadySeconds = params.MinReadySeconds dst.Spec.Replicas = params.ReplicaCount diff --git a/internal/controller/operator/factory/vlagent/vlagent.go b/internal/controller/operator/factory/vlagent/vlagent.go index 4fb025467..ae269fdec 100644 --- a/internal/controller/operator/factory/vlagent/vlagent.go +++ b/internal/controller/operator/factory/vlagent/vlagent.go @@ -425,6 +425,7 @@ func newPodSpec(cr *vmv1.VLAgent) (*corev1.PodSpec, error) { agentVolumeMounts = append(agentVolumeMounts, cvm) } volumes, agentVolumeMounts = addRemoteWriteAssetsToVolumes(volumes, agentVolumeMounts, cr) + args = build.AddHTTPShutdownDelayArg(args, cr.Spec.ExtraArgs, cr.Spec.TerminationGracePeriodSeconds) args = build.AddExtraArgsOverrideDefaults(args, cr.Spec.ExtraArgs, "-") sort.Strings(args) diff --git a/internal/controller/operator/factory/vlagent/vlagent_test.go b/internal/controller/operator/factory/vlagent/vlagent_test.go index c8856f2c2..faf086e8d 100644 --- a/internal/controller/operator/factory/vlagent/vlagent_test.go +++ b/internal/controller/operator/factory/vlagent/vlagent_test.go @@ -294,6 +294,34 @@ func TestCreateOrUpdate(t *testing.T) { }, }, }) + + // test custom terminationGracePeriodSeconds is propagated to pod spec and shutdownDelay + f(opts{ + cr: &vmv1.VLAgent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example-agent-grace", + Namespace: "default", + }, + Spec: vmv1.VLAgentSpec{ + RemoteWrite: []vmv1.VLAgentRemoteWriteSpec{ + {URL: "http://remote-write"}, + }, + CommonApplicationDeploymentParams: vmv1beta1.CommonApplicationDeploymentParams{ + ReplicaCount: ptr.To(int32(1)), + TerminationGracePeriodSeconds: ptr.To[int64](60), + }, + }, + }, + validate: func(got *appsv1.StatefulSet) { + // terminationGracePeriodSeconds should be set on the pod spec + assert.NotNil(t, got.Spec.Template.Spec.TerminationGracePeriodSeconds) + assert.Equal(t, int64(60), *got.Spec.Template.Spec.TerminationGracePeriodSeconds) + // http.shutdownDelay should inherit from terminationGracePeriodSeconds + cnt := got.Spec.Template.Spec.Containers[0] + assert.Contains(t, cnt.Args, "-http.shutdownDelay=60s") + }, + }) + } func TestBuildRemoteWriteArgs(t *testing.T) { @@ -775,6 +803,7 @@ containers: - name: vlagent image: vm-repo:v1.97.1 args: + - -http.shutdownDelay=30s - -httpListenAddr=:9425 - -tmpDataPath=/vlagent-data ports: @@ -839,6 +868,7 @@ containers: - name: vlagent image: victoriametrics/vlagent:v1.97.1 args: + - -http.shutdownDelay=30s - -httpListenAddr=:9429 - -tmpDataPath=/vlagent-data ports: @@ -905,6 +935,7 @@ containers: - name: vlagent image: victoriametrics/vlagent:v1.97.1 args: + - -http.shutdownDelay=30s - -httpListenAddr=:9425 - -remoteWrite.maxDiskUsagePerURL=10GB,10GB, - -remoteWrite.url=http://some-url/api/v1/write,http://some-url-2/api/v1/write,http://some-url-3/api/v1/write @@ -978,6 +1009,7 @@ containers: - name: vlagent image: victoriametrics/vlagent:v1.47.0 args: + - -http.shutdownDelay=30s - -httpListenAddr=:9425 - -kubernetesCollector - -kubernetesCollector.includePodLabels @@ -1072,6 +1104,7 @@ containers: - name: vlagent image: victoriametrics/vlagent:v1.97.1 args: + - -http.shutdownDelay=30s - -httpListenAddr=:9425 - -remoteWrite.maxDiskUsagePerURL=10GB,20MB,10GB - -remoteWrite.url=http://some-url/api/v1/write,http://some-url-2/api/v1/write,http://some-url-3/api/v1/write @@ -1148,6 +1181,7 @@ containers: - name: vlagent image: victoriametrics/vlagent:v0.0.1 args: + - -http.shutdownDelay=30s - -httpListenAddr=:9425 - -remoteWrite.maxDiskUsagePerURL=35GiB - -remoteWrite.url=http://some-url/api/v1/write,http://some-url-2/api/v1/write,http://some-url-3/api/v1/write @@ -1186,5 +1220,61 @@ containers: serviceaccountname: vlagent-agent `) + // test custom terminationGrace probe affects both readiness and shutdownDelay + f(&vmv1.VLAgent{ + ObjectMeta: metav1.ObjectMeta{Name: "agent", Namespace: "default"}, + Spec: vmv1.VLAgentSpec{ + CommonDefaultableParams: vmv1beta1.CommonDefaultableParams{ + Image: vmv1beta1.Image{ + Tag: "v1.97.1", + }, + UseDefaultResources: ptr.To(false), + Port: "9429", + }, + CommonApplicationDeploymentParams: vmv1beta1.CommonApplicationDeploymentParams{ + TerminationGracePeriodSeconds: ptr.To[int64](40), + }, + }, + }, []runtime.Object{}, ` +containers: + - name: vlagent + image: victoriametrics/vlagent:v1.97.1 + args: + - -http.shutdownDelay=40s + - -httpListenAddr=:9429 + - -tmpDataPath=/vlagent-data + ports: + - name: http + containerport: 9429 + protocol: TCP + volumemounts: + - name: tmp-data + mountpath: /vlagent-data + livenessprobe: + probehandler: + httpget: + path: /health + port: + intval: 9429 + scheme: HTTP + timeoutseconds: 5 + periodseconds: 5 + successthreshold: 1 + failurethreshold: 10 + readinessprobe: + probehandler: + httpget: + path: /health + port: + intval: 9429 + scheme: HTTP + timeoutseconds: 5 + periodseconds: 5 + successthreshold: 1 + failurethreshold: 10 + terminationmessagepolicy: FallbackToLogsOnError + imagepullpolicy: IfNotPresent +serviceaccountname: vlagent-agent +`) } diff --git a/internal/controller/operator/factory/vlcluster/vlcluster_test.go b/internal/controller/operator/factory/vlcluster/vlcluster_test.go index 05fe96679..a1f5bfe47 100644 --- a/internal/controller/operator/factory/vlcluster/vlcluster_test.go +++ b/internal/controller/operator/factory/vlcluster/vlcluster_test.go @@ -100,7 +100,7 @@ func TestCreateOrUpdate(t *testing.T) { assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentInsert), Namespace: cr.Namespace}, &dep)) assert.Len(t, dep.Spec.Template.Spec.Containers, 1) cnt := dep.Spec.Template.Spec.Containers[0] - assert.Equal(t, cnt.Args, []string{"-httpListenAddr=:9481", "-internalselect.disable=true", "-storageNode=vlstorage-base-0.vlstorage-base.default:9491,vlstorage-base-1.vlstorage-base.default:9491"}) + assert.Equal(t, cnt.Args, []string{"-http.shutdownDelay=30s", "-httpListenAddr=:9481", "-internalselect.disable=true", "-storageNode=vlstorage-base-0.vlstorage-base.default:9491,vlstorage-base-1.vlstorage-base.default:9491"}) assert.Nil(t, dep.Annotations) assert.Equal(t, dep.Labels, cr.FinalLabels(vmv1beta1.ClusterComponentInsert)) @@ -108,7 +108,7 @@ func TestCreateOrUpdate(t *testing.T) { assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect), Namespace: cr.Namespace}, &dep)) assert.Len(t, dep.Spec.Template.Spec.Containers, 1) cnt = dep.Spec.Template.Spec.Containers[0] - assert.Equal(t, cnt.Args, []string{"-httpListenAddr=:9471", "-internalinsert.disable=true", "-storageNode=vlstorage-base-0.vlstorage-base.default:9491,vlstorage-base-1.vlstorage-base.default:9491"}) + assert.Equal(t, cnt.Args, []string{"-http.shutdownDelay=30s", "-httpListenAddr=:9471", "-internalinsert.disable=true", "-storageNode=vlstorage-base-0.vlstorage-base.default:9491,vlstorage-base-1.vlstorage-base.default:9491"}) assert.Nil(t, dep.Annotations) assert.Equal(t, dep.Labels, cr.FinalLabels(vmv1beta1.ClusterComponentSelect)) @@ -117,7 +117,7 @@ func TestCreateOrUpdate(t *testing.T) { assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage), Namespace: cr.Namespace}, &sts)) assert.Len(t, sts.Spec.Template.Spec.Containers, 1) cnt = sts.Spec.Template.Spec.Containers[0] - assert.Equal(t, cnt.Args, []string{"-httpListenAddr=:9491", "-storageDataPath=/vlstorage-data"}) + assert.Equal(t, cnt.Args, []string{"-http.shutdownDelay=30s", "-httpListenAddr=:9491", "-storageDataPath=/vlstorage-data"}) assert.Nil(t, sts.Annotations) assert.Equal(t, sts.Labels, cr.FinalLabels(vmv1beta1.ClusterComponentStorage)) }, @@ -147,7 +147,7 @@ func TestCreateOrUpdate(t *testing.T) { assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage), Namespace: cr.Namespace}, &sts)) assert.Len(t, sts.Spec.Template.Spec.Containers, 1) cnt := sts.Spec.Template.Spec.Containers[0] - assert.Equal(t, cnt.Args, []string{"-futureRetention=2d", "-httpListenAddr=:9491", "-retention.maxDiskSpaceUsageBytes=5GB", "-retentionPeriod=1w", "-storageDataPath=/vlstorage-data"}) + assert.Equal(t, cnt.Args, []string{"-futureRetention=2d", "-http.shutdownDelay=30s", "-httpListenAddr=:9491", "-retention.maxDiskSpaceUsageBytes=5GB", "-retentionPeriod=1w", "-storageDataPath=/vlstorage-data"}) }, }) @@ -184,6 +184,7 @@ func TestCreateOrUpdate(t *testing.T) { assert.Len(t, d.Spec.Template.Spec.Containers, 1) cnt := d.Spec.Template.Spec.Containers[0] assert.Equal(t, cnt.Args, []string{ + "-http.shutdownDelay=30s", "-httpListenAddr=:9471", "-internalinsert.disable=true", "-storageNode=vlstorage-read-only-0.vlstorage-read-only.default:9491,localhost:10101", diff --git a/internal/controller/operator/factory/vlcluster/vlinsert.go b/internal/controller/operator/factory/vlcluster/vlinsert.go index b748bd2dc..568b452f4 100644 --- a/internal/controller/operator/factory/vlcluster/vlinsert.go +++ b/internal/controller/operator/factory/vlcluster/vlinsert.go @@ -206,6 +206,7 @@ func buildVLInsertPodSpec(cr *vmv1.VLCluster) (*corev1.PodTemplateSpec, error) { }) } + args = build.AddHTTPShutdownDelayArg(args, cr.Spec.VLInsert.ExtraArgs, cr.Spec.VLInsert.TerminationGracePeriodSeconds) args = build.AddExtraArgsOverrideDefaults(args, cr.Spec.VLInsert.ExtraArgs, "-") sort.Strings(args) diff --git a/internal/controller/operator/factory/vlcluster/vlselect.go b/internal/controller/operator/factory/vlcluster/vlselect.go index c3d3bf840..65560f54a 100644 --- a/internal/controller/operator/factory/vlcluster/vlselect.go +++ b/internal/controller/operator/factory/vlcluster/vlselect.go @@ -307,6 +307,7 @@ func buildVLSelectPodSpec(cr *vmv1.VLCluster) (*corev1.PodTemplateSpec, error) { }) } + args = build.AddHTTPShutdownDelayArg(args, cr.Spec.VLSelect.ExtraArgs, cr.Spec.VLSelect.TerminationGracePeriodSeconds) args = build.AddExtraArgsOverrideDefaults(args, cr.Spec.VLSelect.ExtraArgs, "-") sort.Strings(args) selectContainers := corev1.Container{ diff --git a/internal/controller/operator/factory/vlcluster/vlstorage.go b/internal/controller/operator/factory/vlcluster/vlstorage.go index 644b130fb..02a59f1f8 100644 --- a/internal/controller/operator/factory/vlcluster/vlstorage.go +++ b/internal/controller/operator/factory/vlcluster/vlstorage.go @@ -309,6 +309,7 @@ func buildVLStoragePodSpec(cr *vmv1.VLCluster) (*corev1.PodTemplateSpec, error) }) } + args = build.AddHTTPShutdownDelayArg(args, cr.Spec.VLStorage.ExtraArgs, cr.Spec.VLStorage.TerminationGracePeriodSeconds) args = build.AddExtraArgsOverrideDefaults(args, cr.Spec.VLStorage.ExtraArgs, "-") sort.Strings(args) vmstorageContainer := corev1.Container{ diff --git a/internal/controller/operator/factory/vlcluster/vmauth_lb.go b/internal/controller/operator/factory/vlcluster/vmauth_lb.go index ee3833c46..4ff7ca676 100644 --- a/internal/controller/operator/factory/vlcluster/vmauth_lb.go +++ b/internal/controller/operator/factory/vlcluster/vmauth_lb.go @@ -161,6 +161,7 @@ func buildVMauthLBDeployment(cr *vmv1.VLCluster) (*appsv1.Deployment, error) { volumes, vmMounts = build.LicenseVolumeTo(volumes, vmMounts, cr.Spec.License, vmv1beta1.SecretsDir) args = build.LicenseArgsTo(args, cr.Spec.License, vmv1beta1.SecretsDir) + args = build.AddHTTPShutdownDelayArg(args, spec.ExtraArgs, spec.TerminationGracePeriodSeconds) args = build.AddExtraArgsOverrideDefaults(args, spec.ExtraArgs, "-") sort.Strings(args) vmauthLBCnt := corev1.Container{ diff --git a/internal/controller/operator/factory/vlsingle/vlsingle.go b/internal/controller/operator/factory/vlsingle/vlsingle.go index 1b644dd35..497778c14 100644 --- a/internal/controller/operator/factory/vlsingle/vlsingle.go +++ b/internal/controller/operator/factory/vlsingle/vlsingle.go @@ -235,6 +235,7 @@ func makePodSpec(r *vmv1.VLSingle) (*corev1.PodTemplateSpec, error) { volumes, vmMounts = build.LicenseVolumeTo(volumes, vmMounts, r.Spec.License, vmv1beta1.SecretsDir) args = build.LicenseArgsTo(args, r.Spec.License, vmv1beta1.SecretsDir) + args = build.AddHTTPShutdownDelayArg(args, r.Spec.ExtraArgs, r.Spec.TerminationGracePeriodSeconds) args = build.AddExtraArgsOverrideDefaults(args, r.Spec.ExtraArgs, "-") sort.Strings(args) vlsingleContainer := corev1.Container{ diff --git a/internal/controller/operator/factory/vmagent/vmagent.go b/internal/controller/operator/factory/vmagent/vmagent.go index ffa145038..6a6d9c1ac 100644 --- a/internal/controller/operator/factory/vmagent/vmagent.go +++ b/internal/controller/operator/factory/vmagent/vmagent.go @@ -656,6 +656,7 @@ func newPodSpec(cr *vmv1beta1.VMAgent, ac *build.AssetsCache) (*corev1.PodSpec, args = build.StreamAggrArgsTo(args, "streamAggr", streamAggrKeys, streamAggrConfigs...) args = build.AppendArgsForInsertPorts(args, cr.Spec.InsertPorts) + args = build.AddHTTPShutdownDelayArg(args, cr.Spec.ExtraArgs, cr.Spec.TerminationGracePeriodSeconds) args = build.AddExtraArgsOverrideDefaults(args, cr.Spec.ExtraArgs, "-") sort.Strings(args) diff --git a/internal/controller/operator/factory/vmagent/vmagent_test.go b/internal/controller/operator/factory/vmagent/vmagent_test.go index d1b6e76cf..a52ac7807 100644 --- a/internal/controller/operator/factory/vmagent/vmagent_test.go +++ b/internal/controller/operator/factory/vmagent/vmagent_test.go @@ -2106,6 +2106,7 @@ containers: - name: vmagent image: vm-repo:v1.97.1 args: + - -http.shutdownDelay=30s - -httpListenAddr=:8425 - -remoteWrite.maxDiskUsagePerURL=1073741824 - -remoteWrite.tmpDataPath=/tmp/vmagent-remotewrite-data @@ -2207,6 +2208,7 @@ containers: - name: vmagent image: victoriametrics/vmagent:v1.97.1 args: + - -http.shutdownDelay=30s - -httpListenAddr=:8429 - -promscrape.config=/etc/vmagent/config_out/vmagent.yaml - -remoteWrite.maxDiskUsagePerURL=1073741824 @@ -2338,6 +2340,7 @@ containers: - name: vmagent image: victoriametrics/vmagent:v1.97.1 args: + - -http.shutdownDelay=30s - -httpListenAddr=:8425 - -remoteWrite.maxDiskUsagePerURL=10GB,10GB,1073741824 - -remoteWrite.tmpDataPath=/tmp/vmagent-remotewrite-data @@ -2424,6 +2427,7 @@ containers: - name: vmagent image: victoriametrics/vmagent:v1.97.1 args: + - -http.shutdownDelay=30s - -httpListenAddr=:8425 - -remoteWrite.maxDiskUsagePerURL=10GB,20MB,10GB - -remoteWrite.tmpDataPath=/tmp/vmagent-remotewrite-data @@ -2517,6 +2521,7 @@ containers: - name: vmagent image: victoriametrics/vmagent:v1.97.1 args: + - -http.shutdownDelay=30s - -httpListenAddr=:8425 - -remoteWrite.forceVMProto=false - -remoteWrite.maxDiskUsagePerURL=35GiB diff --git a/internal/controller/operator/factory/vmalert/vmalert.go b/internal/controller/operator/factory/vmalert/vmalert.go index e5bfdd375..28e665a94 100644 --- a/internal/controller/operator/factory/vmalert/vmalert.go +++ b/internal/controller/operator/factory/vmalert/vmalert.go @@ -522,8 +522,9 @@ func buildArgs(cr *vmv1beta1.VMAlert, ruleConfigMapNames []string, ac *build.Ass } args = build.LicenseArgsTo(args, cr.Spec.License, vmv1beta1.SecretsDir) - + args = build.AddHTTPShutdownDelayArg(args, cr.Spec.ExtraArgs, cr.Spec.TerminationGracePeriodSeconds) args = build.AddExtraArgsOverrideDefaults(args, cr.Spec.ExtraArgs, "-") + sort.Strings(args) return args, nil } diff --git a/internal/controller/operator/factory/vmalert/vmalert_test.go b/internal/controller/operator/factory/vmalert/vmalert_test.go index d23c71f91..aed398fc2 100644 --- a/internal/controller/operator/factory/vmalert/vmalert_test.go +++ b/internal/controller/operator/factory/vmalert/vmalert_test.go @@ -682,7 +682,7 @@ func Test_buildVMAlertArgs(t *testing.T) { }, }, ruleConfigMapNames: []string{"first-rule-cm.yaml"}, - want: []string{"-datasource.url=http://vmsingle-url", "-httpListenAddr=:", "-notifier.url=http://test", "-rule=\"/etc/vmalert/config/first-rule-cm.yaml/*.yaml\""}, + want: []string{"-datasource.url=http://vmsingle-url", "-http.shutdownDelay=30s", "-httpListenAddr=:", "-notifier.url=http://test", "-rule=\"/etc/vmalert/config/first-rule-cm.yaml/*.yaml\""}, }) // with tls args @@ -712,7 +712,7 @@ func Test_buildVMAlertArgs(t *testing.T) { }, }, ruleConfigMapNames: []string{"first-rule-cm.yaml"}, - want: []string{"--datasource.headers=x-org-id:one^^x-org-tenant:5", "-datasource.tlsCAFile=/path/to/sa", "-datasource.tlsInsecureSkipVerify=true", "-datasource.tlsKeyFile=/path/to/key", "-datasource.url=http://vmsingle-url", "-httpListenAddr=:", "-notifier.url=http://test", "-rule=\"/etc/vmalert/config/first-rule-cm.yaml/*.yaml\""}, + want: []string{"--datasource.headers=x-org-id:one^^x-org-tenant:5", "-datasource.tlsCAFile=/path/to/sa", "-datasource.tlsInsecureSkipVerify=true", "-datasource.tlsKeyFile=/path/to/key", "-datasource.url=http://vmsingle-url", "-http.shutdownDelay=30s", "-httpListenAddr=:", "-notifier.url=http://test", "-rule=\"/etc/vmalert/config/first-rule-cm.yaml/*.yaml\""}, }) // with static and selector notifiers @@ -786,6 +786,7 @@ func Test_buildVMAlertArgs(t *testing.T) { }, want: []string{ "-datasource.url=http://some-vm-datasource", + "-http.shutdownDelay=30s", "-httpListenAddr=:", "-notifier.tlsCAFile=,/tmp/ca.cert,,,", "-notifier.tlsCertFile=,/tmp/cert.pem,,,", diff --git a/internal/controller/operator/factory/vmauth/vmauth.go b/internal/controller/operator/factory/vmauth/vmauth.go index 9b6c1f894..3a1efcbb6 100644 --- a/internal/controller/operator/factory/vmauth/vmauth.go +++ b/internal/controller/operator/factory/vmauth/vmauth.go @@ -326,6 +326,7 @@ func makeSpecForVMAuth(cr *vmv1beta1.VMAuth) (*corev1.PodTemplateSpec, error) { return nil, fmt.Errorf("cannot apply patch for initContainers: %w", err) } + args = build.AddHTTPShutdownDelayArg(args, cr.Spec.ExtraArgs, cr.Spec.TerminationGracePeriodSeconds) args = build.AddExtraArgsOverrideDefaults(args, cr.Spec.ExtraArgs, "-") sort.Strings(args) diff --git a/internal/controller/operator/factory/vmauth/vmauth_test.go b/internal/controller/operator/factory/vmauth/vmauth_test.go index ad327154c..923b0096f 100644 --- a/internal/controller/operator/factory/vmauth/vmauth_test.go +++ b/internal/controller/operator/factory/vmauth/vmauth_test.go @@ -471,6 +471,7 @@ containers: imagepullpolicy: IfNotPresent args: - -auth.config=/opt/vmauth/config.yaml + - -http.shutdownDelay=30s - -httpListenAddr=:8429 ports: - name: http @@ -582,6 +583,7 @@ containers: imagepullpolicy: IfNotPresent args: - -auth.config=/opt/vmauth/config.yaml + - -http.shutdownDelay=30s - -httpListenAddr=:8429 ports: - name: http diff --git a/internal/controller/operator/factory/vmcluster/vmcluster.go b/internal/controller/operator/factory/vmcluster/vmcluster.go index cc38202f5..0b107f8bd 100644 --- a/internal/controller/operator/factory/vmcluster/vmcluster.go +++ b/internal/controller/operator/factory/vmcluster/vmcluster.go @@ -657,6 +657,7 @@ func makePodSpecForVMSelect(cr *vmv1beta1.VMCluster) (*corev1.PodTemplateSpec, e volumes, vmMounts = build.LicenseVolumeTo(volumes, vmMounts, cr.Spec.License, vmv1beta1.SecretsDir) args = build.LicenseArgsTo(args, cr.Spec.License, vmv1beta1.SecretsDir) + args = build.AddHTTPShutdownDelayArg(args, cr.Spec.VMSelect.ExtraArgs, cr.Spec.VMSelect.TerminationGracePeriodSeconds) args = build.AddExtraArgsOverrideDefaults(args, cr.Spec.VMSelect.ExtraArgs, "-") sort.Strings(args) vmselectContainer := corev1.Container{ @@ -861,6 +862,7 @@ func makePodSpecForVMInsert(cr *vmv1beta1.VMCluster) (*corev1.PodTemplateSpec, e volumes, vmMounts = build.LicenseVolumeTo(volumes, vmMounts, cr.Spec.License, vmv1beta1.SecretsDir) args = build.LicenseArgsTo(args, cr.Spec.License, vmv1beta1.SecretsDir) + args = build.AddHTTPShutdownDelayArg(args, cr.Spec.VMInsert.ExtraArgs, cr.Spec.VMInsert.TerminationGracePeriodSeconds) args = build.AddExtraArgsOverrideDefaults(args, cr.Spec.VMInsert.ExtraArgs, "-") sort.Strings(args) @@ -1081,6 +1083,7 @@ func makePodSpecForVMStorage(ctx context.Context, cr *vmv1beta1.VMCluster) (*cor volumes, vmMounts = build.LicenseVolumeTo(volumes, vmMounts, cr.Spec.License, vmv1beta1.SecretsDir) args = build.LicenseArgsTo(args, cr.Spec.License, vmv1beta1.SecretsDir) + args = build.AddHTTPShutdownDelayArg(args, cr.Spec.VMStorage.ExtraArgs, cr.Spec.VMStorage.TerminationGracePeriodSeconds) args = build.AddExtraArgsOverrideDefaults(args, cr.Spec.VMStorage.ExtraArgs, "-") sort.Strings(args) vmstorageContainer := corev1.Container{ @@ -1508,6 +1511,7 @@ func buildVMAuthLBDeployment(cr *vmv1beta1.VMCluster) (*appsv1.Deployment, error volumes, vmounts = build.LicenseVolumeTo(volumes, vmounts, cr.Spec.License, vmv1beta1.SecretsDir) args = build.LicenseArgsTo(args, cr.Spec.License, vmv1beta1.SecretsDir) + args = build.AddHTTPShutdownDelayArg(args, spec.ExtraArgs, spec.TerminationGracePeriodSeconds) args = build.AddExtraArgsOverrideDefaults(args, spec.ExtraArgs, "-") sort.Strings(args) vmauthLBCnt := corev1.Container{ diff --git a/internal/controller/operator/factory/vmsingle/vmsingle.go b/internal/controller/operator/factory/vmsingle/vmsingle.go index 71e570c34..86f67441f 100644 --- a/internal/controller/operator/factory/vmsingle/vmsingle.go +++ b/internal/controller/operator/factory/vmsingle/vmsingle.go @@ -329,7 +329,7 @@ func newPodSpec(ctx context.Context, cr *vmv1beta1.VMSingle) (*corev1.PodTemplat volumes, vmMounts = build.LicenseVolumeTo(volumes, vmMounts, cr.Spec.License, vmv1beta1.SecretsDir) args = build.LicenseArgsTo(args, cr.Spec.License, vmv1beta1.SecretsDir) - + args = build.AddHTTPShutdownDelayArg(args, cr.Spec.ExtraArgs, cr.Spec.TerminationGracePeriodSeconds) args = build.AddExtraArgsOverrideDefaults(args, cr.Spec.ExtraArgs, "-") sort.Strings(args) vmsingleContainer := corev1.Container{ diff --git a/internal/controller/operator/factory/vtcluster/cluster_test.go b/internal/controller/operator/factory/vtcluster/cluster_test.go index b295f5fb0..9a0a9922a 100644 --- a/internal/controller/operator/factory/vtcluster/cluster_test.go +++ b/internal/controller/operator/factory/vtcluster/cluster_test.go @@ -100,7 +100,7 @@ func TestCreateOrUpdate(t *testing.T) { assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentInsert), Namespace: cr.Namespace}, &dep)) assert.Len(t, dep.Spec.Template.Spec.Containers, 1) cnt := dep.Spec.Template.Spec.Containers[0] - assert.Equal(t, cnt.Args, []string{"-httpListenAddr=:10481", "-internalselect.disable=true", "-storageNode=vtstorage-base-0.vtstorage-base.default:10491,vtstorage-base-1.vtstorage-base.default:10491"}) + assert.Equal(t, cnt.Args, []string{"-http.shutdownDelay=30s", "-httpListenAddr=:10481", "-internalselect.disable=true", "-storageNode=vtstorage-base-0.vtstorage-base.default:10491,vtstorage-base-1.vtstorage-base.default:10491"}) assert.Nil(t, dep.Annotations) assert.Equal(t, dep.Labels, cr.FinalLabels(vmv1beta1.ClusterComponentInsert)) @@ -108,7 +108,7 @@ func TestCreateOrUpdate(t *testing.T) { assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect), Namespace: cr.Namespace}, &dep)) assert.Len(t, dep.Spec.Template.Spec.Containers, 1) cnt = dep.Spec.Template.Spec.Containers[0] - assert.Equal(t, cnt.Args, []string{"-httpListenAddr=:10471", "-internalinsert.disable=true", "-storageNode=vtstorage-base-0.vtstorage-base.default:10491,vtstorage-base-1.vtstorage-base.default:10491"}) + assert.Equal(t, cnt.Args, []string{"-http.shutdownDelay=30s", "-httpListenAddr=:10471", "-internalinsert.disable=true", "-storageNode=vtstorage-base-0.vtstorage-base.default:10491,vtstorage-base-1.vtstorage-base.default:10491"}) assert.Nil(t, dep.Annotations) assert.Equal(t, dep.Labels, cr.FinalLabels(vmv1beta1.ClusterComponentSelect)) @@ -117,7 +117,7 @@ func TestCreateOrUpdate(t *testing.T) { assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage), Namespace: cr.Namespace}, &sts)) assert.Len(t, sts.Spec.Template.Spec.Containers, 1) cnt = sts.Spec.Template.Spec.Containers[0] - assert.Equal(t, cnt.Args, []string{"-httpListenAddr=:10491", "-storageDataPath=/vtstorage-data"}) + assert.Equal(t, cnt.Args, []string{"-http.shutdownDelay=30s", "-httpListenAddr=:10491", "-storageDataPath=/vtstorage-data"}) assert.Nil(t, sts.Annotations) assert.Equal(t, sts.Labels, cr.FinalLabels(vmv1beta1.ClusterComponentStorage)) }, @@ -147,7 +147,7 @@ func TestCreateOrUpdate(t *testing.T) { assert.Nil(t, rclient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage), Namespace: cr.Namespace}, &sts)) assert.Len(t, sts.Spec.Template.Spec.Containers, 1) cnt := sts.Spec.Template.Spec.Containers[0] - assert.Equal(t, cnt.Args, []string{"-futureRetention=2d", "-httpListenAddr=:10491", "-retention.maxDiskSpaceUsageBytes=5GB", "-retentionPeriod=1w", "-storageDataPath=/vtstorage-data"}) + assert.Equal(t, cnt.Args, []string{"-futureRetention=2d", "-http.shutdownDelay=30s", "-httpListenAddr=:10491", "-retention.maxDiskSpaceUsageBytes=5GB", "-retentionPeriod=1w", "-storageDataPath=/vtstorage-data"}) }, }) diff --git a/internal/controller/operator/factory/vtcluster/insert.go b/internal/controller/operator/factory/vtcluster/insert.go index d99928ba9..9e0f88bb4 100644 --- a/internal/controller/operator/factory/vtcluster/insert.go +++ b/internal/controller/operator/factory/vtcluster/insert.go @@ -196,6 +196,7 @@ func buildVTInsertPodSpec(cr *vmv1.VTCluster) (*corev1.PodTemplateSpec, error) { }) } + args = build.AddHTTPShutdownDelayArg(args, cr.Spec.Insert.ExtraArgs, cr.Spec.Insert.TerminationGracePeriodSeconds) args = build.AddExtraArgsOverrideDefaults(args, cr.Spec.Insert.ExtraArgs, "-") sort.Strings(args) diff --git a/internal/controller/operator/factory/vtcluster/select.go b/internal/controller/operator/factory/vtcluster/select.go index a6fd1b175..b22018a3e 100644 --- a/internal/controller/operator/factory/vtcluster/select.go +++ b/internal/controller/operator/factory/vtcluster/select.go @@ -303,6 +303,7 @@ func buildVTSelectPodSpec(cr *vmv1.VTCluster) (*corev1.PodTemplateSpec, error) { }) } + args = build.AddHTTPShutdownDelayArg(args, cr.Spec.Select.ExtraArgs, cr.Spec.Select.TerminationGracePeriodSeconds) args = build.AddExtraArgsOverrideDefaults(args, cr.Spec.Select.ExtraArgs, "-") sort.Strings(args) selectContainers := corev1.Container{ diff --git a/internal/controller/operator/factory/vtcluster/storage.go b/internal/controller/operator/factory/vtcluster/storage.go index 213e97dbe..17331b786 100644 --- a/internal/controller/operator/factory/vtcluster/storage.go +++ b/internal/controller/operator/factory/vtcluster/storage.go @@ -303,6 +303,7 @@ func buildVTStoragePodSpec(cr *vmv1.VTCluster) (*corev1.PodTemplateSpec, error) }) } + args = build.AddHTTPShutdownDelayArg(args, cr.Spec.Storage.ExtraArgs, cr.Spec.Storage.TerminationGracePeriodSeconds) args = build.AddExtraArgsOverrideDefaults(args, cr.Spec.Storage.ExtraArgs, "-") sort.Strings(args) vmstorageContainer := corev1.Container{ diff --git a/internal/controller/operator/factory/vtsingle/vtsingle.go b/internal/controller/operator/factory/vtsingle/vtsingle.go index 7c9d8e2d7..35d0ebdee 100644 --- a/internal/controller/operator/factory/vtsingle/vtsingle.go +++ b/internal/controller/operator/factory/vtsingle/vtsingle.go @@ -225,6 +225,8 @@ func makePodSpec(r *vmv1.VTSingle) (*corev1.PodTemplateSpec, error) { MountPath: path.Join(vmv1beta1.ConfigMapsDir, c), }) } + + args = build.AddHTTPShutdownDelayArg(args, r.Spec.ExtraArgs, r.Spec.TerminationGracePeriodSeconds) args = build.AddExtraArgsOverrideDefaults(args, r.Spec.ExtraArgs, "-") sort.Strings(args) vtsingleContainer := corev1.Container{