From b9c863bdd8c8b6177ab7f9ccc02b73f57477fce9 Mon Sep 17 00:00:00 2001 From: Mariusz Mazur Date: Fri, 10 Jul 2026 17:16:56 +0200 Subject: [PATCH 1/2] feat(observability): add Kusto-only alert lanes Add service and HCP alert generator lanes that route only to the Event Hub action group used for Kusto ingestion. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 31aff0de-9d6d-4870-9732-600ab6570a24 --- .../modules/metrics/kusto-hcp-rules.bicep | 19 +++++++++++++++++++ .../modules/metrics/kusto-service-rules.bicep | 19 +++++++++++++++++++ ...dKustoOnlyHCPPrometheusAlertingRules.bicep | 11 +++++++++++ ...toOnlyServicePrometheusAlertingRules.bicep | 11 +++++++++++ dev-infrastructure/templates/monitoring.bicep | 18 ++++++++++++++++++ observability/alerts-kusto-hcps.yaml | 4 ++++ observability/alerts-kusto-services.yaml | 4 ++++ tooling/prometheus-rules/Makefile | 14 ++++++++++++-- 8 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 dev-infrastructure/modules/metrics/kusto-hcp-rules.bicep create mode 100644 dev-infrastructure/modules/metrics/kusto-service-rules.bicep create mode 100644 dev-infrastructure/modules/metrics/rules/generatedKustoOnlyHCPPrometheusAlertingRules.bicep create mode 100644 dev-infrastructure/modules/metrics/rules/generatedKustoOnlyServicePrometheusAlertingRules.bicep create mode 100644 observability/alerts-kusto-hcps.yaml create mode 100644 observability/alerts-kusto-services.yaml diff --git a/dev-infrastructure/modules/metrics/kusto-hcp-rules.bicep b/dev-infrastructure/modules/metrics/kusto-hcp-rules.bicep new file mode 100644 index 00000000000..1f4cda0bbb0 --- /dev/null +++ b/dev-infrastructure/modules/metrics/kusto-hcp-rules.bicep @@ -0,0 +1,19 @@ +// Kusto-only alerts that apply to hosted control planes. + +@description('The Azure resource ID of the Azure Monitor Workspace (stores prometheus metrics for HCPs)') +param azureMonitoringWorkspaceId string + +@description('Action group resource IDs to notify when alerts fire') +param actionGroups array + +@description('The minimum IcM severity level (highest priority) that alerts can fire at. Alerts more critical than this ceiling will be degraded to this value. 0 means no ceiling.') +param severityCeiling int = 0 + +module generatedAlerts 'rules/generatedKustoOnlyHCPPrometheusAlertingRules.bicep' = { + name: 'generatedKustoOnlyHCPPrometheusAlertingRules' + params: { + azureMonitoring: azureMonitoringWorkspaceId + actionGroups: actionGroups + severityCeiling: severityCeiling + } +} diff --git a/dev-infrastructure/modules/metrics/kusto-service-rules.bicep b/dev-infrastructure/modules/metrics/kusto-service-rules.bicep new file mode 100644 index 00000000000..f4e5f00d01d --- /dev/null +++ b/dev-infrastructure/modules/metrics/kusto-service-rules.bicep @@ -0,0 +1,19 @@ +// Kusto-only alerts that apply to service clusters and services. + +@description('The Azure resource ID of the Azure Monitor Workspace (stores prometheus metrics for services/aks level metrics)') +param azureMonitoringWorkspaceId string + +@description('Action group resource IDs to notify when alerts fire') +param actionGroups array + +@description('The minimum IcM severity level (highest priority) that alerts can fire at. Alerts more critical than this ceiling will be degraded to this value. 0 means no ceiling.') +param severityCeiling int = 0 + +module generatedAlerts 'rules/generatedKustoOnlyServicePrometheusAlertingRules.bicep' = { + name: 'generatedKustoOnlyServicePrometheusAlertingRules' + params: { + azureMonitoring: azureMonitoringWorkspaceId + actionGroups: actionGroups + severityCeiling: severityCeiling + } +} diff --git a/dev-infrastructure/modules/metrics/rules/generatedKustoOnlyHCPPrometheusAlertingRules.bicep b/dev-infrastructure/modules/metrics/rules/generatedKustoOnlyHCPPrometheusAlertingRules.bicep new file mode 100644 index 00000000000..74c11f5f18c --- /dev/null +++ b/dev-infrastructure/modules/metrics/rules/generatedKustoOnlyHCPPrometheusAlertingRules.bicep @@ -0,0 +1,11 @@ +#disable-next-line no-unused-params +param azureMonitoring string + +#disable-next-line no-unused-params +param actionGroups array + +@description('The minimum IcM severity level (highest priority) that alerts can fire at. Alerts more critical than this ceiling will be degraded to this value. 0 means no ceiling.') +param severityCeiling int = 0 + +#disable-next-line no-unused-params +param location string = resourceGroup().location diff --git a/dev-infrastructure/modules/metrics/rules/generatedKustoOnlyServicePrometheusAlertingRules.bicep b/dev-infrastructure/modules/metrics/rules/generatedKustoOnlyServicePrometheusAlertingRules.bicep new file mode 100644 index 00000000000..74c11f5f18c --- /dev/null +++ b/dev-infrastructure/modules/metrics/rules/generatedKustoOnlyServicePrometheusAlertingRules.bicep @@ -0,0 +1,11 @@ +#disable-next-line no-unused-params +param azureMonitoring string + +#disable-next-line no-unused-params +param actionGroups array + +@description('The minimum IcM severity level (highest priority) that alerts can fire at. Alerts more critical than this ceiling will be degraded to this value. 0 means no ceiling.') +param severityCeiling int = 0 + +#disable-next-line no-unused-params +param location string = resourceGroup().location diff --git a/dev-infrastructure/templates/monitoring.bicep b/dev-infrastructure/templates/monitoring.bicep index 1c7a1aa62dd..d5c6bfe9c09 100644 --- a/dev-infrastructure/templates/monitoring.bicep +++ b/dev-infrastructure/templates/monitoring.bicep @@ -161,6 +161,15 @@ module svcKubeAlerts '../modules/metrics/svc-kube-rules.bicep' = { } } +module kustoServiceAlerts '../modules/metrics/kusto-service-rules.bicep' = { + name: 'kustoServiceAlerts' + params: { + azureMonitoringWorkspaceId: azureMonitoringWorkspaceId + actionGroups: ehActionGroups + severityCeiling: alertSeverityCeiling + } +} + module hcpAlerts '../modules/metrics/hcp-rules.bicep' = { name: 'hcpAlerts' params: { @@ -179,6 +188,15 @@ module slHcpAlerts '../modules/metrics/sl-hcp-rules.bicep' = { } } +module kustoHcpAlerts '../modules/metrics/kusto-hcp-rules.bicep' = { + name: 'kustoHcpAlerts' + params: { + azureMonitoringWorkspaceId: hcpAzureMonitoringWorkspaceId + actionGroups: ehActionGroups + severityCeiling: alertSeverityCeiling + } +} + module sreServiceAlerts '../modules/metrics/sre-service-rules.bicep' = { name: 'sreServiceAlerts' params: { diff --git a/observability/alerts-kusto-hcps.yaml b/observability/alerts-kusto-hcps.yaml new file mode 100644 index 00000000000..3f5ac16ffcb --- /dev/null +++ b/observability/alerts-kusto-hcps.yaml @@ -0,0 +1,4 @@ +prometheusRules: + rulesFolders: [] + untestedRules: [] + outputBicep: ../dev-infrastructure/modules/metrics/rules/generatedKustoOnlyHCPPrometheusAlertingRules.bicep diff --git a/observability/alerts-kusto-services.yaml b/observability/alerts-kusto-services.yaml new file mode 100644 index 00000000000..bda13a93c1f --- /dev/null +++ b/observability/alerts-kusto-services.yaml @@ -0,0 +1,4 @@ +prometheusRules: + rulesFolders: [] + untestedRules: [] + outputBicep: ../dev-infrastructure/modules/metrics/rules/generatedKustoOnlyServicePrometheusAlertingRules.bicep diff --git a/tooling/prometheus-rules/Makefile b/tooling/prometheus-rules/Makefile index 6c15c1260c6..edc88fbc3e3 100644 --- a/tooling/prometheus-rules/Makefile +++ b/tooling/prometheus-rules/Makefile @@ -23,7 +23,7 @@ fmt-devinfra: run: alerts recording-rules .PHONY: run -alerts: run-sl-services run-sl-svc-kube run-sl-hcps run-sre-hcps run-sre-services run-rp-services run-msft-services +alerts: run-sl-services run-sl-svc-kube run-sl-hcps run-sre-hcps run-sre-services run-rp-services run-msft-services run-kusto-services run-kusto-hcps $(MAKE) fmt-devinfra .PHONY: alerts @@ -59,6 +59,14 @@ run-msft-services: go run ./... --config-file ../../observability/alerts-msft-services.yaml $(EXTRA_FLAGS) .PHONY: run-msft-services +run-kusto-services: + go run ./... --config-file ../../observability/alerts-kusto-services.yaml $(EXTRA_FLAGS) +.PHONY: run-kusto-services + +run-kusto-hcps: + go run ./... --config-file ../../observability/alerts-kusto-hcps.yaml $(EXTRA_FLAGS) +.PHONY: run-kusto-hcps + run-recording-rules-services: go run ./... --config-file ../../observability/recording-rules-services.yaml $(EXTRA_FLAGS) .PHONY: run-recording-rules-services @@ -74,7 +82,9 @@ ALERT_CONFIGS = \ ../../observability/alerts-sre-hcps.yaml \ ../../observability/alerts-sre-services.yaml \ ../../observability/alerts-rp-services.yaml \ - ../../observability/alerts-msft-services.yaml + ../../observability/alerts-msft-services.yaml \ + ../../observability/alerts-kusto-services.yaml \ + ../../observability/alerts-kusto-hcps.yaml correlation-map: go run ./... --correlation-map $(ALERT_CONFIGS) > $(OUTPUT) From 0526c9d637d8b0d28b757fdbb98ef6ab34c3661c Mon Sep 17 00:00:00 2001 From: Mariusz Mazur Date: Fri, 10 Jul 2026 17:18:42 +0200 Subject: [PATCH 2/2] refactor(observability): move svc kube alerts to Kusto-only lane Route the service-workspace copy of the imported Kubernetes alerts through the Kusto-only service lane and remove the old SL svc-kube lane. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 31aff0de-9d6d-4870-9732-600ab6570a24 --- ...toOnlyServicePrometheusAlertingRules.bicep | 1631 ++++++++++++++++ ...eratedSvcKubePrometheusAlertingRules.bicep | 1642 ----------------- .../modules/metrics/svc-kube-rules.bicep | 20 - dev-infrastructure/templates/monitoring.bicep | 9 - observability/alerts-kusto-services.yaml | 21 +- observability/alerts-sl-svc-kube.yaml | 23 - tooling/prometheus-rules/Makefile | 7 +- 7 files changed, 1652 insertions(+), 1701 deletions(-) delete mode 100644 dev-infrastructure/modules/metrics/rules/generatedSvcKubePrometheusAlertingRules.bicep delete mode 100644 dev-infrastructure/modules/metrics/svc-kube-rules.bicep delete mode 100644 observability/alerts-sl-svc-kube.yaml diff --git a/dev-infrastructure/modules/metrics/rules/generatedKustoOnlyServicePrometheusAlertingRules.bicep b/dev-infrastructure/modules/metrics/rules/generatedKustoOnlyServicePrometheusAlertingRules.bicep index 74c11f5f18c..baeacda3b15 100644 --- a/dev-infrastructure/modules/metrics/rules/generatedKustoOnlyServicePrometheusAlertingRules.bicep +++ b/dev-infrastructure/modules/metrics/rules/generatedKustoOnlyServicePrometheusAlertingRules.bicep @@ -9,3 +9,1634 @@ param severityCeiling int = 0 #disable-next-line no-unused-params param location string = resourceGroup().location + +resource svcKubernetesApps 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'svc-kubernetes-apps' + location: location + properties: { + interval: 'PT1M' + rules: [ + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubePodCrashLooping' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubePodCrashLooping/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.pod }}/{{ $labels.container }}' + description: 'Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is in waiting state (reason: "CrashLoopBackOff").' + info: 'Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is in waiting state (reason: "CrashLoopBackOff").' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodcrashlooping' + summary: 'Pod is crash looping.' + title: 'Pod is crash looping. namespace:{{ $labels.namespace }} pod:{{ $labels.pod }} container:{{ $labels.container }}' + } + expression: 'max_over_time(kube_pod_container_status_waiting_reason{job="kube-state-metrics",reason="CrashLoopBackOff"}[5m]) >= 1' + for: 'PT15M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubePodNotReady' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubePodNotReady/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.pod }}' + description: 'Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than 15 minutes.' + info: 'Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than 15 minutes.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready' + summary: 'Pod has been in a non-ready state for more than 15 minutes.' + title: 'Pod has been in a non-ready state for more than 15 minutes. namespace:{{ $labels.namespace }} pod:{{ $labels.pod }}' + } + expression: 'sum by (namespace, pod, cluster) (max by (namespace, pod, cluster) (kube_pod_status_phase{job="kube-state-metrics",phase=~"Pending|Unknown|Failed"}) * on (namespace, pod, cluster) group_left (owner_kind) topk by (namespace, pod, cluster) (1, max by (namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"}))) > 0' + for: 'PT15M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeDeploymentGenerationMismatch' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeDeploymentGenerationMismatch/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.deployment }}' + description: 'Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment }} does not match, this indicates that the Deployment has failed but has not been rolled back.' + info: 'Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment }} does not match, this indicates that the Deployment has failed but has not been rolled back.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentgenerationmismatch' + summary: 'Deployment generation mismatch due to possible roll-back' + title: 'Deployment generation mismatch due to possible roll-back namespace:{{ $labels.namespace }} deployment:{{ $labels.deployment }}' + } + expression: 'kube_deployment_status_observed_generation{job="kube-state-metrics"} != kube_deployment_metadata_generation{job="kube-state-metrics"}' + for: 'PT15M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeDeploymentReplicasMismatch' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeDeploymentReplicasMismatch/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.deployment }}' + description: 'Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has not matched the expected number of replicas for longer than 30 minutes.' + info: 'Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has not matched the expected number of replicas for longer than 30 minutes.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentreplicasmismatch' + summary: 'Deployment has not matched the expected number of replicas.' + title: 'Deployment has not matched the expected number of replicas. namespace:{{ $labels.namespace }} deployment:{{ $labels.deployment }}' + } + expression: '(kube_deployment_spec_replicas{job="kube-state-metrics"} > kube_deployment_status_replicas_available{job="kube-state-metrics"}) and (changes(kube_deployment_status_replicas_updated{job="kube-state-metrics"}[10m]) == 0)' + for: 'PT30M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeDeploymentRolloutStuck' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeDeploymentRolloutStuck/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.deployment }}' + description: 'Rollout of deployment {{ $labels.namespace }}/{{ $labels.deployment }} is not progressing for longer than 30 minutes.' + info: 'Rollout of deployment {{ $labels.namespace }}/{{ $labels.deployment }} is not progressing for longer than 30 minutes.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentrolloutstuck' + summary: 'Deployment rollout is not progressing.' + title: 'Deployment rollout is not progressing. namespace:{{ $labels.namespace }} deployment:{{ $labels.deployment }}' + } + expression: 'kube_deployment_status_condition{condition="Progressing",job="kube-state-metrics",status="false"} != 0' + for: 'PT30M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeStatefulSetReplicasMismatch' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeStatefulSetReplicasMismatch/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.statefulset }}' + description: 'StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has not matched the expected number of replicas for longer than 15 minutes.' + info: 'StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has not matched the expected number of replicas for longer than 15 minutes.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetreplicasmismatch' + summary: 'StatefulSet has not matched the expected number of replicas.' + title: 'StatefulSet has not matched the expected number of replicas. namespace:{{ $labels.namespace }} statefulset:{{ $labels.statefulset }}' + } + expression: '(kube_statefulset_status_replicas_ready{job="kube-state-metrics"} != kube_statefulset_status_replicas{job="kube-state-metrics"}) and (changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics"}[10m]) == 0)' + for: 'PT15M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeStatefulSetGenerationMismatch' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeStatefulSetGenerationMismatch/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.statefulset }}' + description: 'StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset }} does not match, this indicates that the StatefulSet has failed but has not been rolled back.' + info: 'StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset }} does not match, this indicates that the StatefulSet has failed but has not been rolled back.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetgenerationmismatch' + summary: 'StatefulSet generation mismatch due to possible roll-back' + title: 'StatefulSet generation mismatch due to possible roll-back namespace:{{ $labels.namespace }} statefulset:{{ $labels.statefulset }}' + } + expression: 'kube_statefulset_status_observed_generation{job="kube-state-metrics"} != kube_statefulset_metadata_generation{job="kube-state-metrics"}' + for: 'PT15M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeStatefulSetUpdateNotRolledOut' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeStatefulSetUpdateNotRolledOut/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.statefulset }}' + description: 'StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been rolled out.' + info: 'StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been rolled out.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetupdatenotrolledout' + summary: 'StatefulSet update has not been rolled out.' + title: 'StatefulSet update has not been rolled out. namespace:{{ $labels.namespace }} statefulset:{{ $labels.statefulset }}' + } + expression: '(max by (namespace, statefulset, job, cluster) (kube_statefulset_status_current_revision{job="kube-state-metrics"} unless kube_statefulset_status_update_revision{job="kube-state-metrics"}) * (kube_statefulset_replicas{job="kube-state-metrics"} != kube_statefulset_status_replicas_updated{job="kube-state-metrics"})) and (changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics"}[5m]) == 0)' + for: 'PT15M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeDaemonSetRolloutStuck' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeDaemonSetRolloutStuck/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.daemonset }}' + description: 'DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} has not finished or progressed for at least 15 minutes.' + info: 'DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} has not finished or progressed for at least 15 minutes.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetrolloutstuck' + summary: 'DaemonSet rollout is stuck.' + title: 'DaemonSet rollout is stuck. namespace:{{ $labels.namespace }} daemonset:{{ $labels.daemonset }}' + } + expression: '((kube_daemonset_status_current_number_scheduled{job="kube-state-metrics"} != kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}) or (kube_daemonset_status_number_misscheduled{job="kube-state-metrics"} != 0) or (kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics"} != kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}) or (kube_daemonset_status_number_available{job="kube-state-metrics"} != kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"})) and (changes(kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics"}[5m]) == 0)' + for: 'PT15M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeContainerWaiting' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeContainerWaiting/{{ $labels.cluster }}/{{ $labels.pod }}/{{ $labels.namespace }}/{{ $labels.container }}' + description: 'pod/{{ $labels.pod }} in namespace {{ $labels.namespace }} on container {{ $labels.container}} has been in waiting state for longer than 1 hour.' + info: 'pod/{{ $labels.pod }} in namespace {{ $labels.namespace }} on container {{ $labels.container}} has been in waiting state for longer than 1 hour.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecontainerwaiting' + summary: 'Pod container waiting longer than 1 hour' + title: 'Pod container waiting longer than 1 hour pod:{{ $labels.pod }} namespace:{{ $labels.namespace }} container:{{ $labels.container }}' + } + expression: 'sum by (namespace, pod, container, cluster) (kube_pod_container_status_waiting_reason{job="kube-state-metrics"}) > 0' + for: 'PT1H' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeDaemonSetNotScheduled' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeDaemonSetNotScheduled/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.daemonset }}' + description: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are not scheduled.' + info: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are not scheduled.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetnotscheduled' + summary: 'DaemonSet pods are not scheduled.' + title: 'DaemonSet pods are not scheduled. namespace:{{ $labels.namespace }} daemonset:{{ $labels.daemonset }}' + } + expression: 'kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"} - kube_daemonset_status_current_number_scheduled{job="kube-state-metrics"} > 0' + for: 'PT10M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeDaemonSetMisScheduled' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeDaemonSetMisScheduled/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.daemonset }}' + description: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are running where they are not supposed to run.' + info: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are running where they are not supposed to run.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetmisscheduled' + summary: 'DaemonSet pods are misscheduled.' + title: 'DaemonSet pods are misscheduled. namespace:{{ $labels.namespace }} daemonset:{{ $labels.daemonset }}' + } + expression: 'kube_daemonset_status_number_misscheduled{job="kube-state-metrics"} > 0' + for: 'PT15M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeJobNotCompleted' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeJobNotCompleted/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.job_name }}' + description: 'Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than {{ "43200" | humanizeDuration }} to complete.' + info: 'Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than {{ "43200" | humanizeDuration }} to complete.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubejobnotcompleted' + summary: 'Job did not complete in time' + title: 'Job did not complete in time namespace:{{ $labels.namespace }} job_name:{{ $labels.job_name }}' + } + expression: 'time() - max by (namespace, job_name, cluster) (kube_job_status_start_time{job="kube-state-metrics"} and kube_job_status_active{job="kube-state-metrics"} > 0) > 43200' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeJobFailed' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeJobFailed/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.job_name }}' + description: 'Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete. Removing failed job after investigation should clear this alert.' + info: 'Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete. Removing failed job after investigation should clear this alert.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubejobfailed' + summary: 'Job failed to complete.' + title: 'Job failed to complete. namespace:{{ $labels.namespace }} job_name:{{ $labels.job_name }}' + } + expression: 'kube_job_failed{job="kube-state-metrics"} > 0' + for: 'PT15M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeHpaReplicasMismatch' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeHpaReplicasMismatch/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }}' + description: 'HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has not matched the desired number of replicas for longer than 15 minutes.' + info: 'HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has not matched the desired number of replicas for longer than 15 minutes.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubehpareplicasmismatch' + summary: 'HPA has not matched desired number of replicas.' + title: 'HPA has not matched desired number of replicas. namespace:{{ $labels.namespace }} horizontalpodautoscaler:{{ $labels.horizontalpodautoscaler }}' + } + expression: '(kube_horizontalpodautoscaler_status_desired_replicas{job="kube-state-metrics"} != kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}) and (kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"} > kube_horizontalpodautoscaler_spec_min_replicas{job="kube-state-metrics"}) and (kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"} < kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics"}) and changes(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}[15m]) == 0' + for: 'PT15M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeHpaMaxedOut' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeHpaMaxedOut/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }}' + description: 'HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has been running at max replicas for longer than 15 minutes.' + info: 'HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has been running at max replicas for longer than 15 minutes.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubehpamaxedout' + summary: 'HPA is running at max replicas' + title: 'HPA is running at max replicas namespace:{{ $labels.namespace }} horizontalpodautoscaler:{{ $labels.horizontalpodautoscaler }}' + } + expression: 'kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"} == kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics"}' + for: 'PT15M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + ] + scopes: [ + azureMonitoring + ] + } +} + +resource svcKubernetesResources 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'svc-kubernetes-resources' + location: location + properties: { + interval: 'PT1M' + rules: [ + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeCPUOvercommit' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeCPUOvercommit/{{ $labels.cluster }}' + description: 'Cluster {{ $labels.cluster }} has overcommitted CPU resource requests for Pods by {{ $value }} CPU shares and cannot tolerate node failure.' + info: 'Cluster {{ $labels.cluster }} has overcommitted CPU resource requests for Pods by {{ $value }} CPU shares and cannot tolerate node failure.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecpuovercommit' + summary: 'Cluster has overcommitted CPU resource requests.' + title: 'Cluster has overcommitted CPU resource requests. cluster:{{ $labels.cluster }}' + } + expression: 'sum by (cluster) (namespace_cpu:kube_pod_container_resource_requests:sum) - (sum by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) - max by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"})) > 0 and (sum by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) - max by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"})) > 0' + for: 'PT10M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeMemoryOvercommit' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeMemoryOvercommit/{{ $labels.cluster }}' + description: 'Cluster {{ $labels.cluster }} has overcommitted memory resource requests for Pods by {{ $value | humanize }} bytes and cannot tolerate node failure.' + info: 'Cluster {{ $labels.cluster }} has overcommitted memory resource requests for Pods by {{ $value | humanize }} bytes and cannot tolerate node failure.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryovercommit' + summary: 'Cluster has overcommitted memory resource requests.' + title: 'Cluster has overcommitted memory resource requests. cluster:{{ $labels.cluster }}' + } + expression: 'sum by (cluster) (namespace_memory:kube_pod_container_resource_requests:sum) - (sum by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="memory"}) - max by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="memory"})) > 0 and (sum by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="memory"}) - max by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="memory"})) > 0' + for: 'PT10M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeCPUQuotaOvercommit' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeCPUQuotaOvercommit/{{ $labels.cluster }}' + description: 'Cluster {{ $labels.cluster }} has overcommitted CPU resource requests for Namespaces.' + info: 'Cluster {{ $labels.cluster }} has overcommitted CPU resource requests for Namespaces.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecpuquotaovercommit' + summary: 'Cluster has overcommitted CPU resource requests.' + title: 'Cluster has overcommitted CPU resource requests. cluster:{{ $labels.cluster }}' + } + expression: 'sum by (cluster) (min without (resource) (kube_resourcequota{job="kube-state-metrics",resource=~"(cpu|requests.cpu)",type="hard"})) / sum by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) > 1.5' + for: 'PT5M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeMemoryQuotaOvercommit' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeMemoryQuotaOvercommit/{{ $labels.cluster }}' + description: 'Cluster {{ $labels.cluster }} has overcommitted memory resource requests for Namespaces.' + info: 'Cluster {{ $labels.cluster }} has overcommitted memory resource requests for Namespaces.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryquotaovercommit' + summary: 'Cluster has overcommitted memory resource requests.' + title: 'Cluster has overcommitted memory resource requests. cluster:{{ $labels.cluster }}' + } + expression: 'sum by (cluster) (min without (resource) (kube_resourcequota{job="kube-state-metrics",resource=~"(memory|requests.memory)",type="hard"})) / sum by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="memory"}) > 1.5' + for: 'PT5M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeQuotaAlmostFull' + enabled: true + labels: { + severity: 'info' + } + annotations: { + correlationId: 'KubeQuotaAlmostFull/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.resource }}' + description: 'Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.' + info: 'Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotaalmostfull' + summary: 'Namespace quota is going to be full.' + title: 'Namespace quota is going to be full. namespace:{{ $labels.namespace }} resource:{{ $labels.resource }}' + } + expression: 'kube_resourcequota{job="kube-state-metrics",type="used"} / ignoring (instance, job, type) (kube_resourcequota{job="kube-state-metrics",type="hard"} > 0) > 0.9 < 1' + for: 'PT15M' + severity: severityCeiling > 0 ? max(4, severityCeiling) : 4 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeQuotaFullyUsed' + enabled: true + labels: { + severity: 'info' + } + annotations: { + correlationId: 'KubeQuotaFullyUsed/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.resource }}' + description: 'Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.' + info: 'Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotafullyused' + summary: 'Namespace quota is fully used.' + title: 'Namespace quota is fully used. namespace:{{ $labels.namespace }} resource:{{ $labels.resource }}' + } + expression: 'kube_resourcequota{job="kube-state-metrics",type="used"} / ignoring (instance, job, type) (kube_resourcequota{job="kube-state-metrics",type="hard"} > 0) == 1' + for: 'PT15M' + severity: severityCeiling > 0 ? max(4, severityCeiling) : 4 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeQuotaExceeded' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeQuotaExceeded/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.resource }}' + description: 'Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.' + info: 'Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotaexceeded' + summary: 'Namespace quota has exceeded the limits.' + title: 'Namespace quota has exceeded the limits. namespace:{{ $labels.namespace }} resource:{{ $labels.resource }}' + } + expression: 'kube_resourcequota{job="kube-state-metrics",type="used"} / ignoring (instance, job, type) (kube_resourcequota{job="kube-state-metrics",type="hard"} > 0) > 1' + for: 'PT15M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'CPUThrottlingHigh' + enabled: true + labels: { + severity: 'info' + } + annotations: { + correlationId: 'CPUThrottlingHigh/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.container }}/{{ $labels.pod }}' + description: '{{ $value | humanizePercentage }} throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod }}.' + info: '{{ $value | humanizePercentage }} throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod }}.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/cputhrottlinghigh' + summary: 'Processes experience elevated CPU throttling.' + title: 'Processes experience elevated CPU throttling. namespace:{{ $labels.namespace }} container:{{ $labels.container }} pod:{{ $labels.pod }}' + } + expression: 'sum by (cluster, container, pod, namespace) (increase(container_cpu_cfs_throttled_periods_total{container!=""}[5m])) / sum by (cluster, container, pod, namespace) (increase(container_cpu_cfs_periods_total[5m])) > (25 / 100)' + for: 'PT15M' + severity: severityCeiling > 0 ? max(4, severityCeiling) : 4 + } + ] + scopes: [ + azureMonitoring + ] + } +} + +resource svcKubernetesStorage 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'svc-kubernetes-storage' + location: location + properties: { + interval: 'PT1M' + rules: [ + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubePersistentVolumeFillingUp' + enabled: true + labels: { + severity: 'critical' + } + annotations: { + correlationId: 'KubePersistentVolumeFillingUp/{{ $labels.cluster }}/{{ $labels.persistentvolumeclaim }}/{{ $labels.namespace }}' + description: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} is only {{ $value | humanizePercentage }} free.' + info: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} is only {{ $value | humanizePercentage }} free.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup' + summary: 'PersistentVolume is filling up.' + title: 'PersistentVolume is filling up. persistentvolumeclaim:{{ $labels.persistentvolumeclaim }} namespace:{{ $labels.namespace }} cluster:{{ $labels.cluster }}' + } + expression: '(kubelet_volume_stats_available_bytes{job="kubelet",metrics_path="/metrics"} / kubelet_volume_stats_capacity_bytes{job="kubelet",metrics_path="/metrics"}) < 0.03 and kubelet_volume_stats_used_bytes{job="kubelet",metrics_path="/metrics"} > 0 unless on (cluster, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{access_mode="ReadOnlyMany"} == 1 unless on (cluster, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1' + for: 'PT1M' + severity: severityCeiling > 0 ? max(2, severityCeiling) : 2 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubePersistentVolumeFillingUp' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubePersistentVolumeFillingUp/{{ $labels.cluster }}/{{ $labels.persistentvolumeclaim }}/{{ $labels.namespace }}' + description: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} is expected to fill up within four days. Currently {{ $value | humanizePercentage }} is available.' + info: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} is expected to fill up within four days. Currently {{ $value | humanizePercentage }} is available.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup' + summary: 'PersistentVolume is filling up.' + title: 'PersistentVolume is filling up. persistentvolumeclaim:{{ $labels.persistentvolumeclaim }} namespace:{{ $labels.namespace }} cluster:{{ $labels.cluster }}' + } + expression: '(kubelet_volume_stats_available_bytes{job="kubelet",metrics_path="/metrics"} / kubelet_volume_stats_capacity_bytes{job="kubelet",metrics_path="/metrics"}) < 0.15 and kubelet_volume_stats_used_bytes{job="kubelet",metrics_path="/metrics"} > 0 and predict_linear(kubelet_volume_stats_available_bytes{job="kubelet",metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 unless on (cluster, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{access_mode="ReadOnlyMany"} == 1 unless on (cluster, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1' + for: 'PT1H' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubePersistentVolumeInodesFillingUp' + enabled: true + labels: { + severity: 'critical' + } + annotations: { + correlationId: 'KubePersistentVolumeInodesFillingUp/{{ $labels.cluster }}/{{ $labels.persistentvolumeclaim }}/{{ $labels.namespace }}' + description: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} only has {{ $value | humanizePercentage }} free inodes.' + info: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} only has {{ $value | humanizePercentage }} free inodes.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeinodesfillingup' + summary: 'PersistentVolumeInodes are filling up.' + title: 'PersistentVolumeInodes are filling up. persistentvolumeclaim:{{ $labels.persistentvolumeclaim }} namespace:{{ $labels.namespace }} cluster:{{ $labels.cluster }}' + } + expression: '(kubelet_volume_stats_inodes_free{job="kubelet",metrics_path="/metrics"} / kubelet_volume_stats_inodes{job="kubelet",metrics_path="/metrics"}) < 0.03 and kubelet_volume_stats_inodes_used{job="kubelet",metrics_path="/metrics"} > 0 unless on (cluster, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{access_mode="ReadOnlyMany"} == 1 unless on (cluster, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1' + for: 'PT1M' + severity: severityCeiling > 0 ? max(2, severityCeiling) : 2 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubePersistentVolumeInodesFillingUp' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubePersistentVolumeInodesFillingUp/{{ $labels.cluster }}/{{ $labels.persistentvolumeclaim }}/{{ $labels.namespace }}' + description: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} is expected to run out of inodes within four days. Currently {{ $value | humanizePercentage }} of its inodes are free.' + info: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} is expected to run out of inodes within four days. Currently {{ $value | humanizePercentage }} of its inodes are free.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeinodesfillingup' + summary: 'PersistentVolumeInodes are filling up.' + title: 'PersistentVolumeInodes are filling up. persistentvolumeclaim:{{ $labels.persistentvolumeclaim }} namespace:{{ $labels.namespace }} cluster:{{ $labels.cluster }}' + } + expression: '(kubelet_volume_stats_inodes_free{job="kubelet",metrics_path="/metrics"} / kubelet_volume_stats_inodes{job="kubelet",metrics_path="/metrics"}) < 0.15 and kubelet_volume_stats_inodes_used{job="kubelet",metrics_path="/metrics"} > 0 and predict_linear(kubelet_volume_stats_inodes_free{job="kubelet",metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 unless on (cluster, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{access_mode="ReadOnlyMany"} == 1 unless on (cluster, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1' + for: 'PT1H' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubePersistentVolumeErrors' + enabled: true + labels: { + severity: 'critical' + } + annotations: { + correlationId: 'KubePersistentVolumeErrors/{{ $labels.cluster }}/{{ $labels.persistentvolume }}/{{ $labels.phase }}' + description: 'The persistent volume {{ $labels.persistentvolume }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} has status {{ $labels.phase }}.' + info: 'The persistent volume {{ $labels.persistentvolume }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} has status {{ $labels.phase }}.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeerrors' + summary: 'PersistentVolume is having issues with provisioning.' + title: 'PersistentVolume is having issues with provisioning. persistentvolume:{{ $labels.persistentvolume }} cluster:{{ $labels.cluster }} phase:{{ $labels.phase }}' + } + expression: 'kube_persistentvolume_status_phase{job="kube-state-metrics",phase=~"Failed|Pending"} > 0' + for: 'PT5M' + severity: severityCeiling > 0 ? max(2, severityCeiling) : 2 + } + ] + scopes: [ + azureMonitoring + ] + } +} + +resource svcKubernetesSystem 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'svc-kubernetes-system' + location: location + properties: { + interval: 'PT1M' + rules: [ + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeVersionMismatch' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeVersionMismatch/{{ $labels.cluster }}' + description: 'There are {{ $value }} different semantic versions of Kubernetes components running.' + info: 'There are {{ $value }} different semantic versions of Kubernetes components running.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeversionmismatch' + summary: 'Different semantic versions of Kubernetes components running.' + title: 'Different semantic versions of Kubernetes components running.' + } + expression: 'count by (cluster) (count by (git_version, cluster) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"}, "git_version", "$1", "git_version", "(v[0-9]*.[0-9]*).*"))) > 1' + for: 'PT1H30M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeClientErrors' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeClientErrors/{{ $labels.cluster }}/{{ $labels.job }}/{{ $labels.instance }}' + description: 'Kubernetes API server client \'{{ $labels.job }}/{{ $labels.instance }}\' is experiencing {{ $value | humanizePercentage }} errors.\'' + info: 'Kubernetes API server client \'{{ $labels.job }}/{{ $labels.instance }}\' is experiencing {{ $value | humanizePercentage }} errors.\'' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclienterrors' + summary: 'Kubernetes API server client is experiencing errors.' + title: 'Kubernetes API server client is experiencing errors. job:{{ $labels.job }} instance:{{ $labels.instance }}' + } + expression: '(sum by (cluster, instance, job, namespace) (rate(rest_client_requests_total{code=~"5..",job="controlplane-apiserver"}[5m])) / sum by (cluster, instance, job, namespace) (rate(rest_client_requests_total{job="controlplane-apiserver"}[5m]))) > 0.01' + for: 'PT15M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + ] + scopes: [ + azureMonitoring + ] + } +} + +resource svcKubeApiserverSlos 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'svc-kube-apiserver-slos' + location: location + properties: { + interval: 'PT1M' + rules: [ + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeAPIErrorBudgetBurn' + enabled: true + labels: { + long: '1h' + severity: 'critical' + short: '5m' + } + annotations: { + correlationId: 'KubeAPIErrorBudgetBurn/{{ $labels.cluster }}' + description: 'The API server is burning too much error budget.' + info: 'The API server is burning too much error budget.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn' + summary: 'The API server is burning too much error budget.' + title: 'The API server is burning too much error budget.' + } + expression: 'sum(apiserver_request:burnrate1h) > (14.4 * 0.01) and sum(apiserver_request:burnrate5m) > (14.4 * 0.01)' + for: 'PT2M' + severity: severityCeiling > 0 ? max(2, severityCeiling) : 2 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeAPIErrorBudgetBurn' + enabled: true + labels: { + long: '6h' + severity: 'critical' + short: '30m' + } + annotations: { + correlationId: 'KubeAPIErrorBudgetBurn/{{ $labels.cluster }}' + description: 'The API server is burning too much error budget.' + info: 'The API server is burning too much error budget.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn' + summary: 'The API server is burning too much error budget.' + title: 'The API server is burning too much error budget.' + } + expression: 'sum(apiserver_request:burnrate6h) > (6 * 0.01) and sum(apiserver_request:burnrate30m) > (6 * 0.01)' + for: 'PT15M' + severity: severityCeiling > 0 ? max(2, severityCeiling) : 2 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeAPIErrorBudgetBurn' + enabled: true + labels: { + long: '1d' + severity: 'warning' + short: '2h' + } + annotations: { + correlationId: 'KubeAPIErrorBudgetBurn/{{ $labels.cluster }}' + description: 'The API server is burning too much error budget.' + info: 'The API server is burning too much error budget.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn' + summary: 'The API server is burning too much error budget.' + title: 'The API server is burning too much error budget.' + } + expression: 'sum(apiserver_request:burnrate1d) > (3 * 0.01) and sum(apiserver_request:burnrate2h) > (3 * 0.01)' + for: 'PT1H' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeAPIErrorBudgetBurn' + enabled: true + labels: { + long: '3d' + severity: 'warning' + short: '6h' + } + annotations: { + correlationId: 'KubeAPIErrorBudgetBurn/{{ $labels.cluster }}' + description: 'The API server is burning too much error budget.' + info: 'The API server is burning too much error budget.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn' + summary: 'The API server is burning too much error budget.' + title: 'The API server is burning too much error budget.' + } + expression: 'sum(apiserver_request:burnrate3d) > (1 * 0.01) and sum(apiserver_request:burnrate6h) > (1 * 0.01)' + for: 'PT3H' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + ] + scopes: [ + azureMonitoring + ] + } +} + +resource svcKubernetesSystemApiserver 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'svc-kubernetes-system-apiserver' + location: location + properties: { + interval: 'PT1M' + rules: [ + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeClientCertificateExpiration' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeClientCertificateExpiration/{{ $labels.cluster }}' + description: 'A client certificate used to authenticate to kubernetes apiserver is expiring in less than 7.0 days.' + info: 'A client certificate used to authenticate to kubernetes apiserver is expiring in less than 7.0 days.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration' + summary: 'Client certificate is about to expire.' + title: 'Client certificate is about to expire.' + } + expression: 'apiserver_client_certificate_expiration_seconds_count{job="controlplane-apiserver"} > 0 and on (job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="controlplane-apiserver"}[5m]))) < 604800' + for: 'PT5M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeClientCertificateExpiration' + enabled: true + labels: { + severity: 'critical' + } + annotations: { + correlationId: 'KubeClientCertificateExpiration/{{ $labels.cluster }}' + description: 'A client certificate used to authenticate to kubernetes apiserver is expiring in less than 24.0 hours.' + info: 'A client certificate used to authenticate to kubernetes apiserver is expiring in less than 24.0 hours.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration' + summary: 'Client certificate is about to expire.' + title: 'Client certificate is about to expire.' + } + expression: 'apiserver_client_certificate_expiration_seconds_count{job="controlplane-apiserver"} > 0 and on (job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="controlplane-apiserver"}[5m]))) < 86400' + for: 'PT5M' + severity: severityCeiling > 0 ? max(2, severityCeiling) : 2 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeAggregatedAPIErrors' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeAggregatedAPIErrors/{{ $labels.cluster }}/{{ $labels.name }}/{{ $labels.namespace }}' + description: 'Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has reported errors. It has appeared unavailable {{ $value | humanize }} times averaged over the past 10m.' + info: 'Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has reported errors. It has appeared unavailable {{ $value | humanize }} times averaged over the past 10m.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapierrors' + summary: 'Kubernetes aggregated API has reported errors.' + title: 'Kubernetes aggregated API has reported errors. name:{{ $labels.name }} namespace:{{ $labels.namespace }}' + } + expression: 'sum by (name, namespace, cluster) (increase(aggregator_unavailable_apiservice_total{job="controlplane-apiserver"}[10m])) > 4' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeAggregatedAPIDown' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeAggregatedAPIDown/{{ $labels.cluster }}/{{ $labels.name }}/{{ $labels.namespace }}' + description: 'Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has been only {{ $value | humanize }}% available over the last 10m.' + info: 'Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has been only {{ $value | humanize }}% available over the last 10m.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapidown' + summary: 'Kubernetes aggregated API is down.' + title: 'Kubernetes aggregated API is down. name:{{ $labels.name }} namespace:{{ $labels.namespace }}' + } + expression: '(1 - max by (name, namespace, cluster) (avg_over_time(aggregator_unavailable_apiservice{job="controlplane-apiserver"}[10m]))) * 100 < 85' + for: 'PT5M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeAPIDown' + enabled: true + labels: { + severity: 'critical' + } + annotations: { + correlationId: 'KubeAPIDown/{{ $labels.cluster }}' + description: 'KubeAPI has disappeared from Prometheus target discovery.' + info: 'KubeAPI has disappeared from Prometheus target discovery.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapidown' + summary: 'Target disappeared from Prometheus target discovery.' + title: 'Target disappeared from Prometheus target discovery.' + } + expression: 'count by (cluster) (up{job="controlplane-apiserver"} == 1) == 0' + for: 'PT15M' + severity: severityCeiling > 0 ? max(2, severityCeiling) : 2 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeAPITerminatedRequests' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeAPITerminatedRequests/{{ $labels.cluster }}' + description: 'The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.' + info: 'The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapiterminatedrequests' + summary: 'The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.' + title: 'The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.' + } + expression: 'sum(rate(apiserver_request_terminations_total{job="controlplane-apiserver"}[10m])) / (sum(rate(apiserver_request_total{job="controlplane-apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="controlplane-apiserver"}[10m]))) > 0.2' + for: 'PT5M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + ] + scopes: [ + azureMonitoring + ] + } +} + +resource svcKubernetesSystemKubelet 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'svc-kubernetes-system-kubelet' + location: location + properties: { + interval: 'PT1M' + rules: [ + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeNodeNotReady' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeNodeNotReady/{{ $labels.cluster }}/{{ $labels.node }}' + description: '{{ $labels.node }} has been unready for more than 15 minutes.' + info: '{{ $labels.node }} has been unready for more than 15 minutes.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodenotready' + summary: 'Node is not ready.' + title: 'Node is not ready. node:{{ $labels.node }}' + } + expression: 'kube_node_status_condition{condition="Ready",job="kube-state-metrics",status="true"} == 0' + for: 'PT15M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeNodeUnreachable' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeNodeUnreachable/{{ $labels.cluster }}/{{ $labels.node }}' + description: '{{ $labels.node }} is unreachable and some workloads may be rescheduled.' + info: '{{ $labels.node }} is unreachable and some workloads may be rescheduled.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodeunreachable' + summary: 'Node is unreachable.' + title: 'Node is unreachable. node:{{ $labels.node }}' + } + expression: '(kube_node_spec_taint{effect="NoSchedule",job="kube-state-metrics",key="node.kubernetes.io/unreachable"} unless ignoring (key, value) kube_node_spec_taint{job="kube-state-metrics",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) == 1' + for: 'PT15M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeletTooManyPods' + enabled: true + labels: { + severity: 'info' + } + annotations: { + correlationId: 'KubeletTooManyPods/{{ $labels.cluster }}/{{ $labels.node }}' + description: 'Kubelet \'{{ $labels.node }}\' is running at {{ $value | humanizePercentage }} of its Pod capacity.' + info: 'Kubelet \'{{ $labels.node }}\' is running at {{ $value | humanizePercentage }} of its Pod capacity.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubelettoomanypods' + summary: 'Kubelet is running at capacity.' + title: 'Kubelet is running at capacity. node:{{ $labels.node }}' + } + expression: 'count by (cluster, node) ((kube_pod_status_phase{job="kube-state-metrics",phase="Running"} == 1) * on (instance, pod, namespace, cluster) group_left (node) topk by (instance, pod, namespace, cluster) (1, kube_pod_info{job="kube-state-metrics"})) / max by (cluster, node) (kube_node_status_capacity{job="kube-state-metrics",resource="pods"} != 1) > 0.95' + for: 'PT15M' + severity: severityCeiling > 0 ? max(4, severityCeiling) : 4 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeNodeReadinessFlapping' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeNodeReadinessFlapping/{{ $labels.cluster }}/{{ $labels.node }}' + description: 'The readiness status of node {{ $labels.node }} has changed {{ $value }} times in the last 15 minutes.' + info: 'The readiness status of node {{ $labels.node }} has changed {{ $value }} times in the last 15 minutes.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodereadinessflapping' + summary: 'Node readiness status is flapping.' + title: 'Node readiness status is flapping. node:{{ $labels.node }}' + } + expression: 'sum by (cluster, node) (changes(kube_node_status_condition{condition="Ready",job="kube-state-metrics",status="true"}[15m])) > 2' + for: 'PT15M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeletPlegDurationHigh' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeletPlegDurationHigh/{{ $labels.cluster }}/{{ $labels.node }}' + description: 'The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{ $value }} seconds on node {{ $labels.node }}.' + info: 'The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{ $value }} seconds on node {{ $labels.node }}.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletplegdurationhigh' + summary: 'Kubelet Pod Lifecycle Event Generator is taking too long to relist.' + title: 'Kubelet Pod Lifecycle Event Generator is taking too long to relist. node:{{ $labels.node }}' + } + expression: 'node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10' + for: 'PT5M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeletPodStartUpLatencyHigh' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeletPodStartUpLatencyHigh/{{ $labels.cluster }}/{{ $labels.node }}' + description: 'Kubelet Pod startup 99th percentile latency is {{ $value }} seconds on node {{ $labels.node }}.' + info: 'Kubelet Pod startup 99th percentile latency is {{ $value }} seconds on node {{ $labels.node }}.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletpodstartuplatencyhigh' + summary: 'Kubelet Pod startup latency is too high.' + title: 'Kubelet Pod startup latency is too high. node:{{ $labels.node }}' + } + expression: 'histogram_quantile(0.99, sum by (cluster, instance, le) (rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet",metrics_path="/metrics"}[5m]))) * on (cluster, instance) group_left (node) kubelet_node_name{job="kubelet",metrics_path="/metrics"} > 60' + for: 'PT15M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeletClientCertificateExpiration' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeletClientCertificateExpiration/{{ $labels.cluster }}/{{ $labels.node }}' + description: 'Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.' + info: 'Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletclientcertificateexpiration' + summary: 'Kubelet client certificate is about to expire.' + title: 'Kubelet client certificate is about to expire. node:{{ $labels.node }}' + } + expression: 'kubelet_certificate_manager_client_ttl_seconds < 604800' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeletClientCertificateExpiration' + enabled: true + labels: { + severity: 'critical' + } + annotations: { + correlationId: 'KubeletClientCertificateExpiration/{{ $labels.cluster }}/{{ $labels.node }}' + description: 'Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.' + info: 'Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletclientcertificateexpiration' + summary: 'Kubelet client certificate is about to expire.' + title: 'Kubelet client certificate is about to expire. node:{{ $labels.node }}' + } + expression: 'kubelet_certificate_manager_client_ttl_seconds < 86400' + severity: severityCeiling > 0 ? max(2, severityCeiling) : 2 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeletServerCertificateExpiration' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeletServerCertificateExpiration/{{ $labels.cluster }}/{{ $labels.node }}' + description: 'Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.' + info: 'Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletservercertificateexpiration' + summary: 'Kubelet server certificate is about to expire.' + title: 'Kubelet server certificate is about to expire. node:{{ $labels.node }}' + } + expression: 'kubelet_certificate_manager_server_ttl_seconds < 604800' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeletServerCertificateExpiration' + enabled: true + labels: { + severity: 'critical' + } + annotations: { + correlationId: 'KubeletServerCertificateExpiration/{{ $labels.cluster }}/{{ $labels.node }}' + description: 'Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.' + info: 'Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletservercertificateexpiration' + summary: 'Kubelet server certificate is about to expire.' + title: 'Kubelet server certificate is about to expire. node:{{ $labels.node }}' + } + expression: 'kubelet_certificate_manager_server_ttl_seconds < 86400' + severity: severityCeiling > 0 ? max(2, severityCeiling) : 2 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeletClientCertificateRenewalErrors' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeletClientCertificateRenewalErrors/{{ $labels.cluster }}/{{ $labels.node }}' + description: 'Kubelet on node {{ $labels.node }} has failed to renew its client certificate ({{ $value | humanize }} errors in the last 5 minutes).' + info: 'Kubelet on node {{ $labels.node }} has failed to renew its client certificate ({{ $value | humanize }} errors in the last 5 minutes).' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletclientcertificaterenewalerrors' + summary: 'Kubelet has failed to renew its client certificate.' + title: 'Kubelet has failed to renew its client certificate. node:{{ $labels.node }}' + } + expression: 'increase(kubelet_certificate_manager_client_expiration_renew_errors[5m]) > 0' + for: 'PT15M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeletServerCertificateRenewalErrors' + enabled: true + labels: { + severity: 'warning' + } + annotations: { + correlationId: 'KubeletServerCertificateRenewalErrors/{{ $labels.cluster }}/{{ $labels.node }}' + description: 'Kubelet on node {{ $labels.node }} has failed to renew its server certificate ({{ $value | humanize }} errors in the last 5 minutes).' + info: 'Kubelet on node {{ $labels.node }} has failed to renew its server certificate ({{ $value | humanize }} errors in the last 5 minutes).' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletservercertificaterenewalerrors' + summary: 'Kubelet has failed to renew its server certificate.' + title: 'Kubelet has failed to renew its server certificate. node:{{ $labels.node }}' + } + expression: 'increase(kubelet_server_expiration_renew_errors[5m]) > 0' + for: 'PT15M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeletDown' + enabled: true + labels: { + severity: 'critical' + } + annotations: { + correlationId: 'KubeletDown/{{ $labels.cluster }}' + description: 'Kubelet has disappeared from Prometheus target discovery.' + info: 'Kubelet has disappeared from Prometheus target discovery.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletdown' + summary: 'Target disappeared from Prometheus target discovery.' + title: 'Target disappeared from Prometheus target discovery.' + } + expression: 'count by (cluster) (up{job="kubelet",metrics_path="/metrics"} == 1) == 0' + for: 'PT15M' + severity: severityCeiling > 0 ? max(2, severityCeiling) : 2 + } + ] + scopes: [ + azureMonitoring + ] + } +} + +resource svcKubernetesSystemScheduler 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'svc-kubernetes-system-scheduler' + location: location + properties: { + interval: 'PT1M' + rules: [ + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeSchedulerDown' + enabled: true + labels: { + severity: 'critical' + } + annotations: { + correlationId: 'KubeSchedulerDown/{{ $labels.cluster }}' + description: 'KubeScheduler has disappeared from Prometheus target discovery.' + info: 'KubeScheduler has disappeared from Prometheus target discovery.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeschedulerdown' + summary: 'Target disappeared from Prometheus target discovery.' + title: 'Target disappeared from Prometheus target discovery.' + } + expression: 'count by (cluster) (up{job="controlplane-kube-scheduler"} == 1) == 0' + for: 'PT15M' + severity: severityCeiling > 0 ? max(2, severityCeiling) : 2 + } + ] + scopes: [ + azureMonitoring + ] + } +} + +resource svcKubernetesSystemControllerManager 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'svc-kubernetes-system-controller-manager' + location: location + properties: { + interval: 'PT1M' + rules: [ + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'KubeControllerManagerDown' + enabled: true + labels: { + severity: 'critical' + } + annotations: { + correlationId: 'KubeControllerManagerDown/{{ $labels.cluster }}' + description: 'KubeControllerManager has disappeared from Prometheus target discovery.' + info: 'KubeControllerManager has disappeared from Prometheus target discovery.' + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecontrollermanagerdown' + summary: 'Target disappeared from Prometheus target discovery.' + title: 'Target disappeared from Prometheus target discovery.' + } + expression: 'count by (cluster) (up{job="controlplane-kube-controller-manager"} == 1) == 0' + for: 'PT15M' + severity: severityCeiling > 0 ? max(2, severityCeiling) : 2 + } + ] + scopes: [ + azureMonitoring + ] + } +} diff --git a/dev-infrastructure/modules/metrics/rules/generatedSvcKubePrometheusAlertingRules.bicep b/dev-infrastructure/modules/metrics/rules/generatedSvcKubePrometheusAlertingRules.bicep deleted file mode 100644 index baeacda3b15..00000000000 --- a/dev-infrastructure/modules/metrics/rules/generatedSvcKubePrometheusAlertingRules.bicep +++ /dev/null @@ -1,1642 +0,0 @@ -#disable-next-line no-unused-params -param azureMonitoring string - -#disable-next-line no-unused-params -param actionGroups array - -@description('The minimum IcM severity level (highest priority) that alerts can fire at. Alerts more critical than this ceiling will be degraded to this value. 0 means no ceiling.') -param severityCeiling int = 0 - -#disable-next-line no-unused-params -param location string = resourceGroup().location - -resource svcKubernetesApps 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { - name: 'svc-kubernetes-apps' - location: location - properties: { - interval: 'PT1M' - rules: [ - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubePodCrashLooping' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubePodCrashLooping/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.pod }}/{{ $labels.container }}' - description: 'Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is in waiting state (reason: "CrashLoopBackOff").' - info: 'Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is in waiting state (reason: "CrashLoopBackOff").' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodcrashlooping' - summary: 'Pod is crash looping.' - title: 'Pod is crash looping. namespace:{{ $labels.namespace }} pod:{{ $labels.pod }} container:{{ $labels.container }}' - } - expression: 'max_over_time(kube_pod_container_status_waiting_reason{job="kube-state-metrics",reason="CrashLoopBackOff"}[5m]) >= 1' - for: 'PT15M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubePodNotReady' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubePodNotReady/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.pod }}' - description: 'Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than 15 minutes.' - info: 'Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than 15 minutes.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready' - summary: 'Pod has been in a non-ready state for more than 15 minutes.' - title: 'Pod has been in a non-ready state for more than 15 minutes. namespace:{{ $labels.namespace }} pod:{{ $labels.pod }}' - } - expression: 'sum by (namespace, pod, cluster) (max by (namespace, pod, cluster) (kube_pod_status_phase{job="kube-state-metrics",phase=~"Pending|Unknown|Failed"}) * on (namespace, pod, cluster) group_left (owner_kind) topk by (namespace, pod, cluster) (1, max by (namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"}))) > 0' - for: 'PT15M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeDeploymentGenerationMismatch' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeDeploymentGenerationMismatch/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.deployment }}' - description: 'Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment }} does not match, this indicates that the Deployment has failed but has not been rolled back.' - info: 'Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment }} does not match, this indicates that the Deployment has failed but has not been rolled back.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentgenerationmismatch' - summary: 'Deployment generation mismatch due to possible roll-back' - title: 'Deployment generation mismatch due to possible roll-back namespace:{{ $labels.namespace }} deployment:{{ $labels.deployment }}' - } - expression: 'kube_deployment_status_observed_generation{job="kube-state-metrics"} != kube_deployment_metadata_generation{job="kube-state-metrics"}' - for: 'PT15M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeDeploymentReplicasMismatch' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeDeploymentReplicasMismatch/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.deployment }}' - description: 'Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has not matched the expected number of replicas for longer than 30 minutes.' - info: 'Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has not matched the expected number of replicas for longer than 30 minutes.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentreplicasmismatch' - summary: 'Deployment has not matched the expected number of replicas.' - title: 'Deployment has not matched the expected number of replicas. namespace:{{ $labels.namespace }} deployment:{{ $labels.deployment }}' - } - expression: '(kube_deployment_spec_replicas{job="kube-state-metrics"} > kube_deployment_status_replicas_available{job="kube-state-metrics"}) and (changes(kube_deployment_status_replicas_updated{job="kube-state-metrics"}[10m]) == 0)' - for: 'PT30M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeDeploymentRolloutStuck' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeDeploymentRolloutStuck/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.deployment }}' - description: 'Rollout of deployment {{ $labels.namespace }}/{{ $labels.deployment }} is not progressing for longer than 30 minutes.' - info: 'Rollout of deployment {{ $labels.namespace }}/{{ $labels.deployment }} is not progressing for longer than 30 minutes.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentrolloutstuck' - summary: 'Deployment rollout is not progressing.' - title: 'Deployment rollout is not progressing. namespace:{{ $labels.namespace }} deployment:{{ $labels.deployment }}' - } - expression: 'kube_deployment_status_condition{condition="Progressing",job="kube-state-metrics",status="false"} != 0' - for: 'PT30M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeStatefulSetReplicasMismatch' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeStatefulSetReplicasMismatch/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.statefulset }}' - description: 'StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has not matched the expected number of replicas for longer than 15 minutes.' - info: 'StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has not matched the expected number of replicas for longer than 15 minutes.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetreplicasmismatch' - summary: 'StatefulSet has not matched the expected number of replicas.' - title: 'StatefulSet has not matched the expected number of replicas. namespace:{{ $labels.namespace }} statefulset:{{ $labels.statefulset }}' - } - expression: '(kube_statefulset_status_replicas_ready{job="kube-state-metrics"} != kube_statefulset_status_replicas{job="kube-state-metrics"}) and (changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics"}[10m]) == 0)' - for: 'PT15M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeStatefulSetGenerationMismatch' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeStatefulSetGenerationMismatch/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.statefulset }}' - description: 'StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset }} does not match, this indicates that the StatefulSet has failed but has not been rolled back.' - info: 'StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset }} does not match, this indicates that the StatefulSet has failed but has not been rolled back.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetgenerationmismatch' - summary: 'StatefulSet generation mismatch due to possible roll-back' - title: 'StatefulSet generation mismatch due to possible roll-back namespace:{{ $labels.namespace }} statefulset:{{ $labels.statefulset }}' - } - expression: 'kube_statefulset_status_observed_generation{job="kube-state-metrics"} != kube_statefulset_metadata_generation{job="kube-state-metrics"}' - for: 'PT15M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeStatefulSetUpdateNotRolledOut' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeStatefulSetUpdateNotRolledOut/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.statefulset }}' - description: 'StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been rolled out.' - info: 'StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been rolled out.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetupdatenotrolledout' - summary: 'StatefulSet update has not been rolled out.' - title: 'StatefulSet update has not been rolled out. namespace:{{ $labels.namespace }} statefulset:{{ $labels.statefulset }}' - } - expression: '(max by (namespace, statefulset, job, cluster) (kube_statefulset_status_current_revision{job="kube-state-metrics"} unless kube_statefulset_status_update_revision{job="kube-state-metrics"}) * (kube_statefulset_replicas{job="kube-state-metrics"} != kube_statefulset_status_replicas_updated{job="kube-state-metrics"})) and (changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics"}[5m]) == 0)' - for: 'PT15M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeDaemonSetRolloutStuck' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeDaemonSetRolloutStuck/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.daemonset }}' - description: 'DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} has not finished or progressed for at least 15 minutes.' - info: 'DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} has not finished or progressed for at least 15 minutes.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetrolloutstuck' - summary: 'DaemonSet rollout is stuck.' - title: 'DaemonSet rollout is stuck. namespace:{{ $labels.namespace }} daemonset:{{ $labels.daemonset }}' - } - expression: '((kube_daemonset_status_current_number_scheduled{job="kube-state-metrics"} != kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}) or (kube_daemonset_status_number_misscheduled{job="kube-state-metrics"} != 0) or (kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics"} != kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}) or (kube_daemonset_status_number_available{job="kube-state-metrics"} != kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"})) and (changes(kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics"}[5m]) == 0)' - for: 'PT15M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeContainerWaiting' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeContainerWaiting/{{ $labels.cluster }}/{{ $labels.pod }}/{{ $labels.namespace }}/{{ $labels.container }}' - description: 'pod/{{ $labels.pod }} in namespace {{ $labels.namespace }} on container {{ $labels.container}} has been in waiting state for longer than 1 hour.' - info: 'pod/{{ $labels.pod }} in namespace {{ $labels.namespace }} on container {{ $labels.container}} has been in waiting state for longer than 1 hour.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecontainerwaiting' - summary: 'Pod container waiting longer than 1 hour' - title: 'Pod container waiting longer than 1 hour pod:{{ $labels.pod }} namespace:{{ $labels.namespace }} container:{{ $labels.container }}' - } - expression: 'sum by (namespace, pod, container, cluster) (kube_pod_container_status_waiting_reason{job="kube-state-metrics"}) > 0' - for: 'PT1H' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeDaemonSetNotScheduled' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeDaemonSetNotScheduled/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.daemonset }}' - description: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are not scheduled.' - info: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are not scheduled.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetnotscheduled' - summary: 'DaemonSet pods are not scheduled.' - title: 'DaemonSet pods are not scheduled. namespace:{{ $labels.namespace }} daemonset:{{ $labels.daemonset }}' - } - expression: 'kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"} - kube_daemonset_status_current_number_scheduled{job="kube-state-metrics"} > 0' - for: 'PT10M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeDaemonSetMisScheduled' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeDaemonSetMisScheduled/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.daemonset }}' - description: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are running where they are not supposed to run.' - info: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are running where they are not supposed to run.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetmisscheduled' - summary: 'DaemonSet pods are misscheduled.' - title: 'DaemonSet pods are misscheduled. namespace:{{ $labels.namespace }} daemonset:{{ $labels.daemonset }}' - } - expression: 'kube_daemonset_status_number_misscheduled{job="kube-state-metrics"} > 0' - for: 'PT15M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeJobNotCompleted' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeJobNotCompleted/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.job_name }}' - description: 'Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than {{ "43200" | humanizeDuration }} to complete.' - info: 'Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than {{ "43200" | humanizeDuration }} to complete.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubejobnotcompleted' - summary: 'Job did not complete in time' - title: 'Job did not complete in time namespace:{{ $labels.namespace }} job_name:{{ $labels.job_name }}' - } - expression: 'time() - max by (namespace, job_name, cluster) (kube_job_status_start_time{job="kube-state-metrics"} and kube_job_status_active{job="kube-state-metrics"} > 0) > 43200' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeJobFailed' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeJobFailed/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.job_name }}' - description: 'Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete. Removing failed job after investigation should clear this alert.' - info: 'Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete. Removing failed job after investigation should clear this alert.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubejobfailed' - summary: 'Job failed to complete.' - title: 'Job failed to complete. namespace:{{ $labels.namespace }} job_name:{{ $labels.job_name }}' - } - expression: 'kube_job_failed{job="kube-state-metrics"} > 0' - for: 'PT15M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeHpaReplicasMismatch' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeHpaReplicasMismatch/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }}' - description: 'HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has not matched the desired number of replicas for longer than 15 minutes.' - info: 'HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has not matched the desired number of replicas for longer than 15 minutes.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubehpareplicasmismatch' - summary: 'HPA has not matched desired number of replicas.' - title: 'HPA has not matched desired number of replicas. namespace:{{ $labels.namespace }} horizontalpodautoscaler:{{ $labels.horizontalpodautoscaler }}' - } - expression: '(kube_horizontalpodautoscaler_status_desired_replicas{job="kube-state-metrics"} != kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}) and (kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"} > kube_horizontalpodautoscaler_spec_min_replicas{job="kube-state-metrics"}) and (kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"} < kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics"}) and changes(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}[15m]) == 0' - for: 'PT15M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeHpaMaxedOut' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeHpaMaxedOut/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }}' - description: 'HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has been running at max replicas for longer than 15 minutes.' - info: 'HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has been running at max replicas for longer than 15 minutes.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubehpamaxedout' - summary: 'HPA is running at max replicas' - title: 'HPA is running at max replicas namespace:{{ $labels.namespace }} horizontalpodautoscaler:{{ $labels.horizontalpodautoscaler }}' - } - expression: 'kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"} == kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics"}' - for: 'PT15M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - ] - scopes: [ - azureMonitoring - ] - } -} - -resource svcKubernetesResources 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { - name: 'svc-kubernetes-resources' - location: location - properties: { - interval: 'PT1M' - rules: [ - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeCPUOvercommit' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeCPUOvercommit/{{ $labels.cluster }}' - description: 'Cluster {{ $labels.cluster }} has overcommitted CPU resource requests for Pods by {{ $value }} CPU shares and cannot tolerate node failure.' - info: 'Cluster {{ $labels.cluster }} has overcommitted CPU resource requests for Pods by {{ $value }} CPU shares and cannot tolerate node failure.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecpuovercommit' - summary: 'Cluster has overcommitted CPU resource requests.' - title: 'Cluster has overcommitted CPU resource requests. cluster:{{ $labels.cluster }}' - } - expression: 'sum by (cluster) (namespace_cpu:kube_pod_container_resource_requests:sum) - (sum by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) - max by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"})) > 0 and (sum by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) - max by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"})) > 0' - for: 'PT10M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeMemoryOvercommit' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeMemoryOvercommit/{{ $labels.cluster }}' - description: 'Cluster {{ $labels.cluster }} has overcommitted memory resource requests for Pods by {{ $value | humanize }} bytes and cannot tolerate node failure.' - info: 'Cluster {{ $labels.cluster }} has overcommitted memory resource requests for Pods by {{ $value | humanize }} bytes and cannot tolerate node failure.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryovercommit' - summary: 'Cluster has overcommitted memory resource requests.' - title: 'Cluster has overcommitted memory resource requests. cluster:{{ $labels.cluster }}' - } - expression: 'sum by (cluster) (namespace_memory:kube_pod_container_resource_requests:sum) - (sum by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="memory"}) - max by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="memory"})) > 0 and (sum by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="memory"}) - max by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="memory"})) > 0' - for: 'PT10M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeCPUQuotaOvercommit' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeCPUQuotaOvercommit/{{ $labels.cluster }}' - description: 'Cluster {{ $labels.cluster }} has overcommitted CPU resource requests for Namespaces.' - info: 'Cluster {{ $labels.cluster }} has overcommitted CPU resource requests for Namespaces.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecpuquotaovercommit' - summary: 'Cluster has overcommitted CPU resource requests.' - title: 'Cluster has overcommitted CPU resource requests. cluster:{{ $labels.cluster }}' - } - expression: 'sum by (cluster) (min without (resource) (kube_resourcequota{job="kube-state-metrics",resource=~"(cpu|requests.cpu)",type="hard"})) / sum by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) > 1.5' - for: 'PT5M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeMemoryQuotaOvercommit' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeMemoryQuotaOvercommit/{{ $labels.cluster }}' - description: 'Cluster {{ $labels.cluster }} has overcommitted memory resource requests for Namespaces.' - info: 'Cluster {{ $labels.cluster }} has overcommitted memory resource requests for Namespaces.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryquotaovercommit' - summary: 'Cluster has overcommitted memory resource requests.' - title: 'Cluster has overcommitted memory resource requests. cluster:{{ $labels.cluster }}' - } - expression: 'sum by (cluster) (min without (resource) (kube_resourcequota{job="kube-state-metrics",resource=~"(memory|requests.memory)",type="hard"})) / sum by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="memory"}) > 1.5' - for: 'PT5M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeQuotaAlmostFull' - enabled: true - labels: { - severity: 'info' - } - annotations: { - correlationId: 'KubeQuotaAlmostFull/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.resource }}' - description: 'Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.' - info: 'Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotaalmostfull' - summary: 'Namespace quota is going to be full.' - title: 'Namespace quota is going to be full. namespace:{{ $labels.namespace }} resource:{{ $labels.resource }}' - } - expression: 'kube_resourcequota{job="kube-state-metrics",type="used"} / ignoring (instance, job, type) (kube_resourcequota{job="kube-state-metrics",type="hard"} > 0) > 0.9 < 1' - for: 'PT15M' - severity: severityCeiling > 0 ? max(4, severityCeiling) : 4 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeQuotaFullyUsed' - enabled: true - labels: { - severity: 'info' - } - annotations: { - correlationId: 'KubeQuotaFullyUsed/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.resource }}' - description: 'Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.' - info: 'Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotafullyused' - summary: 'Namespace quota is fully used.' - title: 'Namespace quota is fully used. namespace:{{ $labels.namespace }} resource:{{ $labels.resource }}' - } - expression: 'kube_resourcequota{job="kube-state-metrics",type="used"} / ignoring (instance, job, type) (kube_resourcequota{job="kube-state-metrics",type="hard"} > 0) == 1' - for: 'PT15M' - severity: severityCeiling > 0 ? max(4, severityCeiling) : 4 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeQuotaExceeded' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeQuotaExceeded/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.resource }}' - description: 'Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.' - info: 'Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotaexceeded' - summary: 'Namespace quota has exceeded the limits.' - title: 'Namespace quota has exceeded the limits. namespace:{{ $labels.namespace }} resource:{{ $labels.resource }}' - } - expression: 'kube_resourcequota{job="kube-state-metrics",type="used"} / ignoring (instance, job, type) (kube_resourcequota{job="kube-state-metrics",type="hard"} > 0) > 1' - for: 'PT15M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'CPUThrottlingHigh' - enabled: true - labels: { - severity: 'info' - } - annotations: { - correlationId: 'CPUThrottlingHigh/{{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.container }}/{{ $labels.pod }}' - description: '{{ $value | humanizePercentage }} throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod }}.' - info: '{{ $value | humanizePercentage }} throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod }}.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/cputhrottlinghigh' - summary: 'Processes experience elevated CPU throttling.' - title: 'Processes experience elevated CPU throttling. namespace:{{ $labels.namespace }} container:{{ $labels.container }} pod:{{ $labels.pod }}' - } - expression: 'sum by (cluster, container, pod, namespace) (increase(container_cpu_cfs_throttled_periods_total{container!=""}[5m])) / sum by (cluster, container, pod, namespace) (increase(container_cpu_cfs_periods_total[5m])) > (25 / 100)' - for: 'PT15M' - severity: severityCeiling > 0 ? max(4, severityCeiling) : 4 - } - ] - scopes: [ - azureMonitoring - ] - } -} - -resource svcKubernetesStorage 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { - name: 'svc-kubernetes-storage' - location: location - properties: { - interval: 'PT1M' - rules: [ - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubePersistentVolumeFillingUp' - enabled: true - labels: { - severity: 'critical' - } - annotations: { - correlationId: 'KubePersistentVolumeFillingUp/{{ $labels.cluster }}/{{ $labels.persistentvolumeclaim }}/{{ $labels.namespace }}' - description: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} is only {{ $value | humanizePercentage }} free.' - info: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} is only {{ $value | humanizePercentage }} free.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup' - summary: 'PersistentVolume is filling up.' - title: 'PersistentVolume is filling up. persistentvolumeclaim:{{ $labels.persistentvolumeclaim }} namespace:{{ $labels.namespace }} cluster:{{ $labels.cluster }}' - } - expression: '(kubelet_volume_stats_available_bytes{job="kubelet",metrics_path="/metrics"} / kubelet_volume_stats_capacity_bytes{job="kubelet",metrics_path="/metrics"}) < 0.03 and kubelet_volume_stats_used_bytes{job="kubelet",metrics_path="/metrics"} > 0 unless on (cluster, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{access_mode="ReadOnlyMany"} == 1 unless on (cluster, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1' - for: 'PT1M' - severity: severityCeiling > 0 ? max(2, severityCeiling) : 2 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubePersistentVolumeFillingUp' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubePersistentVolumeFillingUp/{{ $labels.cluster }}/{{ $labels.persistentvolumeclaim }}/{{ $labels.namespace }}' - description: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} is expected to fill up within four days. Currently {{ $value | humanizePercentage }} is available.' - info: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} is expected to fill up within four days. Currently {{ $value | humanizePercentage }} is available.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup' - summary: 'PersistentVolume is filling up.' - title: 'PersistentVolume is filling up. persistentvolumeclaim:{{ $labels.persistentvolumeclaim }} namespace:{{ $labels.namespace }} cluster:{{ $labels.cluster }}' - } - expression: '(kubelet_volume_stats_available_bytes{job="kubelet",metrics_path="/metrics"} / kubelet_volume_stats_capacity_bytes{job="kubelet",metrics_path="/metrics"}) < 0.15 and kubelet_volume_stats_used_bytes{job="kubelet",metrics_path="/metrics"} > 0 and predict_linear(kubelet_volume_stats_available_bytes{job="kubelet",metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 unless on (cluster, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{access_mode="ReadOnlyMany"} == 1 unless on (cluster, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1' - for: 'PT1H' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubePersistentVolumeInodesFillingUp' - enabled: true - labels: { - severity: 'critical' - } - annotations: { - correlationId: 'KubePersistentVolumeInodesFillingUp/{{ $labels.cluster }}/{{ $labels.persistentvolumeclaim }}/{{ $labels.namespace }}' - description: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} only has {{ $value | humanizePercentage }} free inodes.' - info: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} only has {{ $value | humanizePercentage }} free inodes.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeinodesfillingup' - summary: 'PersistentVolumeInodes are filling up.' - title: 'PersistentVolumeInodes are filling up. persistentvolumeclaim:{{ $labels.persistentvolumeclaim }} namespace:{{ $labels.namespace }} cluster:{{ $labels.cluster }}' - } - expression: '(kubelet_volume_stats_inodes_free{job="kubelet",metrics_path="/metrics"} / kubelet_volume_stats_inodes{job="kubelet",metrics_path="/metrics"}) < 0.03 and kubelet_volume_stats_inodes_used{job="kubelet",metrics_path="/metrics"} > 0 unless on (cluster, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{access_mode="ReadOnlyMany"} == 1 unless on (cluster, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1' - for: 'PT1M' - severity: severityCeiling > 0 ? max(2, severityCeiling) : 2 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubePersistentVolumeInodesFillingUp' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubePersistentVolumeInodesFillingUp/{{ $labels.cluster }}/{{ $labels.persistentvolumeclaim }}/{{ $labels.namespace }}' - description: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} is expected to run out of inodes within four days. Currently {{ $value | humanizePercentage }} of its inodes are free.' - info: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} is expected to run out of inodes within four days. Currently {{ $value | humanizePercentage }} of its inodes are free.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeinodesfillingup' - summary: 'PersistentVolumeInodes are filling up.' - title: 'PersistentVolumeInodes are filling up. persistentvolumeclaim:{{ $labels.persistentvolumeclaim }} namespace:{{ $labels.namespace }} cluster:{{ $labels.cluster }}' - } - expression: '(kubelet_volume_stats_inodes_free{job="kubelet",metrics_path="/metrics"} / kubelet_volume_stats_inodes{job="kubelet",metrics_path="/metrics"}) < 0.15 and kubelet_volume_stats_inodes_used{job="kubelet",metrics_path="/metrics"} > 0 and predict_linear(kubelet_volume_stats_inodes_free{job="kubelet",metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 unless on (cluster, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{access_mode="ReadOnlyMany"} == 1 unless on (cluster, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1' - for: 'PT1H' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubePersistentVolumeErrors' - enabled: true - labels: { - severity: 'critical' - } - annotations: { - correlationId: 'KubePersistentVolumeErrors/{{ $labels.cluster }}/{{ $labels.persistentvolume }}/{{ $labels.phase }}' - description: 'The persistent volume {{ $labels.persistentvolume }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} has status {{ $labels.phase }}.' - info: 'The persistent volume {{ $labels.persistentvolume }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} has status {{ $labels.phase }}.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeerrors' - summary: 'PersistentVolume is having issues with provisioning.' - title: 'PersistentVolume is having issues with provisioning. persistentvolume:{{ $labels.persistentvolume }} cluster:{{ $labels.cluster }} phase:{{ $labels.phase }}' - } - expression: 'kube_persistentvolume_status_phase{job="kube-state-metrics",phase=~"Failed|Pending"} > 0' - for: 'PT5M' - severity: severityCeiling > 0 ? max(2, severityCeiling) : 2 - } - ] - scopes: [ - azureMonitoring - ] - } -} - -resource svcKubernetesSystem 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { - name: 'svc-kubernetes-system' - location: location - properties: { - interval: 'PT1M' - rules: [ - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeVersionMismatch' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeVersionMismatch/{{ $labels.cluster }}' - description: 'There are {{ $value }} different semantic versions of Kubernetes components running.' - info: 'There are {{ $value }} different semantic versions of Kubernetes components running.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeversionmismatch' - summary: 'Different semantic versions of Kubernetes components running.' - title: 'Different semantic versions of Kubernetes components running.' - } - expression: 'count by (cluster) (count by (git_version, cluster) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"}, "git_version", "$1", "git_version", "(v[0-9]*.[0-9]*).*"))) > 1' - for: 'PT1H30M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeClientErrors' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeClientErrors/{{ $labels.cluster }}/{{ $labels.job }}/{{ $labels.instance }}' - description: 'Kubernetes API server client \'{{ $labels.job }}/{{ $labels.instance }}\' is experiencing {{ $value | humanizePercentage }} errors.\'' - info: 'Kubernetes API server client \'{{ $labels.job }}/{{ $labels.instance }}\' is experiencing {{ $value | humanizePercentage }} errors.\'' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclienterrors' - summary: 'Kubernetes API server client is experiencing errors.' - title: 'Kubernetes API server client is experiencing errors. job:{{ $labels.job }} instance:{{ $labels.instance }}' - } - expression: '(sum by (cluster, instance, job, namespace) (rate(rest_client_requests_total{code=~"5..",job="controlplane-apiserver"}[5m])) / sum by (cluster, instance, job, namespace) (rate(rest_client_requests_total{job="controlplane-apiserver"}[5m]))) > 0.01' - for: 'PT15M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - ] - scopes: [ - azureMonitoring - ] - } -} - -resource svcKubeApiserverSlos 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { - name: 'svc-kube-apiserver-slos' - location: location - properties: { - interval: 'PT1M' - rules: [ - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeAPIErrorBudgetBurn' - enabled: true - labels: { - long: '1h' - severity: 'critical' - short: '5m' - } - annotations: { - correlationId: 'KubeAPIErrorBudgetBurn/{{ $labels.cluster }}' - description: 'The API server is burning too much error budget.' - info: 'The API server is burning too much error budget.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn' - summary: 'The API server is burning too much error budget.' - title: 'The API server is burning too much error budget.' - } - expression: 'sum(apiserver_request:burnrate1h) > (14.4 * 0.01) and sum(apiserver_request:burnrate5m) > (14.4 * 0.01)' - for: 'PT2M' - severity: severityCeiling > 0 ? max(2, severityCeiling) : 2 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeAPIErrorBudgetBurn' - enabled: true - labels: { - long: '6h' - severity: 'critical' - short: '30m' - } - annotations: { - correlationId: 'KubeAPIErrorBudgetBurn/{{ $labels.cluster }}' - description: 'The API server is burning too much error budget.' - info: 'The API server is burning too much error budget.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn' - summary: 'The API server is burning too much error budget.' - title: 'The API server is burning too much error budget.' - } - expression: 'sum(apiserver_request:burnrate6h) > (6 * 0.01) and sum(apiserver_request:burnrate30m) > (6 * 0.01)' - for: 'PT15M' - severity: severityCeiling > 0 ? max(2, severityCeiling) : 2 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeAPIErrorBudgetBurn' - enabled: true - labels: { - long: '1d' - severity: 'warning' - short: '2h' - } - annotations: { - correlationId: 'KubeAPIErrorBudgetBurn/{{ $labels.cluster }}' - description: 'The API server is burning too much error budget.' - info: 'The API server is burning too much error budget.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn' - summary: 'The API server is burning too much error budget.' - title: 'The API server is burning too much error budget.' - } - expression: 'sum(apiserver_request:burnrate1d) > (3 * 0.01) and sum(apiserver_request:burnrate2h) > (3 * 0.01)' - for: 'PT1H' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeAPIErrorBudgetBurn' - enabled: true - labels: { - long: '3d' - severity: 'warning' - short: '6h' - } - annotations: { - correlationId: 'KubeAPIErrorBudgetBurn/{{ $labels.cluster }}' - description: 'The API server is burning too much error budget.' - info: 'The API server is burning too much error budget.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn' - summary: 'The API server is burning too much error budget.' - title: 'The API server is burning too much error budget.' - } - expression: 'sum(apiserver_request:burnrate3d) > (1 * 0.01) and sum(apiserver_request:burnrate6h) > (1 * 0.01)' - for: 'PT3H' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - ] - scopes: [ - azureMonitoring - ] - } -} - -resource svcKubernetesSystemApiserver 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { - name: 'svc-kubernetes-system-apiserver' - location: location - properties: { - interval: 'PT1M' - rules: [ - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeClientCertificateExpiration' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeClientCertificateExpiration/{{ $labels.cluster }}' - description: 'A client certificate used to authenticate to kubernetes apiserver is expiring in less than 7.0 days.' - info: 'A client certificate used to authenticate to kubernetes apiserver is expiring in less than 7.0 days.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration' - summary: 'Client certificate is about to expire.' - title: 'Client certificate is about to expire.' - } - expression: 'apiserver_client_certificate_expiration_seconds_count{job="controlplane-apiserver"} > 0 and on (job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="controlplane-apiserver"}[5m]))) < 604800' - for: 'PT5M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeClientCertificateExpiration' - enabled: true - labels: { - severity: 'critical' - } - annotations: { - correlationId: 'KubeClientCertificateExpiration/{{ $labels.cluster }}' - description: 'A client certificate used to authenticate to kubernetes apiserver is expiring in less than 24.0 hours.' - info: 'A client certificate used to authenticate to kubernetes apiserver is expiring in less than 24.0 hours.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration' - summary: 'Client certificate is about to expire.' - title: 'Client certificate is about to expire.' - } - expression: 'apiserver_client_certificate_expiration_seconds_count{job="controlplane-apiserver"} > 0 and on (job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="controlplane-apiserver"}[5m]))) < 86400' - for: 'PT5M' - severity: severityCeiling > 0 ? max(2, severityCeiling) : 2 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeAggregatedAPIErrors' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeAggregatedAPIErrors/{{ $labels.cluster }}/{{ $labels.name }}/{{ $labels.namespace }}' - description: 'Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has reported errors. It has appeared unavailable {{ $value | humanize }} times averaged over the past 10m.' - info: 'Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has reported errors. It has appeared unavailable {{ $value | humanize }} times averaged over the past 10m.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapierrors' - summary: 'Kubernetes aggregated API has reported errors.' - title: 'Kubernetes aggregated API has reported errors. name:{{ $labels.name }} namespace:{{ $labels.namespace }}' - } - expression: 'sum by (name, namespace, cluster) (increase(aggregator_unavailable_apiservice_total{job="controlplane-apiserver"}[10m])) > 4' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeAggregatedAPIDown' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeAggregatedAPIDown/{{ $labels.cluster }}/{{ $labels.name }}/{{ $labels.namespace }}' - description: 'Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has been only {{ $value | humanize }}% available over the last 10m.' - info: 'Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has been only {{ $value | humanize }}% available over the last 10m.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapidown' - summary: 'Kubernetes aggregated API is down.' - title: 'Kubernetes aggregated API is down. name:{{ $labels.name }} namespace:{{ $labels.namespace }}' - } - expression: '(1 - max by (name, namespace, cluster) (avg_over_time(aggregator_unavailable_apiservice{job="controlplane-apiserver"}[10m]))) * 100 < 85' - for: 'PT5M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeAPIDown' - enabled: true - labels: { - severity: 'critical' - } - annotations: { - correlationId: 'KubeAPIDown/{{ $labels.cluster }}' - description: 'KubeAPI has disappeared from Prometheus target discovery.' - info: 'KubeAPI has disappeared from Prometheus target discovery.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapidown' - summary: 'Target disappeared from Prometheus target discovery.' - title: 'Target disappeared from Prometheus target discovery.' - } - expression: 'count by (cluster) (up{job="controlplane-apiserver"} == 1) == 0' - for: 'PT15M' - severity: severityCeiling > 0 ? max(2, severityCeiling) : 2 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeAPITerminatedRequests' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeAPITerminatedRequests/{{ $labels.cluster }}' - description: 'The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.' - info: 'The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapiterminatedrequests' - summary: 'The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.' - title: 'The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.' - } - expression: 'sum(rate(apiserver_request_terminations_total{job="controlplane-apiserver"}[10m])) / (sum(rate(apiserver_request_total{job="controlplane-apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="controlplane-apiserver"}[10m]))) > 0.2' - for: 'PT5M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - ] - scopes: [ - azureMonitoring - ] - } -} - -resource svcKubernetesSystemKubelet 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { - name: 'svc-kubernetes-system-kubelet' - location: location - properties: { - interval: 'PT1M' - rules: [ - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeNodeNotReady' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeNodeNotReady/{{ $labels.cluster }}/{{ $labels.node }}' - description: '{{ $labels.node }} has been unready for more than 15 minutes.' - info: '{{ $labels.node }} has been unready for more than 15 minutes.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodenotready' - summary: 'Node is not ready.' - title: 'Node is not ready. node:{{ $labels.node }}' - } - expression: 'kube_node_status_condition{condition="Ready",job="kube-state-metrics",status="true"} == 0' - for: 'PT15M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeNodeUnreachable' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeNodeUnreachable/{{ $labels.cluster }}/{{ $labels.node }}' - description: '{{ $labels.node }} is unreachable and some workloads may be rescheduled.' - info: '{{ $labels.node }} is unreachable and some workloads may be rescheduled.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodeunreachable' - summary: 'Node is unreachable.' - title: 'Node is unreachable. node:{{ $labels.node }}' - } - expression: '(kube_node_spec_taint{effect="NoSchedule",job="kube-state-metrics",key="node.kubernetes.io/unreachable"} unless ignoring (key, value) kube_node_spec_taint{job="kube-state-metrics",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) == 1' - for: 'PT15M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeletTooManyPods' - enabled: true - labels: { - severity: 'info' - } - annotations: { - correlationId: 'KubeletTooManyPods/{{ $labels.cluster }}/{{ $labels.node }}' - description: 'Kubelet \'{{ $labels.node }}\' is running at {{ $value | humanizePercentage }} of its Pod capacity.' - info: 'Kubelet \'{{ $labels.node }}\' is running at {{ $value | humanizePercentage }} of its Pod capacity.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubelettoomanypods' - summary: 'Kubelet is running at capacity.' - title: 'Kubelet is running at capacity. node:{{ $labels.node }}' - } - expression: 'count by (cluster, node) ((kube_pod_status_phase{job="kube-state-metrics",phase="Running"} == 1) * on (instance, pod, namespace, cluster) group_left (node) topk by (instance, pod, namespace, cluster) (1, kube_pod_info{job="kube-state-metrics"})) / max by (cluster, node) (kube_node_status_capacity{job="kube-state-metrics",resource="pods"} != 1) > 0.95' - for: 'PT15M' - severity: severityCeiling > 0 ? max(4, severityCeiling) : 4 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeNodeReadinessFlapping' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeNodeReadinessFlapping/{{ $labels.cluster }}/{{ $labels.node }}' - description: 'The readiness status of node {{ $labels.node }} has changed {{ $value }} times in the last 15 minutes.' - info: 'The readiness status of node {{ $labels.node }} has changed {{ $value }} times in the last 15 minutes.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodereadinessflapping' - summary: 'Node readiness status is flapping.' - title: 'Node readiness status is flapping. node:{{ $labels.node }}' - } - expression: 'sum by (cluster, node) (changes(kube_node_status_condition{condition="Ready",job="kube-state-metrics",status="true"}[15m])) > 2' - for: 'PT15M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeletPlegDurationHigh' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeletPlegDurationHigh/{{ $labels.cluster }}/{{ $labels.node }}' - description: 'The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{ $value }} seconds on node {{ $labels.node }}.' - info: 'The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{ $value }} seconds on node {{ $labels.node }}.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletplegdurationhigh' - summary: 'Kubelet Pod Lifecycle Event Generator is taking too long to relist.' - title: 'Kubelet Pod Lifecycle Event Generator is taking too long to relist. node:{{ $labels.node }}' - } - expression: 'node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10' - for: 'PT5M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeletPodStartUpLatencyHigh' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeletPodStartUpLatencyHigh/{{ $labels.cluster }}/{{ $labels.node }}' - description: 'Kubelet Pod startup 99th percentile latency is {{ $value }} seconds on node {{ $labels.node }}.' - info: 'Kubelet Pod startup 99th percentile latency is {{ $value }} seconds on node {{ $labels.node }}.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletpodstartuplatencyhigh' - summary: 'Kubelet Pod startup latency is too high.' - title: 'Kubelet Pod startup latency is too high. node:{{ $labels.node }}' - } - expression: 'histogram_quantile(0.99, sum by (cluster, instance, le) (rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet",metrics_path="/metrics"}[5m]))) * on (cluster, instance) group_left (node) kubelet_node_name{job="kubelet",metrics_path="/metrics"} > 60' - for: 'PT15M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeletClientCertificateExpiration' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeletClientCertificateExpiration/{{ $labels.cluster }}/{{ $labels.node }}' - description: 'Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.' - info: 'Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletclientcertificateexpiration' - summary: 'Kubelet client certificate is about to expire.' - title: 'Kubelet client certificate is about to expire. node:{{ $labels.node }}' - } - expression: 'kubelet_certificate_manager_client_ttl_seconds < 604800' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeletClientCertificateExpiration' - enabled: true - labels: { - severity: 'critical' - } - annotations: { - correlationId: 'KubeletClientCertificateExpiration/{{ $labels.cluster }}/{{ $labels.node }}' - description: 'Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.' - info: 'Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletclientcertificateexpiration' - summary: 'Kubelet client certificate is about to expire.' - title: 'Kubelet client certificate is about to expire. node:{{ $labels.node }}' - } - expression: 'kubelet_certificate_manager_client_ttl_seconds < 86400' - severity: severityCeiling > 0 ? max(2, severityCeiling) : 2 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeletServerCertificateExpiration' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeletServerCertificateExpiration/{{ $labels.cluster }}/{{ $labels.node }}' - description: 'Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.' - info: 'Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletservercertificateexpiration' - summary: 'Kubelet server certificate is about to expire.' - title: 'Kubelet server certificate is about to expire. node:{{ $labels.node }}' - } - expression: 'kubelet_certificate_manager_server_ttl_seconds < 604800' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeletServerCertificateExpiration' - enabled: true - labels: { - severity: 'critical' - } - annotations: { - correlationId: 'KubeletServerCertificateExpiration/{{ $labels.cluster }}/{{ $labels.node }}' - description: 'Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.' - info: 'Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletservercertificateexpiration' - summary: 'Kubelet server certificate is about to expire.' - title: 'Kubelet server certificate is about to expire. node:{{ $labels.node }}' - } - expression: 'kubelet_certificate_manager_server_ttl_seconds < 86400' - severity: severityCeiling > 0 ? max(2, severityCeiling) : 2 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeletClientCertificateRenewalErrors' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeletClientCertificateRenewalErrors/{{ $labels.cluster }}/{{ $labels.node }}' - description: 'Kubelet on node {{ $labels.node }} has failed to renew its client certificate ({{ $value | humanize }} errors in the last 5 minutes).' - info: 'Kubelet on node {{ $labels.node }} has failed to renew its client certificate ({{ $value | humanize }} errors in the last 5 minutes).' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletclientcertificaterenewalerrors' - summary: 'Kubelet has failed to renew its client certificate.' - title: 'Kubelet has failed to renew its client certificate. node:{{ $labels.node }}' - } - expression: 'increase(kubelet_certificate_manager_client_expiration_renew_errors[5m]) > 0' - for: 'PT15M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeletServerCertificateRenewalErrors' - enabled: true - labels: { - severity: 'warning' - } - annotations: { - correlationId: 'KubeletServerCertificateRenewalErrors/{{ $labels.cluster }}/{{ $labels.node }}' - description: 'Kubelet on node {{ $labels.node }} has failed to renew its server certificate ({{ $value | humanize }} errors in the last 5 minutes).' - info: 'Kubelet on node {{ $labels.node }} has failed to renew its server certificate ({{ $value | humanize }} errors in the last 5 minutes).' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletservercertificaterenewalerrors' - summary: 'Kubelet has failed to renew its server certificate.' - title: 'Kubelet has failed to renew its server certificate. node:{{ $labels.node }}' - } - expression: 'increase(kubelet_server_expiration_renew_errors[5m]) > 0' - for: 'PT15M' - severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeletDown' - enabled: true - labels: { - severity: 'critical' - } - annotations: { - correlationId: 'KubeletDown/{{ $labels.cluster }}' - description: 'Kubelet has disappeared from Prometheus target discovery.' - info: 'Kubelet has disappeared from Prometheus target discovery.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletdown' - summary: 'Target disappeared from Prometheus target discovery.' - title: 'Target disappeared from Prometheus target discovery.' - } - expression: 'count by (cluster) (up{job="kubelet",metrics_path="/metrics"} == 1) == 0' - for: 'PT15M' - severity: severityCeiling > 0 ? max(2, severityCeiling) : 2 - } - ] - scopes: [ - azureMonitoring - ] - } -} - -resource svcKubernetesSystemScheduler 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { - name: 'svc-kubernetes-system-scheduler' - location: location - properties: { - interval: 'PT1M' - rules: [ - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeSchedulerDown' - enabled: true - labels: { - severity: 'critical' - } - annotations: { - correlationId: 'KubeSchedulerDown/{{ $labels.cluster }}' - description: 'KubeScheduler has disappeared from Prometheus target discovery.' - info: 'KubeScheduler has disappeared from Prometheus target discovery.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeschedulerdown' - summary: 'Target disappeared from Prometheus target discovery.' - title: 'Target disappeared from Prometheus target discovery.' - } - expression: 'count by (cluster) (up{job="controlplane-kube-scheduler"} == 1) == 0' - for: 'PT15M' - severity: severityCeiling > 0 ? max(2, severityCeiling) : 2 - } - ] - scopes: [ - azureMonitoring - ] - } -} - -resource svcKubernetesSystemControllerManager 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { - name: 'svc-kubernetes-system-controller-manager' - location: location - properties: { - interval: 'PT1M' - rules: [ - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'KubeControllerManagerDown' - enabled: true - labels: { - severity: 'critical' - } - annotations: { - correlationId: 'KubeControllerManagerDown/{{ $labels.cluster }}' - description: 'KubeControllerManager has disappeared from Prometheus target discovery.' - info: 'KubeControllerManager has disappeared from Prometheus target discovery.' - runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecontrollermanagerdown' - summary: 'Target disappeared from Prometheus target discovery.' - title: 'Target disappeared from Prometheus target discovery.' - } - expression: 'count by (cluster) (up{job="controlplane-kube-controller-manager"} == 1) == 0' - for: 'PT15M' - severity: severityCeiling > 0 ? max(2, severityCeiling) : 2 - } - ] - scopes: [ - azureMonitoring - ] - } -} diff --git a/dev-infrastructure/modules/metrics/svc-kube-rules.bicep b/dev-infrastructure/modules/metrics/svc-kube-rules.bicep deleted file mode 100644 index cd9dd3294fa..00000000000 --- a/dev-infrastructure/modules/metrics/svc-kube-rules.bicep +++ /dev/null @@ -1,20 +0,0 @@ -// Kubernetes control plane alerts scoped to the svc workspace. -// These are the same upstream Kubernetes alerts also deployed to the hcp workspace, -// duplicated here so that gather-observability can detect them on the svc workspace. - -@description('The Azure resource ID of the Azure Monitor Workspace (stores prometheus metrics for services/aks level metrics)') -param azureMonitoringWorkspaceId string - -param actionGroups array - -@description('The minimum IcM severity level (highest priority) that alerts can fire at. Alerts more critical than this ceiling will be degraded to this value. 0 means no ceiling.') -param severityCeiling int = 0 - -module generatedAlerts 'rules/generatedSvcKubePrometheusAlertingRules.bicep' = { - name: 'generatedSvcKubePrometheusAlertingRules' - params: { - azureMonitoring: azureMonitoringWorkspaceId - actionGroups: actionGroups - severityCeiling: severityCeiling - } -} diff --git a/dev-infrastructure/templates/monitoring.bicep b/dev-infrastructure/templates/monitoring.bicep index d5c6bfe9c09..f6ad497766b 100644 --- a/dev-infrastructure/templates/monitoring.bicep +++ b/dev-infrastructure/templates/monitoring.bicep @@ -152,15 +152,6 @@ module serviceAlerts '../modules/metrics/service-rules.bicep' = { } } -module svcKubeAlerts '../modules/metrics/svc-kube-rules.bicep' = { - name: 'svcKubeAlerts' - params: { - azureMonitoringWorkspaceId: azureMonitoringWorkspaceId - actionGroups: slActionGroups - severityCeiling: alertSeverityCeiling - } -} - module kustoServiceAlerts '../modules/metrics/kusto-service-rules.bicep' = { name: 'kustoServiceAlerts' params: { diff --git a/observability/alerts-kusto-services.yaml b/observability/alerts-kusto-services.yaml index bda13a93c1f..8bda34b7184 100644 --- a/observability/alerts-kusto-services.yaml +++ b/observability/alerts-kusto-services.yaml @@ -1,4 +1,23 @@ prometheusRules: rulesFolders: [] - untestedRules: [] + untestedRules: + - alerts/kubernetesControlPlane-prometheusRule.yaml outputBicep: ../dev-infrastructure/modules/metrics/rules/generatedKustoOnlyServicePrometheusAlertingRules.bicep + groupNamePrefix: "svc-" + outputReplacements: + - from: 'job="kube-scheduler"' + to: 'job="controlplane-kube-scheduler"' + - from: 'job="apiserver"' + to: 'job="controlplane-apiserver"' + - from: 'job="etcd"' + to: 'job="controlplane-etcd"' + - from: 'job="cluster-autoscaler"' + to: 'job="controlplane-cluster-autoscaler"' + - from: 'job="kube-controller-manager"' + to: 'job="controlplane-kube-controller-manager"' + regexOutputReplacements: + # absent alerts don't work correctly because they aren't associated with a specific cluster + # The alert ends up firing with {{target.cluster}} in the title and correlation id instead of + # an actual cluster name + - from: 'absent\(up{job="(.+)"} == 1\)' + to: 'count by (cluster) (up{job="$1"} == 1) == 0' diff --git a/observability/alerts-sl-svc-kube.yaml b/observability/alerts-sl-svc-kube.yaml deleted file mode 100644 index 665c7db45d7..00000000000 --- a/observability/alerts-sl-svc-kube.yaml +++ /dev/null @@ -1,23 +0,0 @@ -prometheusRules: - rulesFolders: [] - untestedRules: - - alerts/kubernetesControlPlane-prometheusRule.yaml - outputBicep: ../dev-infrastructure/modules/metrics/rules/generatedSvcKubePrometheusAlertingRules.bicep - groupNamePrefix: "svc-" - outputReplacements: - - from: 'job="kube-scheduler"' - to: 'job="controlplane-kube-scheduler"' - - from: 'job="apiserver"' - to: 'job="controlplane-apiserver"' - - from: 'job="etcd"' - to: 'job="controlplane-etcd"' - - from: 'job="cluster-autoscaler"' - to: 'job="controlplane-cluster-autoscaler"' - - from: 'job="kube-controller-manager"' - to: 'job="controlplane-kube-controller-manager"' - regexOutputReplacements: - # absent alerts don't work correctly because they aren't associated with a specific cluster - # The alert ends up firing with {{target.cluster}} in the title and correlation id instead of - # an actual cluster name - - from: 'absent\(up{job="(.+)"} == 1\)' - to: 'count by (cluster) (up{job="$1"} == 1) == 0' diff --git a/tooling/prometheus-rules/Makefile b/tooling/prometheus-rules/Makefile index edc88fbc3e3..d4dd3d0457b 100644 --- a/tooling/prometheus-rules/Makefile +++ b/tooling/prometheus-rules/Makefile @@ -23,7 +23,7 @@ fmt-devinfra: run: alerts recording-rules .PHONY: run -alerts: run-sl-services run-sl-svc-kube run-sl-hcps run-sre-hcps run-sre-services run-rp-services run-msft-services run-kusto-services run-kusto-hcps +alerts: run-sl-services run-sl-hcps run-sre-hcps run-sre-services run-rp-services run-msft-services run-kusto-services run-kusto-hcps $(MAKE) fmt-devinfra .PHONY: alerts @@ -35,10 +35,6 @@ run-sl-services: go run ./... --config-file ../../observability/alerts-sl-services.yaml $(EXTRA_FLAGS) .PHONY: run-sl-services -run-sl-svc-kube: - go run ./... --config-file ../../observability/alerts-sl-svc-kube.yaml $(EXTRA_FLAGS) -.PHONY: run-sl-svc-kube - run-sl-hcps: go run ./... --config-file ../../observability/alerts-sl-hcps.yaml $(EXTRA_FLAGS) .PHONY: run-sl-hcps @@ -77,7 +73,6 @@ run-recording-rules-hcps: ALERT_CONFIGS = \ ../../observability/alerts-sl-services.yaml \ - ../../observability/alerts-sl-svc-kube.yaml \ ../../observability/alerts-sl-hcps.yaml \ ../../observability/alerts-sre-hcps.yaml \ ../../observability/alerts-sre-services.yaml \