From ab53a974014dafcb71dc8ff647c3d7e0a3913aaf Mon Sep 17 00:00:00 2001 From: Steve Kuznetsov Date: Tue, 7 Jul 2026 08:33:02 -0600 Subject: [PATCH] backend: add alerting for stuck async operations We have end-to-end test assertions that our async operations never take longer than $timeout. We have seen a few cases where customer interactions with the ARM API ended up stuck, and had no visibility into this, even though our end-to-end test suite, had it run on that region at that time, would have clearly caught this. This new set of alerts is an assertion from the backend that async operations never spend more than a certain amount of time in a particular phase. We still benefit from having our async operations not having a clear timeout in the backend, as fixing the underlying problem will cause them to get un-stuck and finish, but we want to know if we're failing to progress. Existing SLO alerting is for failed/terminal phases of async operations, which do not trigger for 'stuck' operations which never exit 'progressing.' Signed-off-by: Steve Kuznetsov --- ...ckend-async-operations-prometheusRule.yaml | 361 ++++++++++ ...-async-operations-prometheusRule_test.yaml | 632 ++++++++++++++++++ .../generatedPrometheusAlertingRules.bicep | 325 +++++++++ observability/alerts-sl-services.yaml | 1 + 4 files changed, 1319 insertions(+) create mode 100644 backend/alerts/backend-async-operations-prometheusRule.yaml create mode 100644 backend/alerts/backend-async-operations-prometheusRule_test.yaml diff --git a/backend/alerts/backend-async-operations-prometheusRule.yaml b/backend/alerts/backend-async-operations-prometheusRule.yaml new file mode 100644 index 00000000000..bdbb4e5f049 --- /dev/null +++ b/backend/alerts/backend-async-operations-prometheusRule.yaml @@ -0,0 +1,361 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + app.kubernetes.io/name: kube-prometheus + app.kubernetes.io/part-of: kube-prometheus + prometheus: k8s + role: alert-rules + name: backend-async-operations-rules + namespace: monitoring +spec: + groups: + # ======================================== + # Backend Async Operation Stuck Alerts + # ======================================== + # These alerts fire when an async operation has been running longer than + # the corresponding e2e test timeout. Thresholds are anchored to the + # canonical timeout constants in test/util/framework/constants.go. + # + # Metric: backend_resource_operation_start_time_seconds (total wall-clock + # time since the operation was created). Combined with phase_info to + # restrict to operations still in non-terminal phases. + # + # No `for` clause: the phase filter already ensures we only alert on + # in-flight operations. Once an operation reaches succeeded/failed/canceled, + # the phase no longer matches and the alert resolves immediately. + # + # All expressions use max by(...) to deduplicate across Prometheus + # replicas/shards, preventing duplicate alerts for the same operation. + - name: backend-async-operations + rules: + # ---------------------------------------- + # Cluster operations + # ---------------------------------------- + # Cluster create: e2e timeout = 20 minutes (ClusterCreationTimeout) + - alert: BackendAsyncOperationClusterCreateStuck + expr: | + ( + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + time() - backend_resource_operation_start_time_seconds{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", + operation_type="create" + } + ) + and + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + backend_resource_operation_phase_info{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", + operation_type="create", + phase=~"accepted|provisioning" + } + ) == 1 + ) > 1200 + labels: + severity: "3" + annotations: + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + summary: 'Cluster create operation stuck' + description: 'Cluster create operation for {{ $labels.resource_id }} has been running for over 20 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: ClusterCreationTimeout).' + runbook_url: TBD + # Cluster update: e2e timeout = 45 minutes (HCPClusterVersionUpgradeTimeout) + # This covers both property updates and version upgrades, since both + # use operation_type=update. The higher timeout accommodates upgrades. + - alert: BackendAsyncOperationClusterUpdateStuck + expr: | + ( + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + time() - backend_resource_operation_start_time_seconds{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", + operation_type="update" + } + ) + and + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + backend_resource_operation_phase_info{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", + operation_type="update", + phase=~"accepted|updating" + } + ) == 1 + ) > 2700 + labels: + severity: "3" + annotations: + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + summary: 'Cluster update operation stuck' + description: 'Cluster update operation for {{ $labels.resource_id }} has been running for over 45 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: HCPClusterVersionUpgradeTimeout).' + runbook_url: TBD + # Cluster delete: e2e timeout = 25 minutes (HCPClusterDeletionTimeout) + # Delete operations start directly in "deleting" phase, skipping "accepted". + - alert: BackendAsyncOperationClusterDeleteStuck + expr: | + ( + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + time() - backend_resource_operation_start_time_seconds{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", + operation_type="delete" + } + ) + and + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + backend_resource_operation_phase_info{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", + operation_type="delete", + phase="deleting" + } + ) == 1 + ) > 1500 + labels: + severity: "3" + annotations: + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + summary: 'Cluster delete operation stuck' + description: 'Cluster delete operation for {{ $labels.resource_id }} has been running for over 25 minutes and is currently in deleting phase (e2e timeout: HCPClusterDeletionTimeout).' + runbook_url: TBD + # ---------------------------------------- + # Credential operations + # ---------------------------------------- + # Request credential: e2e timeout = 10 minutes (GetAdminRESTConfigTimeout) + - alert: BackendAsyncOperationCredentialRequestStuck + expr: | + ( + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + time() - backend_resource_operation_start_time_seconds{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", + operation_type="requestcredential" + } + ) + and + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + backend_resource_operation_phase_info{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", + operation_type="requestcredential", + phase=~"accepted|provisioning" + } + ) == 1 + ) > 600 + labels: + severity: "3" + annotations: + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + summary: 'Credential request operation stuck' + description: 'Credential request operation for {{ $labels.resource_id }} has been running for over 10 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: GetAdminRESTConfigTimeout).' + runbook_url: TBD + # Revoke credentials: e2e timeout = 15 minutes (ExternalAuthDeletionTimeout used as proxy) + - alert: BackendAsyncOperationCredentialRevokeStuck + expr: | + ( + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + time() - backend_resource_operation_start_time_seconds{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", + operation_type="revokecredentials" + } + ) + and + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + backend_resource_operation_phase_info{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", + operation_type="revokecredentials", + phase=~"accepted|deleting" + } + ) == 1 + ) > 900 + labels: + severity: "3" + annotations: + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + summary: 'Credential revoke operation stuck' + description: 'Credential revoke operation for {{ $labels.resource_id }} has been running for over 15 minutes and is currently in {{ $labels.phase }} phase.' + runbook_url: TBD + # ---------------------------------------- + # Node pool operations + # ---------------------------------------- + # Node pool create: e2e timeout = 20 minutes (NodePoolCreationTimeout) + - alert: BackendAsyncOperationNodePoolCreateStuck + expr: | + ( + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + time() - backend_resource_operation_start_time_seconds{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", + operation_type="create" + } + ) + and + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + backend_resource_operation_phase_info{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", + operation_type="create", + phase=~"accepted|provisioning" + } + ) == 1 + ) > 1200 + labels: + severity: "3" + annotations: + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + summary: 'Node pool create operation stuck' + description: 'Node pool create operation for {{ $labels.resource_id }} has been running for over 20 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: NodePoolCreationTimeout).' + runbook_url: TBD + # Node pool update: e2e timeout = 45 minutes (NodePoolVersionUpgradeTimeout) + # Covers both property updates and version upgrades. + - alert: BackendAsyncOperationNodePoolUpdateStuck + expr: | + ( + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + time() - backend_resource_operation_start_time_seconds{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", + operation_type="update" + } + ) + and + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + backend_resource_operation_phase_info{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", + operation_type="update", + phase=~"accepted|updating" + } + ) == 1 + ) > 2700 + labels: + severity: "3" + annotations: + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + summary: 'Node pool update operation stuck' + description: 'Node pool update operation for {{ $labels.resource_id }} has been running for over 45 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: NodePoolVersionUpgradeTimeout).' + runbook_url: TBD + # Node pool delete: e2e timeout = 25 minutes (NodePoolDeletionTimeout) + - alert: BackendAsyncOperationNodePoolDeleteStuck + expr: | + ( + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + time() - backend_resource_operation_start_time_seconds{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", + operation_type="delete" + } + ) + and + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + backend_resource_operation_phase_info{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", + operation_type="delete", + phase="deleting" + } + ) == 1 + ) > 1500 + labels: + severity: "3" + annotations: + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + summary: 'Node pool delete operation stuck' + description: 'Node pool delete operation for {{ $labels.resource_id }} has been running for over 25 minutes and is currently in deleting phase (e2e timeout: NodePoolDeletionTimeout).' + runbook_url: TBD + # ---------------------------------------- + # External auth operations + # ---------------------------------------- + # External auth create: e2e timeout = 15 minutes (ExternalAuthCreationTimeout) + # Includes the awaitingsecret phase, which is specific to external auth. + - alert: BackendAsyncOperationExternalAuthCreateStuck + expr: | + ( + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + time() - backend_resource_operation_start_time_seconds{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", + operation_type="create" + } + ) + and + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + backend_resource_operation_phase_info{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", + operation_type="create", + phase=~"accepted|awaitingsecret|provisioning" + } + ) == 1 + ) > 900 + labels: + severity: "3" + annotations: + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + summary: 'External auth create operation stuck' + description: 'External auth create operation for {{ $labels.resource_id }} has been running for over 15 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: ExternalAuthCreationTimeout).' + runbook_url: TBD + # External auth update: no explicit e2e constant, using 10 minutes as + # a reasonable default (no VM provisioning involved). + - alert: BackendAsyncOperationExternalAuthUpdateStuck + expr: | + ( + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + time() - backend_resource_operation_start_time_seconds{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", + operation_type="update" + } + ) + and + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + backend_resource_operation_phase_info{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", + operation_type="update", + phase=~"accepted|updating" + } + ) == 1 + ) > 600 + labels: + severity: "3" + annotations: + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + summary: 'External auth update operation stuck' + description: 'External auth update operation for {{ $labels.resource_id }} has been running for over 10 minutes and is currently in {{ $labels.phase }} phase.' + runbook_url: TBD + # External auth delete: e2e timeout = 15 minutes (ExternalAuthDeletionTimeout) + - alert: BackendAsyncOperationExternalAuthDeleteStuck + expr: | + ( + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + time() - backend_resource_operation_start_time_seconds{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", + operation_type="delete" + } + ) + and + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + backend_resource_operation_phase_info{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", + operation_type="delete", + phase="deleting" + } + ) == 1 + ) > 900 + labels: + severity: "3" + annotations: + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + summary: 'External auth delete operation stuck' + description: 'External auth delete operation for {{ $labels.resource_id }} has been running for over 15 minutes and is currently in deleting phase (e2e timeout: ExternalAuthDeletionTimeout).' + runbook_url: TBD + # ---------------------------------------- + # Backstop: any async operation not covered above + # ---------------------------------------- + # Catches any future operation types that don't have explicit alerts. + # Threshold (60 min) is higher than the longest specific alert (45 min for upgrades). + # Double-fires with specific alerts correlate to the same IcM incident via shared correlationId. + - alert: BackendAsyncOperationStuck + expr: | + ( + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + time() - backend_resource_operation_start_time_seconds + ) + and + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + backend_resource_operation_phase_info{ + phase=~"accepted|provisioning|updating|deleting|awaitingsecret" + } + ) == 1 + ) > 3600 + labels: + severity: "3" + annotations: + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + summary: 'Async operation stuck' + description: 'Async {{ $labels.operation_type }} operation for {{ $labels.resource_id }} ({{ $labels.resource_type }}) has been running for over 60 minutes and is currently in {{ $labels.phase }} phase.' + runbook_url: TBD diff --git a/backend/alerts/backend-async-operations-prometheusRule_test.yaml b/backend/alerts/backend-async-operations-prometheusRule_test.yaml new file mode 100644 index 00000000000..40070852389 --- /dev/null +++ b/backend/alerts/backend-async-operations-prometheusRule_test.yaml @@ -0,0 +1,632 @@ +rule_files: +- backend-async-operations-prometheusRule.yaml +evaluation_interval: 1m +tests: +# ======================================== +# Cluster create: threshold 1200s (20 min) +# ======================================== +# Test: fires when cluster create exceeds 20 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/1/cluster/a", subscription_id="sub-1", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1"}' + values: "0+0x25" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/1/cluster/a", subscription_id="sub-1", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1"}' + values: "1+0x25" + alert_rule_test: + # At 21m, time()=1260 > 1200 threshold + - eval_time: 21m + alertname: BackendAsyncOperationClusterCreateStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/1/cluster/a" + subscription_id: "sub-1" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters" + operation_type: "create" + phase: "provisioning" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/1/cluster/a' + summary: 'Cluster create operation stuck' + description: 'Cluster create operation for /sub/1/cluster/a has been running for over 20 minutes and is currently in provisioning phase (e2e timeout: ClusterCreationTimeout).' + runbook_url: TBD +# Test: does not fire when cluster create is under 20 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/1/cluster/b", subscription_id="sub-1", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1"}' + values: "0+0x20" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/1/cluster/b", subscription_id="sub-1", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1"}' + values: "1+0x20" + alert_rule_test: + # At 19m, time()=1140 < 1200 threshold + - eval_time: 19m + alertname: BackendAsyncOperationClusterCreateStuck + exp_alerts: [] +# Test: does not fire when cluster create has succeeded +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/1/cluster/c", subscription_id="sub-1", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="succeeded", cluster="mgmt-1"}' + values: "0+0x25" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/1/cluster/c", subscription_id="sub-1", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="succeeded", cluster="mgmt-1"}' + values: "1+0x25" + alert_rule_test: + - eval_time: 21m + alertname: BackendAsyncOperationClusterCreateStuck + exp_alerts: [] +# Test: deduplicates across prometheus replicas (only one alert instance) +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/1/cluster/d", subscription_id="sub-1", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-0"}' + values: "0+0x25" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/1/cluster/d", subscription_id="sub-1", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-0"}' + values: "1+0x25" + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/1/cluster/d", subscription_id="sub-1", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-1"}' + values: "0+0x25" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/1/cluster/d", subscription_id="sub-1", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-1"}' + values: "1+0x25" + alert_rule_test: + - eval_time: 21m + alertname: BackendAsyncOperationClusterCreateStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/1/cluster/d" + subscription_id: "sub-1" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters" + operation_type: "create" + phase: "provisioning" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/1/cluster/d' + summary: 'Cluster create operation stuck' + description: 'Cluster create operation for /sub/1/cluster/d has been running for over 20 minutes and is currently in provisioning phase (e2e timeout: ClusterCreationTimeout).' + runbook_url: TBD +# ======================================== +# Cluster update: threshold 2700s (45 min) +# ======================================== +# Test: fires when cluster update exceeds 45 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/2/cluster/a", subscription_id="sub-2", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="update", phase="updating", cluster="mgmt-1"}' + values: "0+0x50" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/2/cluster/a", subscription_id="sub-2", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="update", phase="updating", cluster="mgmt-1"}' + values: "1+0x50" + alert_rule_test: + # At 46m, time()=2760 > 2700 threshold + - eval_time: 46m + alertname: BackendAsyncOperationClusterUpdateStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/2/cluster/a" + subscription_id: "sub-2" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters" + operation_type: "update" + phase: "updating" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/2/cluster/a' + summary: 'Cluster update operation stuck' + description: 'Cluster update operation for /sub/2/cluster/a has been running for over 45 minutes and is currently in updating phase (e2e timeout: HCPClusterVersionUpgradeTimeout).' + runbook_url: TBD +# Test: does not fire when cluster update is under 45 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/2/cluster/b", subscription_id="sub-2", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="update", phase="updating", cluster="mgmt-1"}' + values: "0+0x45" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/2/cluster/b", subscription_id="sub-2", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="update", phase="updating", cluster="mgmt-1"}' + values: "1+0x45" + alert_rule_test: + # At 44m, time()=2640 < 2700 threshold + - eval_time: 44m + alertname: BackendAsyncOperationClusterUpdateStuck + exp_alerts: [] +# ======================================== +# Cluster delete: threshold 1500s (25 min) +# ======================================== +# Test: fires when cluster delete exceeds 25 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/3/cluster/a", subscription_id="sub-3", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="delete", phase="deleting", cluster="mgmt-1"}' + values: "0+0x30" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/3/cluster/a", subscription_id="sub-3", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="delete", phase="deleting", cluster="mgmt-1"}' + values: "1+0x30" + alert_rule_test: + - eval_time: 26m + alertname: BackendAsyncOperationClusterDeleteStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/3/cluster/a" + subscription_id: "sub-3" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters" + operation_type: "delete" + phase: "deleting" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/3/cluster/a' + summary: 'Cluster delete operation stuck' + description: 'Cluster delete operation for /sub/3/cluster/a has been running for over 25 minutes and is currently in deleting phase (e2e timeout: HCPClusterDeletionTimeout).' + runbook_url: TBD +# Test: does not fire when cluster delete is under 25 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/3/cluster/b", subscription_id="sub-3", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="delete", phase="deleting", cluster="mgmt-1"}' + values: "0+0x25" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/3/cluster/b", subscription_id="sub-3", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="delete", phase="deleting", cluster="mgmt-1"}' + values: "1+0x25" + alert_rule_test: + - eval_time: 24m + alertname: BackendAsyncOperationClusterDeleteStuck + exp_alerts: [] +# ======================================== +# Credential request: threshold 600s (10 min) +# ======================================== +# Test: fires when credential request exceeds 10 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/4/cluster/a", subscription_id="sub-4", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="requestcredential", phase="provisioning", cluster="mgmt-1"}' + values: "0+0x15" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/4/cluster/a", subscription_id="sub-4", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="requestcredential", phase="provisioning", cluster="mgmt-1"}' + values: "1+0x15" + alert_rule_test: + # At 11m, time()=660 > 600 threshold + - eval_time: 11m + alertname: BackendAsyncOperationCredentialRequestStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/4/cluster/a" + subscription_id: "sub-4" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters" + operation_type: "requestcredential" + phase: "provisioning" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/4/cluster/a' + summary: 'Credential request operation stuck' + description: 'Credential request operation for /sub/4/cluster/a has been running for over 10 minutes and is currently in provisioning phase (e2e timeout: GetAdminRESTConfigTimeout).' + runbook_url: TBD +# Test: does not fire when credential request is under 10 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/4/cluster/b", subscription_id="sub-4", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="requestcredential", phase="provisioning", cluster="mgmt-1"}' + values: "0+0x10" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/4/cluster/b", subscription_id="sub-4", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="requestcredential", phase="provisioning", cluster="mgmt-1"}' + values: "1+0x10" + alert_rule_test: + - eval_time: 9m + alertname: BackendAsyncOperationCredentialRequestStuck + exp_alerts: [] +# Test: deduplicates credential request across prometheus replicas +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/4/cluster/c", subscription_id="sub-4", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="requestcredential", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-0"}' + values: "0+0x15" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/4/cluster/c", subscription_id="sub-4", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="requestcredential", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-0"}' + values: "1+0x15" + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/4/cluster/c", subscription_id="sub-4", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="requestcredential", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-1"}' + values: "0+0x15" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/4/cluster/c", subscription_id="sub-4", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="requestcredential", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-1"}' + values: "1+0x15" + alert_rule_test: + - eval_time: 11m + alertname: BackendAsyncOperationCredentialRequestStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/4/cluster/c" + subscription_id: "sub-4" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters" + operation_type: "requestcredential" + phase: "provisioning" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/4/cluster/c' + summary: 'Credential request operation stuck' + description: 'Credential request operation for /sub/4/cluster/c has been running for over 10 minutes and is currently in provisioning phase (e2e timeout: GetAdminRESTConfigTimeout).' + runbook_url: TBD +# ======================================== +# Credential revoke: threshold 900s (15 min) +# ======================================== +# Test: fires when credential revoke exceeds 15 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/5/cluster/a", subscription_id="sub-5", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="revokecredentials", phase="deleting", cluster="mgmt-1"}' + values: "0+0x20" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/5/cluster/a", subscription_id="sub-5", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="revokecredentials", phase="deleting", cluster="mgmt-1"}' + values: "1+0x20" + alert_rule_test: + # At 16m, time()=960 > 900 threshold + - eval_time: 16m + alertname: BackendAsyncOperationCredentialRevokeStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/5/cluster/a" + subscription_id: "sub-5" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters" + operation_type: "revokecredentials" + phase: "deleting" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/5/cluster/a' + summary: 'Credential revoke operation stuck' + description: 'Credential revoke operation for /sub/5/cluster/a has been running for over 15 minutes and is currently in deleting phase.' + runbook_url: TBD +# Test: does not fire when credential revoke is under 15 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/5/cluster/b", subscription_id="sub-5", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="revokecredentials", phase="deleting", cluster="mgmt-1"}' + values: "0+0x15" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/5/cluster/b", subscription_id="sub-5", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="revokecredentials", phase="deleting", cluster="mgmt-1"}' + values: "1+0x15" + alert_rule_test: + - eval_time: 14m + alertname: BackendAsyncOperationCredentialRevokeStuck + exp_alerts: [] +# ======================================== +# Node pool create: threshold 1200s (20 min) +# ======================================== +# Test: fires when node pool create exceeds 20 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/6/np/a", subscription_id="sub-6", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="create", phase="provisioning", cluster="mgmt-1"}' + values: "0+0x25" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/6/np/a", subscription_id="sub-6", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="create", phase="provisioning", cluster="mgmt-1"}' + values: "1+0x25" + alert_rule_test: + - eval_time: 21m + alertname: BackendAsyncOperationNodePoolCreateStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/6/np/a" + subscription_id: "sub-6" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters/nodepools" + operation_type: "create" + phase: "provisioning" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/6/np/a' + summary: 'Node pool create operation stuck' + description: 'Node pool create operation for /sub/6/np/a has been running for over 20 minutes and is currently in provisioning phase (e2e timeout: NodePoolCreationTimeout).' + runbook_url: TBD +# Test: does not fire when node pool create is under 20 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/6/np/b", subscription_id="sub-6", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="create", phase="provisioning", cluster="mgmt-1"}' + values: "0+0x20" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/6/np/b", subscription_id="sub-6", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="create", phase="provisioning", cluster="mgmt-1"}' + values: "1+0x20" + alert_rule_test: + - eval_time: 19m + alertname: BackendAsyncOperationNodePoolCreateStuck + exp_alerts: [] +# Test: deduplicates node pool create across prometheus replicas +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/6/np/c", subscription_id="sub-6", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="create", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-0"}' + values: "0+0x25" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/6/np/c", subscription_id="sub-6", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="create", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-0"}' + values: "1+0x25" + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/6/np/c", subscription_id="sub-6", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="create", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-1"}' + values: "0+0x25" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/6/np/c", subscription_id="sub-6", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="create", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-1"}' + values: "1+0x25" + alert_rule_test: + - eval_time: 21m + alertname: BackendAsyncOperationNodePoolCreateStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/6/np/c" + subscription_id: "sub-6" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters/nodepools" + operation_type: "create" + phase: "provisioning" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/6/np/c' + summary: 'Node pool create operation stuck' + description: 'Node pool create operation for /sub/6/np/c has been running for over 20 minutes and is currently in provisioning phase (e2e timeout: NodePoolCreationTimeout).' + runbook_url: TBD +# ======================================== +# Node pool update: threshold 2700s (45 min) +# ======================================== +# Test: fires when node pool update exceeds 45 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/7/np/a", subscription_id="sub-7", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="update", phase="updating", cluster="mgmt-1"}' + values: "0+0x50" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/7/np/a", subscription_id="sub-7", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="update", phase="updating", cluster="mgmt-1"}' + values: "1+0x50" + alert_rule_test: + - eval_time: 46m + alertname: BackendAsyncOperationNodePoolUpdateStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/7/np/a" + subscription_id: "sub-7" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters/nodepools" + operation_type: "update" + phase: "updating" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/7/np/a' + summary: 'Node pool update operation stuck' + description: 'Node pool update operation for /sub/7/np/a has been running for over 45 minutes and is currently in updating phase (e2e timeout: NodePoolVersionUpgradeTimeout).' + runbook_url: TBD +# Test: does not fire when node pool update is under 45 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/7/np/b", subscription_id="sub-7", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="update", phase="updating", cluster="mgmt-1"}' + values: "0+0x45" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/7/np/b", subscription_id="sub-7", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="update", phase="updating", cluster="mgmt-1"}' + values: "1+0x45" + alert_rule_test: + - eval_time: 44m + alertname: BackendAsyncOperationNodePoolUpdateStuck + exp_alerts: [] +# ======================================== +# Node pool delete: threshold 1500s (25 min) +# ======================================== +# Test: fires when node pool delete exceeds 25 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/8/np/a", subscription_id="sub-8", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="delete", phase="deleting", cluster="mgmt-1"}' + values: "0+0x30" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/8/np/a", subscription_id="sub-8", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="delete", phase="deleting", cluster="mgmt-1"}' + values: "1+0x30" + alert_rule_test: + - eval_time: 26m + alertname: BackendAsyncOperationNodePoolDeleteStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/8/np/a" + subscription_id: "sub-8" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters/nodepools" + operation_type: "delete" + phase: "deleting" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/8/np/a' + summary: 'Node pool delete operation stuck' + description: 'Node pool delete operation for /sub/8/np/a has been running for over 25 minutes and is currently in deleting phase (e2e timeout: NodePoolDeletionTimeout).' + runbook_url: TBD +# Test: does not fire when node pool delete is under 25 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/8/np/b", subscription_id="sub-8", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="delete", phase="deleting", cluster="mgmt-1"}' + values: "0+0x25" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/8/np/b", subscription_id="sub-8", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="delete", phase="deleting", cluster="mgmt-1"}' + values: "1+0x25" + alert_rule_test: + - eval_time: 24m + alertname: BackendAsyncOperationNodePoolDeleteStuck + exp_alerts: [] +# ======================================== +# External auth create: threshold 900s (15 min) +# ======================================== +# Test: fires when external auth create exceeds 15 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/9/ea/a", subscription_id="sub-9", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="create", phase="awaitingsecret", cluster="mgmt-1"}' + values: "0+0x20" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/9/ea/a", subscription_id="sub-9", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="create", phase="awaitingsecret", cluster="mgmt-1"}' + values: "1+0x20" + alert_rule_test: + # At 16m, time()=960 > 900 threshold + - eval_time: 16m + alertname: BackendAsyncOperationExternalAuthCreateStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/9/ea/a" + subscription_id: "sub-9" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters/externalauths" + operation_type: "create" + phase: "awaitingsecret" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/9/ea/a' + summary: 'External auth create operation stuck' + description: 'External auth create operation for /sub/9/ea/a has been running for over 15 minutes and is currently in awaitingsecret phase (e2e timeout: ExternalAuthCreationTimeout).' + runbook_url: TBD +# Test: does not fire when external auth create is under 15 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/9/ea/b", subscription_id="sub-9", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="create", phase="awaitingsecret", cluster="mgmt-1"}' + values: "0+0x15" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/9/ea/b", subscription_id="sub-9", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="create", phase="awaitingsecret", cluster="mgmt-1"}' + values: "1+0x15" + alert_rule_test: + - eval_time: 14m + alertname: BackendAsyncOperationExternalAuthCreateStuck + exp_alerts: [] +# Test: deduplicates external auth create across prometheus replicas +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/9/ea/c", subscription_id="sub-9", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="create", phase="awaitingsecret", cluster="mgmt-1", prometheus_replica="prometheus-k8s-0"}' + values: "0+0x20" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/9/ea/c", subscription_id="sub-9", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="create", phase="awaitingsecret", cluster="mgmt-1", prometheus_replica="prometheus-k8s-0"}' + values: "1+0x20" + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/9/ea/c", subscription_id="sub-9", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="create", phase="awaitingsecret", cluster="mgmt-1", prometheus_replica="prometheus-k8s-1"}' + values: "0+0x20" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/9/ea/c", subscription_id="sub-9", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="create", phase="awaitingsecret", cluster="mgmt-1", prometheus_replica="prometheus-k8s-1"}' + values: "1+0x20" + alert_rule_test: + - eval_time: 16m + alertname: BackendAsyncOperationExternalAuthCreateStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/9/ea/c" + subscription_id: "sub-9" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters/externalauths" + operation_type: "create" + phase: "awaitingsecret" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/9/ea/c' + summary: 'External auth create operation stuck' + description: 'External auth create operation for /sub/9/ea/c has been running for over 15 minutes and is currently in awaitingsecret phase (e2e timeout: ExternalAuthCreationTimeout).' + runbook_url: TBD +# ======================================== +# External auth update: threshold 600s (10 min) +# ======================================== +# Test: fires when external auth update exceeds 10 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/10/ea/a", subscription_id="sub-10", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="update", phase="updating", cluster="mgmt-1"}' + values: "0+0x15" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/10/ea/a", subscription_id="sub-10", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="update", phase="updating", cluster="mgmt-1"}' + values: "1+0x15" + alert_rule_test: + # At 11m, time()=660 > 600 threshold + - eval_time: 11m + alertname: BackendAsyncOperationExternalAuthUpdateStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/10/ea/a" + subscription_id: "sub-10" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters/externalauths" + operation_type: "update" + phase: "updating" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/10/ea/a' + summary: 'External auth update operation stuck' + description: 'External auth update operation for /sub/10/ea/a has been running for over 10 minutes and is currently in updating phase.' + runbook_url: TBD +# Test: does not fire when external auth update is under 10 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/10/ea/b", subscription_id="sub-10", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="update", phase="updating", cluster="mgmt-1"}' + values: "0+0x10" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/10/ea/b", subscription_id="sub-10", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="update", phase="updating", cluster="mgmt-1"}' + values: "1+0x10" + alert_rule_test: + - eval_time: 9m + alertname: BackendAsyncOperationExternalAuthUpdateStuck + exp_alerts: [] +# ======================================== +# External auth delete: threshold 900s (15 min) +# ======================================== +# Test: fires when external auth delete exceeds 15 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/11/ea/a", subscription_id="sub-11", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="delete", phase="deleting", cluster="mgmt-1"}' + values: "0+0x20" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/11/ea/a", subscription_id="sub-11", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="delete", phase="deleting", cluster="mgmt-1"}' + values: "1+0x20" + alert_rule_test: + # At 16m, time()=960 > 900 threshold + - eval_time: 16m + alertname: BackendAsyncOperationExternalAuthDeleteStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/11/ea/a" + subscription_id: "sub-11" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters/externalauths" + operation_type: "delete" + phase: "deleting" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/11/ea/a' + summary: 'External auth delete operation stuck' + description: 'External auth delete operation for /sub/11/ea/a has been running for over 15 minutes and is currently in deleting phase (e2e timeout: ExternalAuthDeletionTimeout).' + runbook_url: TBD +# Test: does not fire when external auth delete is under 15 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/11/ea/b", subscription_id="sub-11", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="delete", phase="deleting", cluster="mgmt-1"}' + values: "0+0x15" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/11/ea/b", subscription_id="sub-11", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="delete", phase="deleting", cluster="mgmt-1"}' + values: "1+0x15" + alert_rule_test: + - eval_time: 14m + alertname: BackendAsyncOperationExternalAuthDeleteStuck + exp_alerts: [] +# ======================================== +# Backstop: threshold 3600s (60 min) +# ======================================== +# Test: fires for any operation type exceeding 60 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/12/cluster/a", subscription_id="sub-12", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1"}' + values: "0+0x65" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/12/cluster/a", subscription_id="sub-12", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1"}' + values: "1+0x65" + alert_rule_test: + # At 61m, time()=3660 > 3600 threshold. Both specific and backstop fire. + - eval_time: 61m + alertname: BackendAsyncOperationStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/12/cluster/a" + subscription_id: "sub-12" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters" + operation_type: "create" + phase: "provisioning" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/12/cluster/a' + summary: 'Async operation stuck' + description: 'Async create operation for /sub/12/cluster/a (microsoft.redhatopenshift/hcpopenshiftclusters) has been running for over 60 minutes and is currently in provisioning phase.' + runbook_url: TBD +# Test: backstop does not fire when operation is under 60 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/12/cluster/b", subscription_id="sub-12", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1"}' + values: "0+0x60" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/12/cluster/b", subscription_id="sub-12", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1"}' + values: "1+0x60" + alert_rule_test: + - eval_time: 59m + alertname: BackendAsyncOperationStuck + exp_alerts: [] +# Test: deduplicates backstop alert across prometheus replicas +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/12/cluster/d", subscription_id="sub-12", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-0"}' + values: "0+0x65" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/12/cluster/d", subscription_id="sub-12", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-0"}' + values: "1+0x65" + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/12/cluster/d", subscription_id="sub-12", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-1"}' + values: "0+0x65" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/12/cluster/d", subscription_id="sub-12", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-1"}' + values: "1+0x65" + alert_rule_test: + - eval_time: 61m + alertname: BackendAsyncOperationStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/12/cluster/d" + subscription_id: "sub-12" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters" + operation_type: "create" + phase: "provisioning" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/12/cluster/d' + summary: 'Async operation stuck' + description: 'Async create operation for /sub/12/cluster/d (microsoft.redhatopenshift/hcpopenshiftclusters) has been running for over 60 minutes and is currently in provisioning phase.' + runbook_url: TBD +# Test: backstop does not fire for terminal phase (succeeded) +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/12/cluster/c", subscription_id="sub-12", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="succeeded", cluster="mgmt-1"}' + values: "0+0x65" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/12/cluster/c", subscription_id="sub-12", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="succeeded", cluster="mgmt-1"}' + values: "1+0x65" + alert_rule_test: + - eval_time: 61m + alertname: BackendAsyncOperationStuck + exp_alerts: [] diff --git a/dev-infrastructure/modules/metrics/rules/generatedPrometheusAlertingRules.bicep b/dev-infrastructure/modules/metrics/rules/generatedPrometheusAlertingRules.bicep index 94c2a5c07fd..2aae38dbb7f 100644 --- a/dev-infrastructure/modules/metrics/rules/generatedPrometheusAlertingRules.bicep +++ b/dev-infrastructure/modules/metrics/rules/generatedPrometheusAlertingRules.bicep @@ -661,6 +661,331 @@ resource backend 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = } } +resource backendAsyncOperations 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'backend-async-operations' + location: location + properties: { + interval: 'PT1M' + rules: [ + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'BackendAsyncOperationClusterCreateStuck' + enabled: true + labels: { + severity: '3' + } + annotations: { + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + description: 'Cluster create operation for {{ $labels.resource_id }} has been running for over 20 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: ClusterCreationTimeout).' + info: 'Cluster create operation for {{ $labels.resource_id }} has been running for over 20 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: ClusterCreationTimeout).' + runbook_url: 'TBD' + summary: 'Cluster create operation stuck' + title: 'Cluster create operation stuck resource_id:{{ $labels.resource_id }} phase:{{ $labels.phase }}' + } + expression: '(max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (time() - backend_resource_operation_start_time_seconds{operation_type="create",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters"}) and max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (backend_resource_operation_phase_info{operation_type="create",phase=~"accepted|provisioning",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters"}) == 1) > 1200' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'BackendAsyncOperationClusterUpdateStuck' + enabled: true + labels: { + severity: '3' + } + annotations: { + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + description: 'Cluster update operation for {{ $labels.resource_id }} has been running for over 45 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: HCPClusterVersionUpgradeTimeout).' + info: 'Cluster update operation for {{ $labels.resource_id }} has been running for over 45 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: HCPClusterVersionUpgradeTimeout).' + runbook_url: 'TBD' + summary: 'Cluster update operation stuck' + title: 'Cluster update operation stuck resource_id:{{ $labels.resource_id }} phase:{{ $labels.phase }}' + } + expression: '(max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (time() - backend_resource_operation_start_time_seconds{operation_type="update",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters"}) and max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (backend_resource_operation_phase_info{operation_type="update",phase=~"accepted|updating",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters"}) == 1) > 2700' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'BackendAsyncOperationClusterDeleteStuck' + enabled: true + labels: { + severity: '3' + } + annotations: { + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + description: 'Cluster delete operation for {{ $labels.resource_id }} has been running for over 25 minutes and is currently in deleting phase (e2e timeout: HCPClusterDeletionTimeout).' + info: 'Cluster delete operation for {{ $labels.resource_id }} has been running for over 25 minutes and is currently in deleting phase (e2e timeout: HCPClusterDeletionTimeout).' + runbook_url: 'TBD' + summary: 'Cluster delete operation stuck' + title: 'Cluster delete operation stuck resource_id:{{ $labels.resource_id }}' + } + expression: '(max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (time() - backend_resource_operation_start_time_seconds{operation_type="delete",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters"}) and max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (backend_resource_operation_phase_info{operation_type="delete",phase="deleting",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters"}) == 1) > 1500' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'BackendAsyncOperationCredentialRequestStuck' + enabled: true + labels: { + severity: '3' + } + annotations: { + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + description: 'Credential request operation for {{ $labels.resource_id }} has been running for over 10 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: GetAdminRESTConfigTimeout).' + info: 'Credential request operation for {{ $labels.resource_id }} has been running for over 10 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: GetAdminRESTConfigTimeout).' + runbook_url: 'TBD' + summary: 'Credential request operation stuck' + title: 'Credential request operation stuck resource_id:{{ $labels.resource_id }} phase:{{ $labels.phase }}' + } + expression: '(max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (time() - backend_resource_operation_start_time_seconds{operation_type="requestcredential",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters"}) and max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (backend_resource_operation_phase_info{operation_type="requestcredential",phase=~"accepted|provisioning",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters"}) == 1) > 600' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'BackendAsyncOperationCredentialRevokeStuck' + enabled: true + labels: { + severity: '3' + } + annotations: { + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + description: 'Credential revoke operation for {{ $labels.resource_id }} has been running for over 15 minutes and is currently in {{ $labels.phase }} phase.' + info: 'Credential revoke operation for {{ $labels.resource_id }} has been running for over 15 minutes and is currently in {{ $labels.phase }} phase.' + runbook_url: 'TBD' + summary: 'Credential revoke operation stuck' + title: 'Credential revoke operation stuck resource_id:{{ $labels.resource_id }} phase:{{ $labels.phase }}' + } + expression: '(max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (time() - backend_resource_operation_start_time_seconds{operation_type="revokecredentials",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters"}) and max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (backend_resource_operation_phase_info{operation_type="revokecredentials",phase=~"accepted|deleting",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters"}) == 1) > 900' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'BackendAsyncOperationNodePoolCreateStuck' + enabled: true + labels: { + severity: '3' + } + annotations: { + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + description: 'Node pool create operation for {{ $labels.resource_id }} has been running for over 20 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: NodePoolCreationTimeout).' + info: 'Node pool create operation for {{ $labels.resource_id }} has been running for over 20 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: NodePoolCreationTimeout).' + runbook_url: 'TBD' + summary: 'Node pool create operation stuck' + title: 'Node pool create operation stuck resource_id:{{ $labels.resource_id }} phase:{{ $labels.phase }}' + } + expression: '(max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (time() - backend_resource_operation_start_time_seconds{operation_type="create",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools"}) and max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (backend_resource_operation_phase_info{operation_type="create",phase=~"accepted|provisioning",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools"}) == 1) > 1200' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'BackendAsyncOperationNodePoolUpdateStuck' + enabled: true + labels: { + severity: '3' + } + annotations: { + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + description: 'Node pool update operation for {{ $labels.resource_id }} has been running for over 45 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: NodePoolVersionUpgradeTimeout).' + info: 'Node pool update operation for {{ $labels.resource_id }} has been running for over 45 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: NodePoolVersionUpgradeTimeout).' + runbook_url: 'TBD' + summary: 'Node pool update operation stuck' + title: 'Node pool update operation stuck resource_id:{{ $labels.resource_id }} phase:{{ $labels.phase }}' + } + expression: '(max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (time() - backend_resource_operation_start_time_seconds{operation_type="update",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools"}) and max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (backend_resource_operation_phase_info{operation_type="update",phase=~"accepted|updating",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools"}) == 1) > 2700' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'BackendAsyncOperationNodePoolDeleteStuck' + enabled: true + labels: { + severity: '3' + } + annotations: { + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + description: 'Node pool delete operation for {{ $labels.resource_id }} has been running for over 25 minutes and is currently in deleting phase (e2e timeout: NodePoolDeletionTimeout).' + info: 'Node pool delete operation for {{ $labels.resource_id }} has been running for over 25 minutes and is currently in deleting phase (e2e timeout: NodePoolDeletionTimeout).' + runbook_url: 'TBD' + summary: 'Node pool delete operation stuck' + title: 'Node pool delete operation stuck resource_id:{{ $labels.resource_id }}' + } + expression: '(max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (time() - backend_resource_operation_start_time_seconds{operation_type="delete",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools"}) and max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (backend_resource_operation_phase_info{operation_type="delete",phase="deleting",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools"}) == 1) > 1500' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'BackendAsyncOperationExternalAuthCreateStuck' + enabled: true + labels: { + severity: '3' + } + annotations: { + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + description: 'External auth create operation for {{ $labels.resource_id }} has been running for over 15 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: ExternalAuthCreationTimeout).' + info: 'External auth create operation for {{ $labels.resource_id }} has been running for over 15 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: ExternalAuthCreationTimeout).' + runbook_url: 'TBD' + summary: 'External auth create operation stuck' + title: 'External auth create operation stuck resource_id:{{ $labels.resource_id }} phase:{{ $labels.phase }}' + } + expression: '(max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (time() - backend_resource_operation_start_time_seconds{operation_type="create",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths"}) and max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (backend_resource_operation_phase_info{operation_type="create",phase=~"accepted|awaitingsecret|provisioning",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths"}) == 1) > 900' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'BackendAsyncOperationExternalAuthUpdateStuck' + enabled: true + labels: { + severity: '3' + } + annotations: { + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + description: 'External auth update operation for {{ $labels.resource_id }} has been running for over 10 minutes and is currently in {{ $labels.phase }} phase.' + info: 'External auth update operation for {{ $labels.resource_id }} has been running for over 10 minutes and is currently in {{ $labels.phase }} phase.' + runbook_url: 'TBD' + summary: 'External auth update operation stuck' + title: 'External auth update operation stuck resource_id:{{ $labels.resource_id }} phase:{{ $labels.phase }}' + } + expression: '(max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (time() - backend_resource_operation_start_time_seconds{operation_type="update",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths"}) and max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (backend_resource_operation_phase_info{operation_type="update",phase=~"accepted|updating",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths"}) == 1) > 600' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'BackendAsyncOperationExternalAuthDeleteStuck' + enabled: true + labels: { + severity: '3' + } + annotations: { + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + description: 'External auth delete operation for {{ $labels.resource_id }} has been running for over 15 minutes and is currently in deleting phase (e2e timeout: ExternalAuthDeletionTimeout).' + info: 'External auth delete operation for {{ $labels.resource_id }} has been running for over 15 minutes and is currently in deleting phase (e2e timeout: ExternalAuthDeletionTimeout).' + runbook_url: 'TBD' + summary: 'External auth delete operation stuck' + title: 'External auth delete operation stuck resource_id:{{ $labels.resource_id }}' + } + expression: '(max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (time() - backend_resource_operation_start_time_seconds{operation_type="delete",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths"}) and max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (backend_resource_operation_phase_info{operation_type="delete",phase="deleting",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths"}) == 1) > 900' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'BackendAsyncOperationStuck' + enabled: true + labels: { + severity: '3' + } + annotations: { + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + description: 'Async {{ $labels.operation_type }} operation for {{ $labels.resource_id }} ({{ $labels.resource_type }}) has been running for over 60 minutes and is currently in {{ $labels.phase }} phase.' + info: 'Async {{ $labels.operation_type }} operation for {{ $labels.resource_id }} ({{ $labels.resource_type }}) has been running for over 60 minutes and is currently in {{ $labels.phase }} phase.' + runbook_url: 'TBD' + summary: 'Async operation stuck' + title: 'Async operation stuck operation_type:{{ $labels.operation_type }} resource_id:{{ $labels.resource_id }} resource_type:{{ $labels.resource_type }} phase:{{ $labels.phase }}' + } + expression: '(max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (time() - backend_resource_operation_start_time_seconds) and max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (backend_resource_operation_phase_info{phase=~"accepted|provisioning|updating|deleting|awaitingsecret"}) == 1) > 3600' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + ] + scopes: [ + azureMonitoring + ] + } +} + resource fleet 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { name: 'fleet' location: location diff --git a/observability/alerts-sl-services.yaml b/observability/alerts-sl-services.yaml index 5a9508d947e..4fb78ce149a 100644 --- a/observability/alerts-sl-services.yaml +++ b/observability/alerts-sl-services.yaml @@ -3,6 +3,7 @@ prometheusRules: - alerts/prometheus-prometheusRule.yaml - ../frontend/alerts/frontend-prometheusRule.yaml - ../backend/alerts/backend-prometheusRule.yaml + - ../backend/alerts/backend-async-operations-prometheusRule.yaml - ../fleet/alerts/fleet-prometheusRule.yaml - ../admin/alerts/admin-prometheusRule.yaml - ../maestro/alerts/maestro-prometheusRule.yaml