diff --git a/backend/alerts/backend-async-operations-prometheusRule.yaml b/backend/alerts/backend-async-operations-prometheusRule.yaml new file mode 100644 index 00000000000..bdbb4e5f049 --- /dev/null +++ b/backend/alerts/backend-async-operations-prometheusRule.yaml @@ -0,0 +1,361 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + app.kubernetes.io/name: kube-prometheus + app.kubernetes.io/part-of: kube-prometheus + prometheus: k8s + role: alert-rules + name: backend-async-operations-rules + namespace: monitoring +spec: + groups: + # ======================================== + # Backend Async Operation Stuck Alerts + # ======================================== + # These alerts fire when an async operation has been running longer than + # the corresponding e2e test timeout. Thresholds are anchored to the + # canonical timeout constants in test/util/framework/constants.go. + # + # Metric: backend_resource_operation_start_time_seconds (total wall-clock + # time since the operation was created). Combined with phase_info to + # restrict to operations still in non-terminal phases. + # + # No `for` clause: the phase filter already ensures we only alert on + # in-flight operations. Once an operation reaches succeeded/failed/canceled, + # the phase no longer matches and the alert resolves immediately. + # + # All expressions use max by(...) to deduplicate across Prometheus + # replicas/shards, preventing duplicate alerts for the same operation. + - name: backend-async-operations + rules: + # ---------------------------------------- + # Cluster operations + # ---------------------------------------- + # Cluster create: e2e timeout = 20 minutes (ClusterCreationTimeout) + - alert: BackendAsyncOperationClusterCreateStuck + expr: | + ( + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + time() - backend_resource_operation_start_time_seconds{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", + operation_type="create" + } + ) + and + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + backend_resource_operation_phase_info{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", + operation_type="create", + phase=~"accepted|provisioning" + } + ) == 1 + ) > 1200 + labels: + severity: "3" + annotations: + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + summary: 'Cluster create operation stuck' + description: 'Cluster create operation for {{ $labels.resource_id }} has been running for over 20 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: ClusterCreationTimeout).' + runbook_url: TBD + # Cluster update: e2e timeout = 45 minutes (HCPClusterVersionUpgradeTimeout) + # This covers both property updates and version upgrades, since both + # use operation_type=update. The higher timeout accommodates upgrades. + - alert: BackendAsyncOperationClusterUpdateStuck + expr: | + ( + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + time() - backend_resource_operation_start_time_seconds{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", + operation_type="update" + } + ) + and + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + backend_resource_operation_phase_info{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", + operation_type="update", + phase=~"accepted|updating" + } + ) == 1 + ) > 2700 + labels: + severity: "3" + annotations: + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + summary: 'Cluster update operation stuck' + description: 'Cluster update operation for {{ $labels.resource_id }} has been running for over 45 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: HCPClusterVersionUpgradeTimeout).' + runbook_url: TBD + # Cluster delete: e2e timeout = 25 minutes (HCPClusterDeletionTimeout) + # Delete operations start directly in "deleting" phase, skipping "accepted". + - alert: BackendAsyncOperationClusterDeleteStuck + expr: | + ( + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + time() - backend_resource_operation_start_time_seconds{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", + operation_type="delete" + } + ) + and + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + backend_resource_operation_phase_info{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", + operation_type="delete", + phase="deleting" + } + ) == 1 + ) > 1500 + labels: + severity: "3" + annotations: + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + summary: 'Cluster delete operation stuck' + description: 'Cluster delete operation for {{ $labels.resource_id }} has been running for over 25 minutes and is currently in deleting phase (e2e timeout: HCPClusterDeletionTimeout).' + runbook_url: TBD + # ---------------------------------------- + # Credential operations + # ---------------------------------------- + # Request credential: e2e timeout = 10 minutes (GetAdminRESTConfigTimeout) + - alert: BackendAsyncOperationCredentialRequestStuck + expr: | + ( + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + time() - backend_resource_operation_start_time_seconds{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", + operation_type="requestcredential" + } + ) + and + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + backend_resource_operation_phase_info{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", + operation_type="requestcredential", + phase=~"accepted|provisioning" + } + ) == 1 + ) > 600 + labels: + severity: "3" + annotations: + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + summary: 'Credential request operation stuck' + description: 'Credential request operation for {{ $labels.resource_id }} has been running for over 10 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: GetAdminRESTConfigTimeout).' + runbook_url: TBD + # Revoke credentials: e2e timeout = 15 minutes (ExternalAuthDeletionTimeout used as proxy) + - alert: BackendAsyncOperationCredentialRevokeStuck + expr: | + ( + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + time() - backend_resource_operation_start_time_seconds{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", + operation_type="revokecredentials" + } + ) + and + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + backend_resource_operation_phase_info{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", + operation_type="revokecredentials", + phase=~"accepted|deleting" + } + ) == 1 + ) > 900 + labels: + severity: "3" + annotations: + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + summary: 'Credential revoke operation stuck' + description: 'Credential revoke operation for {{ $labels.resource_id }} has been running for over 15 minutes and is currently in {{ $labels.phase }} phase.' + runbook_url: TBD + # ---------------------------------------- + # Node pool operations + # ---------------------------------------- + # Node pool create: e2e timeout = 20 minutes (NodePoolCreationTimeout) + - alert: BackendAsyncOperationNodePoolCreateStuck + expr: | + ( + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + time() - backend_resource_operation_start_time_seconds{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", + operation_type="create" + } + ) + and + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + backend_resource_operation_phase_info{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", + operation_type="create", + phase=~"accepted|provisioning" + } + ) == 1 + ) > 1200 + labels: + severity: "3" + annotations: + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + summary: 'Node pool create operation stuck' + description: 'Node pool create operation for {{ $labels.resource_id }} has been running for over 20 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: NodePoolCreationTimeout).' + runbook_url: TBD + # Node pool update: e2e timeout = 45 minutes (NodePoolVersionUpgradeTimeout) + # Covers both property updates and version upgrades. + - alert: BackendAsyncOperationNodePoolUpdateStuck + expr: | + ( + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + time() - backend_resource_operation_start_time_seconds{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", + operation_type="update" + } + ) + and + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + backend_resource_operation_phase_info{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", + operation_type="update", + phase=~"accepted|updating" + } + ) == 1 + ) > 2700 + labels: + severity: "3" + annotations: + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + summary: 'Node pool update operation stuck' + description: 'Node pool update operation for {{ $labels.resource_id }} has been running for over 45 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: NodePoolVersionUpgradeTimeout).' + runbook_url: TBD + # Node pool delete: e2e timeout = 25 minutes (NodePoolDeletionTimeout) + - alert: BackendAsyncOperationNodePoolDeleteStuck + expr: | + ( + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + time() - backend_resource_operation_start_time_seconds{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", + operation_type="delete" + } + ) + and + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + backend_resource_operation_phase_info{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", + operation_type="delete", + phase="deleting" + } + ) == 1 + ) > 1500 + labels: + severity: "3" + annotations: + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + summary: 'Node pool delete operation stuck' + description: 'Node pool delete operation for {{ $labels.resource_id }} has been running for over 25 minutes and is currently in deleting phase (e2e timeout: NodePoolDeletionTimeout).' + runbook_url: TBD + # ---------------------------------------- + # External auth operations + # ---------------------------------------- + # External auth create: e2e timeout = 15 minutes (ExternalAuthCreationTimeout) + # Includes the awaitingsecret phase, which is specific to external auth. + - alert: BackendAsyncOperationExternalAuthCreateStuck + expr: | + ( + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + time() - backend_resource_operation_start_time_seconds{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", + operation_type="create" + } + ) + and + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + backend_resource_operation_phase_info{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", + operation_type="create", + phase=~"accepted|awaitingsecret|provisioning" + } + ) == 1 + ) > 900 + labels: + severity: "3" + annotations: + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + summary: 'External auth create operation stuck' + description: 'External auth create operation for {{ $labels.resource_id }} has been running for over 15 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: ExternalAuthCreationTimeout).' + runbook_url: TBD + # External auth update: no explicit e2e constant, using 10 minutes as + # a reasonable default (no VM provisioning involved). + - alert: BackendAsyncOperationExternalAuthUpdateStuck + expr: | + ( + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + time() - backend_resource_operation_start_time_seconds{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", + operation_type="update" + } + ) + and + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + backend_resource_operation_phase_info{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", + operation_type="update", + phase=~"accepted|updating" + } + ) == 1 + ) > 600 + labels: + severity: "3" + annotations: + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + summary: 'External auth update operation stuck' + description: 'External auth update operation for {{ $labels.resource_id }} has been running for over 10 minutes and is currently in {{ $labels.phase }} phase.' + runbook_url: TBD + # External auth delete: e2e timeout = 15 minutes (ExternalAuthDeletionTimeout) + - alert: BackendAsyncOperationExternalAuthDeleteStuck + expr: | + ( + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + time() - backend_resource_operation_start_time_seconds{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", + operation_type="delete" + } + ) + and + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + backend_resource_operation_phase_info{ + resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", + operation_type="delete", + phase="deleting" + } + ) == 1 + ) > 900 + labels: + severity: "3" + annotations: + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + summary: 'External auth delete operation stuck' + description: 'External auth delete operation for {{ $labels.resource_id }} has been running for over 15 minutes and is currently in deleting phase (e2e timeout: ExternalAuthDeletionTimeout).' + runbook_url: TBD + # ---------------------------------------- + # Backstop: any async operation not covered above + # ---------------------------------------- + # Catches any future operation types that don't have explicit alerts. + # Threshold (60 min) is higher than the longest specific alert (45 min for upgrades). + # Double-fires with specific alerts correlate to the same IcM incident via shared correlationId. + - alert: BackendAsyncOperationStuck + expr: | + ( + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + time() - backend_resource_operation_start_time_seconds + ) + and + max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) ( + backend_resource_operation_phase_info{ + phase=~"accepted|provisioning|updating|deleting|awaitingsecret" + } + ) == 1 + ) > 3600 + labels: + severity: "3" + annotations: + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + summary: 'Async operation stuck' + description: 'Async {{ $labels.operation_type }} operation for {{ $labels.resource_id }} ({{ $labels.resource_type }}) has been running for over 60 minutes and is currently in {{ $labels.phase }} phase.' + runbook_url: TBD diff --git a/backend/alerts/backend-async-operations-prometheusRule_test.yaml b/backend/alerts/backend-async-operations-prometheusRule_test.yaml new file mode 100644 index 00000000000..40070852389 --- /dev/null +++ b/backend/alerts/backend-async-operations-prometheusRule_test.yaml @@ -0,0 +1,632 @@ +rule_files: +- backend-async-operations-prometheusRule.yaml +evaluation_interval: 1m +tests: +# ======================================== +# Cluster create: threshold 1200s (20 min) +# ======================================== +# Test: fires when cluster create exceeds 20 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/1/cluster/a", subscription_id="sub-1", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1"}' + values: "0+0x25" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/1/cluster/a", subscription_id="sub-1", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1"}' + values: "1+0x25" + alert_rule_test: + # At 21m, time()=1260 > 1200 threshold + - eval_time: 21m + alertname: BackendAsyncOperationClusterCreateStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/1/cluster/a" + subscription_id: "sub-1" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters" + operation_type: "create" + phase: "provisioning" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/1/cluster/a' + summary: 'Cluster create operation stuck' + description: 'Cluster create operation for /sub/1/cluster/a has been running for over 20 minutes and is currently in provisioning phase (e2e timeout: ClusterCreationTimeout).' + runbook_url: TBD +# Test: does not fire when cluster create is under 20 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/1/cluster/b", subscription_id="sub-1", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1"}' + values: "0+0x20" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/1/cluster/b", subscription_id="sub-1", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1"}' + values: "1+0x20" + alert_rule_test: + # At 19m, time()=1140 < 1200 threshold + - eval_time: 19m + alertname: BackendAsyncOperationClusterCreateStuck + exp_alerts: [] +# Test: does not fire when cluster create has succeeded +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/1/cluster/c", subscription_id="sub-1", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="succeeded", cluster="mgmt-1"}' + values: "0+0x25" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/1/cluster/c", subscription_id="sub-1", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="succeeded", cluster="mgmt-1"}' + values: "1+0x25" + alert_rule_test: + - eval_time: 21m + alertname: BackendAsyncOperationClusterCreateStuck + exp_alerts: [] +# Test: deduplicates across prometheus replicas (only one alert instance) +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/1/cluster/d", subscription_id="sub-1", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-0"}' + values: "0+0x25" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/1/cluster/d", subscription_id="sub-1", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-0"}' + values: "1+0x25" + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/1/cluster/d", subscription_id="sub-1", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-1"}' + values: "0+0x25" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/1/cluster/d", subscription_id="sub-1", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-1"}' + values: "1+0x25" + alert_rule_test: + - eval_time: 21m + alertname: BackendAsyncOperationClusterCreateStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/1/cluster/d" + subscription_id: "sub-1" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters" + operation_type: "create" + phase: "provisioning" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/1/cluster/d' + summary: 'Cluster create operation stuck' + description: 'Cluster create operation for /sub/1/cluster/d has been running for over 20 minutes and is currently in provisioning phase (e2e timeout: ClusterCreationTimeout).' + runbook_url: TBD +# ======================================== +# Cluster update: threshold 2700s (45 min) +# ======================================== +# Test: fires when cluster update exceeds 45 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/2/cluster/a", subscription_id="sub-2", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="update", phase="updating", cluster="mgmt-1"}' + values: "0+0x50" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/2/cluster/a", subscription_id="sub-2", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="update", phase="updating", cluster="mgmt-1"}' + values: "1+0x50" + alert_rule_test: + # At 46m, time()=2760 > 2700 threshold + - eval_time: 46m + alertname: BackendAsyncOperationClusterUpdateStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/2/cluster/a" + subscription_id: "sub-2" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters" + operation_type: "update" + phase: "updating" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/2/cluster/a' + summary: 'Cluster update operation stuck' + description: 'Cluster update operation for /sub/2/cluster/a has been running for over 45 minutes and is currently in updating phase (e2e timeout: HCPClusterVersionUpgradeTimeout).' + runbook_url: TBD +# Test: does not fire when cluster update is under 45 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/2/cluster/b", subscription_id="sub-2", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="update", phase="updating", cluster="mgmt-1"}' + values: "0+0x45" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/2/cluster/b", subscription_id="sub-2", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="update", phase="updating", cluster="mgmt-1"}' + values: "1+0x45" + alert_rule_test: + # At 44m, time()=2640 < 2700 threshold + - eval_time: 44m + alertname: BackendAsyncOperationClusterUpdateStuck + exp_alerts: [] +# ======================================== +# Cluster delete: threshold 1500s (25 min) +# ======================================== +# Test: fires when cluster delete exceeds 25 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/3/cluster/a", subscription_id="sub-3", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="delete", phase="deleting", cluster="mgmt-1"}' + values: "0+0x30" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/3/cluster/a", subscription_id="sub-3", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="delete", phase="deleting", cluster="mgmt-1"}' + values: "1+0x30" + alert_rule_test: + - eval_time: 26m + alertname: BackendAsyncOperationClusterDeleteStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/3/cluster/a" + subscription_id: "sub-3" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters" + operation_type: "delete" + phase: "deleting" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/3/cluster/a' + summary: 'Cluster delete operation stuck' + description: 'Cluster delete operation for /sub/3/cluster/a has been running for over 25 minutes and is currently in deleting phase (e2e timeout: HCPClusterDeletionTimeout).' + runbook_url: TBD +# Test: does not fire when cluster delete is under 25 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/3/cluster/b", subscription_id="sub-3", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="delete", phase="deleting", cluster="mgmt-1"}' + values: "0+0x25" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/3/cluster/b", subscription_id="sub-3", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="delete", phase="deleting", cluster="mgmt-1"}' + values: "1+0x25" + alert_rule_test: + - eval_time: 24m + alertname: BackendAsyncOperationClusterDeleteStuck + exp_alerts: [] +# ======================================== +# Credential request: threshold 600s (10 min) +# ======================================== +# Test: fires when credential request exceeds 10 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/4/cluster/a", subscription_id="sub-4", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="requestcredential", phase="provisioning", cluster="mgmt-1"}' + values: "0+0x15" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/4/cluster/a", subscription_id="sub-4", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="requestcredential", phase="provisioning", cluster="mgmt-1"}' + values: "1+0x15" + alert_rule_test: + # At 11m, time()=660 > 600 threshold + - eval_time: 11m + alertname: BackendAsyncOperationCredentialRequestStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/4/cluster/a" + subscription_id: "sub-4" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters" + operation_type: "requestcredential" + phase: "provisioning" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/4/cluster/a' + summary: 'Credential request operation stuck' + description: 'Credential request operation for /sub/4/cluster/a has been running for over 10 minutes and is currently in provisioning phase (e2e timeout: GetAdminRESTConfigTimeout).' + runbook_url: TBD +# Test: does not fire when credential request is under 10 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/4/cluster/b", subscription_id="sub-4", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="requestcredential", phase="provisioning", cluster="mgmt-1"}' + values: "0+0x10" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/4/cluster/b", subscription_id="sub-4", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="requestcredential", phase="provisioning", cluster="mgmt-1"}' + values: "1+0x10" + alert_rule_test: + - eval_time: 9m + alertname: BackendAsyncOperationCredentialRequestStuck + exp_alerts: [] +# Test: deduplicates credential request across prometheus replicas +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/4/cluster/c", subscription_id="sub-4", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="requestcredential", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-0"}' + values: "0+0x15" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/4/cluster/c", subscription_id="sub-4", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="requestcredential", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-0"}' + values: "1+0x15" + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/4/cluster/c", subscription_id="sub-4", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="requestcredential", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-1"}' + values: "0+0x15" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/4/cluster/c", subscription_id="sub-4", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="requestcredential", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-1"}' + values: "1+0x15" + alert_rule_test: + - eval_time: 11m + alertname: BackendAsyncOperationCredentialRequestStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/4/cluster/c" + subscription_id: "sub-4" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters" + operation_type: "requestcredential" + phase: "provisioning" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/4/cluster/c' + summary: 'Credential request operation stuck' + description: 'Credential request operation for /sub/4/cluster/c has been running for over 10 minutes and is currently in provisioning phase (e2e timeout: GetAdminRESTConfigTimeout).' + runbook_url: TBD +# ======================================== +# Credential revoke: threshold 900s (15 min) +# ======================================== +# Test: fires when credential revoke exceeds 15 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/5/cluster/a", subscription_id="sub-5", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="revokecredentials", phase="deleting", cluster="mgmt-1"}' + values: "0+0x20" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/5/cluster/a", subscription_id="sub-5", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="revokecredentials", phase="deleting", cluster="mgmt-1"}' + values: "1+0x20" + alert_rule_test: + # At 16m, time()=960 > 900 threshold + - eval_time: 16m + alertname: BackendAsyncOperationCredentialRevokeStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/5/cluster/a" + subscription_id: "sub-5" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters" + operation_type: "revokecredentials" + phase: "deleting" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/5/cluster/a' + summary: 'Credential revoke operation stuck' + description: 'Credential revoke operation for /sub/5/cluster/a has been running for over 15 minutes and is currently in deleting phase.' + runbook_url: TBD +# Test: does not fire when credential revoke is under 15 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/5/cluster/b", subscription_id="sub-5", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="revokecredentials", phase="deleting", cluster="mgmt-1"}' + values: "0+0x15" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/5/cluster/b", subscription_id="sub-5", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="revokecredentials", phase="deleting", cluster="mgmt-1"}' + values: "1+0x15" + alert_rule_test: + - eval_time: 14m + alertname: BackendAsyncOperationCredentialRevokeStuck + exp_alerts: [] +# ======================================== +# Node pool create: threshold 1200s (20 min) +# ======================================== +# Test: fires when node pool create exceeds 20 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/6/np/a", subscription_id="sub-6", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="create", phase="provisioning", cluster="mgmt-1"}' + values: "0+0x25" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/6/np/a", subscription_id="sub-6", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="create", phase="provisioning", cluster="mgmt-1"}' + values: "1+0x25" + alert_rule_test: + - eval_time: 21m + alertname: BackendAsyncOperationNodePoolCreateStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/6/np/a" + subscription_id: "sub-6" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters/nodepools" + operation_type: "create" + phase: "provisioning" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/6/np/a' + summary: 'Node pool create operation stuck' + description: 'Node pool create operation for /sub/6/np/a has been running for over 20 minutes and is currently in provisioning phase (e2e timeout: NodePoolCreationTimeout).' + runbook_url: TBD +# Test: does not fire when node pool create is under 20 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/6/np/b", subscription_id="sub-6", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="create", phase="provisioning", cluster="mgmt-1"}' + values: "0+0x20" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/6/np/b", subscription_id="sub-6", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="create", phase="provisioning", cluster="mgmt-1"}' + values: "1+0x20" + alert_rule_test: + - eval_time: 19m + alertname: BackendAsyncOperationNodePoolCreateStuck + exp_alerts: [] +# Test: deduplicates node pool create across prometheus replicas +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/6/np/c", subscription_id="sub-6", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="create", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-0"}' + values: "0+0x25" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/6/np/c", subscription_id="sub-6", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="create", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-0"}' + values: "1+0x25" + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/6/np/c", subscription_id="sub-6", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="create", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-1"}' + values: "0+0x25" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/6/np/c", subscription_id="sub-6", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="create", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-1"}' + values: "1+0x25" + alert_rule_test: + - eval_time: 21m + alertname: BackendAsyncOperationNodePoolCreateStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/6/np/c" + subscription_id: "sub-6" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters/nodepools" + operation_type: "create" + phase: "provisioning" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/6/np/c' + summary: 'Node pool create operation stuck' + description: 'Node pool create operation for /sub/6/np/c has been running for over 20 minutes and is currently in provisioning phase (e2e timeout: NodePoolCreationTimeout).' + runbook_url: TBD +# ======================================== +# Node pool update: threshold 2700s (45 min) +# ======================================== +# Test: fires when node pool update exceeds 45 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/7/np/a", subscription_id="sub-7", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="update", phase="updating", cluster="mgmt-1"}' + values: "0+0x50" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/7/np/a", subscription_id="sub-7", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="update", phase="updating", cluster="mgmt-1"}' + values: "1+0x50" + alert_rule_test: + - eval_time: 46m + alertname: BackendAsyncOperationNodePoolUpdateStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/7/np/a" + subscription_id: "sub-7" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters/nodepools" + operation_type: "update" + phase: "updating" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/7/np/a' + summary: 'Node pool update operation stuck' + description: 'Node pool update operation for /sub/7/np/a has been running for over 45 minutes and is currently in updating phase (e2e timeout: NodePoolVersionUpgradeTimeout).' + runbook_url: TBD +# Test: does not fire when node pool update is under 45 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/7/np/b", subscription_id="sub-7", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="update", phase="updating", cluster="mgmt-1"}' + values: "0+0x45" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/7/np/b", subscription_id="sub-7", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="update", phase="updating", cluster="mgmt-1"}' + values: "1+0x45" + alert_rule_test: + - eval_time: 44m + alertname: BackendAsyncOperationNodePoolUpdateStuck + exp_alerts: [] +# ======================================== +# Node pool delete: threshold 1500s (25 min) +# ======================================== +# Test: fires when node pool delete exceeds 25 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/8/np/a", subscription_id="sub-8", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="delete", phase="deleting", cluster="mgmt-1"}' + values: "0+0x30" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/8/np/a", subscription_id="sub-8", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="delete", phase="deleting", cluster="mgmt-1"}' + values: "1+0x30" + alert_rule_test: + - eval_time: 26m + alertname: BackendAsyncOperationNodePoolDeleteStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/8/np/a" + subscription_id: "sub-8" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters/nodepools" + operation_type: "delete" + phase: "deleting" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/8/np/a' + summary: 'Node pool delete operation stuck' + description: 'Node pool delete operation for /sub/8/np/a has been running for over 25 minutes and is currently in deleting phase (e2e timeout: NodePoolDeletionTimeout).' + runbook_url: TBD +# Test: does not fire when node pool delete is under 25 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/8/np/b", subscription_id="sub-8", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="delete", phase="deleting", cluster="mgmt-1"}' + values: "0+0x25" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/8/np/b", subscription_id="sub-8", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools", operation_type="delete", phase="deleting", cluster="mgmt-1"}' + values: "1+0x25" + alert_rule_test: + - eval_time: 24m + alertname: BackendAsyncOperationNodePoolDeleteStuck + exp_alerts: [] +# ======================================== +# External auth create: threshold 900s (15 min) +# ======================================== +# Test: fires when external auth create exceeds 15 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/9/ea/a", subscription_id="sub-9", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="create", phase="awaitingsecret", cluster="mgmt-1"}' + values: "0+0x20" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/9/ea/a", subscription_id="sub-9", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="create", phase="awaitingsecret", cluster="mgmt-1"}' + values: "1+0x20" + alert_rule_test: + # At 16m, time()=960 > 900 threshold + - eval_time: 16m + alertname: BackendAsyncOperationExternalAuthCreateStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/9/ea/a" + subscription_id: "sub-9" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters/externalauths" + operation_type: "create" + phase: "awaitingsecret" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/9/ea/a' + summary: 'External auth create operation stuck' + description: 'External auth create operation for /sub/9/ea/a has been running for over 15 minutes and is currently in awaitingsecret phase (e2e timeout: ExternalAuthCreationTimeout).' + runbook_url: TBD +# Test: does not fire when external auth create is under 15 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/9/ea/b", subscription_id="sub-9", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="create", phase="awaitingsecret", cluster="mgmt-1"}' + values: "0+0x15" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/9/ea/b", subscription_id="sub-9", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="create", phase="awaitingsecret", cluster="mgmt-1"}' + values: "1+0x15" + alert_rule_test: + - eval_time: 14m + alertname: BackendAsyncOperationExternalAuthCreateStuck + exp_alerts: [] +# Test: deduplicates external auth create across prometheus replicas +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/9/ea/c", subscription_id="sub-9", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="create", phase="awaitingsecret", cluster="mgmt-1", prometheus_replica="prometheus-k8s-0"}' + values: "0+0x20" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/9/ea/c", subscription_id="sub-9", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="create", phase="awaitingsecret", cluster="mgmt-1", prometheus_replica="prometheus-k8s-0"}' + values: "1+0x20" + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/9/ea/c", subscription_id="sub-9", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="create", phase="awaitingsecret", cluster="mgmt-1", prometheus_replica="prometheus-k8s-1"}' + values: "0+0x20" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/9/ea/c", subscription_id="sub-9", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="create", phase="awaitingsecret", cluster="mgmt-1", prometheus_replica="prometheus-k8s-1"}' + values: "1+0x20" + alert_rule_test: + - eval_time: 16m + alertname: BackendAsyncOperationExternalAuthCreateStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/9/ea/c" + subscription_id: "sub-9" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters/externalauths" + operation_type: "create" + phase: "awaitingsecret" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/9/ea/c' + summary: 'External auth create operation stuck' + description: 'External auth create operation for /sub/9/ea/c has been running for over 15 minutes and is currently in awaitingsecret phase (e2e timeout: ExternalAuthCreationTimeout).' + runbook_url: TBD +# ======================================== +# External auth update: threshold 600s (10 min) +# ======================================== +# Test: fires when external auth update exceeds 10 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/10/ea/a", subscription_id="sub-10", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="update", phase="updating", cluster="mgmt-1"}' + values: "0+0x15" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/10/ea/a", subscription_id="sub-10", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="update", phase="updating", cluster="mgmt-1"}' + values: "1+0x15" + alert_rule_test: + # At 11m, time()=660 > 600 threshold + - eval_time: 11m + alertname: BackendAsyncOperationExternalAuthUpdateStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/10/ea/a" + subscription_id: "sub-10" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters/externalauths" + operation_type: "update" + phase: "updating" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/10/ea/a' + summary: 'External auth update operation stuck' + description: 'External auth update operation for /sub/10/ea/a has been running for over 10 minutes and is currently in updating phase.' + runbook_url: TBD +# Test: does not fire when external auth update is under 10 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/10/ea/b", subscription_id="sub-10", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="update", phase="updating", cluster="mgmt-1"}' + values: "0+0x10" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/10/ea/b", subscription_id="sub-10", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="update", phase="updating", cluster="mgmt-1"}' + values: "1+0x10" + alert_rule_test: + - eval_time: 9m + alertname: BackendAsyncOperationExternalAuthUpdateStuck + exp_alerts: [] +# ======================================== +# External auth delete: threshold 900s (15 min) +# ======================================== +# Test: fires when external auth delete exceeds 15 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/11/ea/a", subscription_id="sub-11", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="delete", phase="deleting", cluster="mgmt-1"}' + values: "0+0x20" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/11/ea/a", subscription_id="sub-11", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="delete", phase="deleting", cluster="mgmt-1"}' + values: "1+0x20" + alert_rule_test: + # At 16m, time()=960 > 900 threshold + - eval_time: 16m + alertname: BackendAsyncOperationExternalAuthDeleteStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/11/ea/a" + subscription_id: "sub-11" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters/externalauths" + operation_type: "delete" + phase: "deleting" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/11/ea/a' + summary: 'External auth delete operation stuck' + description: 'External auth delete operation for /sub/11/ea/a has been running for over 15 minutes and is currently in deleting phase (e2e timeout: ExternalAuthDeletionTimeout).' + runbook_url: TBD +# Test: does not fire when external auth delete is under 15 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/11/ea/b", subscription_id="sub-11", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="delete", phase="deleting", cluster="mgmt-1"}' + values: "0+0x15" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/11/ea/b", subscription_id="sub-11", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths", operation_type="delete", phase="deleting", cluster="mgmt-1"}' + values: "1+0x15" + alert_rule_test: + - eval_time: 14m + alertname: BackendAsyncOperationExternalAuthDeleteStuck + exp_alerts: [] +# ======================================== +# Backstop: threshold 3600s (60 min) +# ======================================== +# Test: fires for any operation type exceeding 60 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/12/cluster/a", subscription_id="sub-12", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1"}' + values: "0+0x65" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/12/cluster/a", subscription_id="sub-12", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1"}' + values: "1+0x65" + alert_rule_test: + # At 61m, time()=3660 > 3600 threshold. Both specific and backstop fire. + - eval_time: 61m + alertname: BackendAsyncOperationStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/12/cluster/a" + subscription_id: "sub-12" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters" + operation_type: "create" + phase: "provisioning" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/12/cluster/a' + summary: 'Async operation stuck' + description: 'Async create operation for /sub/12/cluster/a (microsoft.redhatopenshift/hcpopenshiftclusters) has been running for over 60 minutes and is currently in provisioning phase.' + runbook_url: TBD +# Test: backstop does not fire when operation is under 60 minutes +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/12/cluster/b", subscription_id="sub-12", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1"}' + values: "0+0x60" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/12/cluster/b", subscription_id="sub-12", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1"}' + values: "1+0x60" + alert_rule_test: + - eval_time: 59m + alertname: BackendAsyncOperationStuck + exp_alerts: [] +# Test: deduplicates backstop alert across prometheus replicas +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/12/cluster/d", subscription_id="sub-12", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-0"}' + values: "0+0x65" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/12/cluster/d", subscription_id="sub-12", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-0"}' + values: "1+0x65" + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/12/cluster/d", subscription_id="sub-12", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-1"}' + values: "0+0x65" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/12/cluster/d", subscription_id="sub-12", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="provisioning", cluster="mgmt-1", prometheus_replica="prometheus-k8s-1"}' + values: "1+0x65" + alert_rule_test: + - eval_time: 61m + alertname: BackendAsyncOperationStuck + exp_alerts: + - exp_labels: + resource_id: "/sub/12/cluster/d" + subscription_id: "sub-12" + resource_type: "microsoft.redhatopenshift/hcpopenshiftclusters" + operation_type: "create" + phase: "provisioning" + cluster: "mgmt-1" + severity: "3" + exp_annotations: + correlationId: 'BackendAsyncOperationStuck/mgmt-1//sub/12/cluster/d' + summary: 'Async operation stuck' + description: 'Async create operation for /sub/12/cluster/d (microsoft.redhatopenshift/hcpopenshiftclusters) has been running for over 60 minutes and is currently in provisioning phase.' + runbook_url: TBD +# Test: backstop does not fire for terminal phase (succeeded) +- interval: 1m + input_series: + - series: 'backend_resource_operation_start_time_seconds{resource_id="/sub/12/cluster/c", subscription_id="sub-12", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="succeeded", cluster="mgmt-1"}' + values: "0+0x65" + - series: 'backend_resource_operation_phase_info{resource_id="/sub/12/cluster/c", subscription_id="sub-12", resource_type="microsoft.redhatopenshift/hcpopenshiftclusters", operation_type="create", phase="succeeded", cluster="mgmt-1"}' + values: "1+0x65" + alert_rule_test: + - eval_time: 61m + alertname: BackendAsyncOperationStuck + exp_alerts: [] diff --git a/dev-infrastructure/modules/metrics/rules/generatedPrometheusAlertingRules.bicep b/dev-infrastructure/modules/metrics/rules/generatedPrometheusAlertingRules.bicep index 94c2a5c07fd..2aae38dbb7f 100644 --- a/dev-infrastructure/modules/metrics/rules/generatedPrometheusAlertingRules.bicep +++ b/dev-infrastructure/modules/metrics/rules/generatedPrometheusAlertingRules.bicep @@ -661,6 +661,331 @@ resource backend 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = } } +resource backendAsyncOperations 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'backend-async-operations' + location: location + properties: { + interval: 'PT1M' + rules: [ + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'BackendAsyncOperationClusterCreateStuck' + enabled: true + labels: { + severity: '3' + } + annotations: { + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + description: 'Cluster create operation for {{ $labels.resource_id }} has been running for over 20 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: ClusterCreationTimeout).' + info: 'Cluster create operation for {{ $labels.resource_id }} has been running for over 20 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: ClusterCreationTimeout).' + runbook_url: 'TBD' + summary: 'Cluster create operation stuck' + title: 'Cluster create operation stuck resource_id:{{ $labels.resource_id }} phase:{{ $labels.phase }}' + } + expression: '(max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (time() - backend_resource_operation_start_time_seconds{operation_type="create",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters"}) and max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (backend_resource_operation_phase_info{operation_type="create",phase=~"accepted|provisioning",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters"}) == 1) > 1200' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'BackendAsyncOperationClusterUpdateStuck' + enabled: true + labels: { + severity: '3' + } + annotations: { + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + description: 'Cluster update operation for {{ $labels.resource_id }} has been running for over 45 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: HCPClusterVersionUpgradeTimeout).' + info: 'Cluster update operation for {{ $labels.resource_id }} has been running for over 45 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: HCPClusterVersionUpgradeTimeout).' + runbook_url: 'TBD' + summary: 'Cluster update operation stuck' + title: 'Cluster update operation stuck resource_id:{{ $labels.resource_id }} phase:{{ $labels.phase }}' + } + expression: '(max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (time() - backend_resource_operation_start_time_seconds{operation_type="update",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters"}) and max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (backend_resource_operation_phase_info{operation_type="update",phase=~"accepted|updating",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters"}) == 1) > 2700' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'BackendAsyncOperationClusterDeleteStuck' + enabled: true + labels: { + severity: '3' + } + annotations: { + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + description: 'Cluster delete operation for {{ $labels.resource_id }} has been running for over 25 minutes and is currently in deleting phase (e2e timeout: HCPClusterDeletionTimeout).' + info: 'Cluster delete operation for {{ $labels.resource_id }} has been running for over 25 minutes and is currently in deleting phase (e2e timeout: HCPClusterDeletionTimeout).' + runbook_url: 'TBD' + summary: 'Cluster delete operation stuck' + title: 'Cluster delete operation stuck resource_id:{{ $labels.resource_id }}' + } + expression: '(max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (time() - backend_resource_operation_start_time_seconds{operation_type="delete",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters"}) and max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (backend_resource_operation_phase_info{operation_type="delete",phase="deleting",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters"}) == 1) > 1500' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'BackendAsyncOperationCredentialRequestStuck' + enabled: true + labels: { + severity: '3' + } + annotations: { + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + description: 'Credential request operation for {{ $labels.resource_id }} has been running for over 10 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: GetAdminRESTConfigTimeout).' + info: 'Credential request operation for {{ $labels.resource_id }} has been running for over 10 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: GetAdminRESTConfigTimeout).' + runbook_url: 'TBD' + summary: 'Credential request operation stuck' + title: 'Credential request operation stuck resource_id:{{ $labels.resource_id }} phase:{{ $labels.phase }}' + } + expression: '(max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (time() - backend_resource_operation_start_time_seconds{operation_type="requestcredential",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters"}) and max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (backend_resource_operation_phase_info{operation_type="requestcredential",phase=~"accepted|provisioning",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters"}) == 1) > 600' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'BackendAsyncOperationCredentialRevokeStuck' + enabled: true + labels: { + severity: '3' + } + annotations: { + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + description: 'Credential revoke operation for {{ $labels.resource_id }} has been running for over 15 minutes and is currently in {{ $labels.phase }} phase.' + info: 'Credential revoke operation for {{ $labels.resource_id }} has been running for over 15 minutes and is currently in {{ $labels.phase }} phase.' + runbook_url: 'TBD' + summary: 'Credential revoke operation stuck' + title: 'Credential revoke operation stuck resource_id:{{ $labels.resource_id }} phase:{{ $labels.phase }}' + } + expression: '(max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (time() - backend_resource_operation_start_time_seconds{operation_type="revokecredentials",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters"}) and max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (backend_resource_operation_phase_info{operation_type="revokecredentials",phase=~"accepted|deleting",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters"}) == 1) > 900' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'BackendAsyncOperationNodePoolCreateStuck' + enabled: true + labels: { + severity: '3' + } + annotations: { + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + description: 'Node pool create operation for {{ $labels.resource_id }} has been running for over 20 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: NodePoolCreationTimeout).' + info: 'Node pool create operation for {{ $labels.resource_id }} has been running for over 20 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: NodePoolCreationTimeout).' + runbook_url: 'TBD' + summary: 'Node pool create operation stuck' + title: 'Node pool create operation stuck resource_id:{{ $labels.resource_id }} phase:{{ $labels.phase }}' + } + expression: '(max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (time() - backend_resource_operation_start_time_seconds{operation_type="create",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools"}) and max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (backend_resource_operation_phase_info{operation_type="create",phase=~"accepted|provisioning",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools"}) == 1) > 1200' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'BackendAsyncOperationNodePoolUpdateStuck' + enabled: true + labels: { + severity: '3' + } + annotations: { + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + description: 'Node pool update operation for {{ $labels.resource_id }} has been running for over 45 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: NodePoolVersionUpgradeTimeout).' + info: 'Node pool update operation for {{ $labels.resource_id }} has been running for over 45 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: NodePoolVersionUpgradeTimeout).' + runbook_url: 'TBD' + summary: 'Node pool update operation stuck' + title: 'Node pool update operation stuck resource_id:{{ $labels.resource_id }} phase:{{ $labels.phase }}' + } + expression: '(max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (time() - backend_resource_operation_start_time_seconds{operation_type="update",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools"}) and max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (backend_resource_operation_phase_info{operation_type="update",phase=~"accepted|updating",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools"}) == 1) > 2700' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'BackendAsyncOperationNodePoolDeleteStuck' + enabled: true + labels: { + severity: '3' + } + annotations: { + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + description: 'Node pool delete operation for {{ $labels.resource_id }} has been running for over 25 minutes and is currently in deleting phase (e2e timeout: NodePoolDeletionTimeout).' + info: 'Node pool delete operation for {{ $labels.resource_id }} has been running for over 25 minutes and is currently in deleting phase (e2e timeout: NodePoolDeletionTimeout).' + runbook_url: 'TBD' + summary: 'Node pool delete operation stuck' + title: 'Node pool delete operation stuck resource_id:{{ $labels.resource_id }}' + } + expression: '(max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (time() - backend_resource_operation_start_time_seconds{operation_type="delete",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools"}) and max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (backend_resource_operation_phase_info{operation_type="delete",phase="deleting",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/nodepools"}) == 1) > 1500' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'BackendAsyncOperationExternalAuthCreateStuck' + enabled: true + labels: { + severity: '3' + } + annotations: { + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + description: 'External auth create operation for {{ $labels.resource_id }} has been running for over 15 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: ExternalAuthCreationTimeout).' + info: 'External auth create operation for {{ $labels.resource_id }} has been running for over 15 minutes and is currently in {{ $labels.phase }} phase (e2e timeout: ExternalAuthCreationTimeout).' + runbook_url: 'TBD' + summary: 'External auth create operation stuck' + title: 'External auth create operation stuck resource_id:{{ $labels.resource_id }} phase:{{ $labels.phase }}' + } + expression: '(max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (time() - backend_resource_operation_start_time_seconds{operation_type="create",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths"}) and max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (backend_resource_operation_phase_info{operation_type="create",phase=~"accepted|awaitingsecret|provisioning",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths"}) == 1) > 900' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'BackendAsyncOperationExternalAuthUpdateStuck' + enabled: true + labels: { + severity: '3' + } + annotations: { + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + description: 'External auth update operation for {{ $labels.resource_id }} has been running for over 10 minutes and is currently in {{ $labels.phase }} phase.' + info: 'External auth update operation for {{ $labels.resource_id }} has been running for over 10 minutes and is currently in {{ $labels.phase }} phase.' + runbook_url: 'TBD' + summary: 'External auth update operation stuck' + title: 'External auth update operation stuck resource_id:{{ $labels.resource_id }} phase:{{ $labels.phase }}' + } + expression: '(max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (time() - backend_resource_operation_start_time_seconds{operation_type="update",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths"}) and max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (backend_resource_operation_phase_info{operation_type="update",phase=~"accepted|updating",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths"}) == 1) > 600' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'BackendAsyncOperationExternalAuthDeleteStuck' + enabled: true + labels: { + severity: '3' + } + annotations: { + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + description: 'External auth delete operation for {{ $labels.resource_id }} has been running for over 15 minutes and is currently in deleting phase (e2e timeout: ExternalAuthDeletionTimeout).' + info: 'External auth delete operation for {{ $labels.resource_id }} has been running for over 15 minutes and is currently in deleting phase (e2e timeout: ExternalAuthDeletionTimeout).' + runbook_url: 'TBD' + summary: 'External auth delete operation stuck' + title: 'External auth delete operation stuck resource_id:{{ $labels.resource_id }}' + } + expression: '(max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (time() - backend_resource_operation_start_time_seconds{operation_type="delete",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths"}) and max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (backend_resource_operation_phase_info{operation_type="delete",phase="deleting",resource_type="microsoft.redhatopenshift/hcpopenshiftclusters/externalauths"}) == 1) > 900' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'BackendAsyncOperationStuck' + enabled: true + labels: { + severity: '3' + } + annotations: { + correlationId: 'BackendAsyncOperationStuck/{{ $labels.cluster }}/{{ $labels.resource_id }}' + description: 'Async {{ $labels.operation_type }} operation for {{ $labels.resource_id }} ({{ $labels.resource_type }}) has been running for over 60 minutes and is currently in {{ $labels.phase }} phase.' + info: 'Async {{ $labels.operation_type }} operation for {{ $labels.resource_id }} ({{ $labels.resource_type }}) has been running for over 60 minutes and is currently in {{ $labels.phase }} phase.' + runbook_url: 'TBD' + summary: 'Async operation stuck' + title: 'Async operation stuck operation_type:{{ $labels.operation_type }} resource_id:{{ $labels.resource_id }} resource_type:{{ $labels.resource_type }} phase:{{ $labels.phase }}' + } + expression: '(max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (time() - backend_resource_operation_start_time_seconds) and max by (resource_id, subscription_id, resource_type, operation_type, phase, cluster) (backend_resource_operation_phase_info{phase=~"accepted|provisioning|updating|deleting|awaitingsecret"}) == 1) > 3600' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + ] + scopes: [ + azureMonitoring + ] + } +} + resource fleet 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { name: 'fleet' location: location diff --git a/observability/alerts-sl-services.yaml b/observability/alerts-sl-services.yaml index 5a9508d947e..4fb78ce149a 100644 --- a/observability/alerts-sl-services.yaml +++ b/observability/alerts-sl-services.yaml @@ -3,6 +3,7 @@ prometheusRules: - alerts/prometheus-prometheusRule.yaml - ../frontend/alerts/frontend-prometheusRule.yaml - ../backend/alerts/backend-prometheusRule.yaml + - ../backend/alerts/backend-async-operations-prometheusRule.yaml - ../fleet/alerts/fleet-prometheusRule.yaml - ../admin/alerts/admin-prometheusRule.yaml - ../maestro/alerts/maestro-prometheusRule.yaml