Sudobrendan/unrevert 5754

dev-infrastructure/modules/metrics/rp-hcp-rules.bicep

@@ -0,0 +1,18 @@
+// Alerts for HCP KubeAPIServer availability, routed to the RP ICM queue.
+
+@description('The Azure resource ID of the Azure Monitor Workspace (stores prometheus metrics for hosted control planes)')
+param azureMonitoringWorkspaceId string
+
+param actionGroups array
+
+@description('The minimum IcM severity level (highest priority) that alerts can fire at. Alerts more critical than this ceiling will be degraded to this value. 0 means no ceiling.')
+param severityCeiling int = 0
+
+module generatedAlerts 'rules/generatedRPHCPPrometheusAlertingRules.bicep' = {
+  name: 'generatedRPHCPPrometheusAlertingRules'
+  params: {
+    azureMonitoring: azureMonitoringWorkspaceId
+    actionGroups: actionGroups
+    severityCeiling: severityCeiling
+  }
+}

dev-infrastructure/modules/metrics/rules/generatedRPHCPPrometheusAlertingRules.bicep

@@ -0,0 +1,138 @@
+#disable-next-line no-unused-params
+param azureMonitoring string
+
+#disable-next-line no-unused-params
+param actionGroups array
+
+@description('The minimum IcM severity level (highest priority) that alerts can fire at. Alerts more critical than this ceiling will be degraded to this value. 0 means no ceiling.')
+param severityCeiling int = 0
+
+#disable-next-line no-unused-params
+param location string = resourceGroup().location
+
+resource rpUserjourneyKasAvailabilityMonitorRules 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = {
+  name: 'rp-userjourney-kas-availability-monitor-rules'
+  location: location
+  properties: {
+    interval: 'PT1M'
+    rules: [
+      {
+        actions: [
+          for g in actionGroups: {
+            actionGroupId: g
+            actionProperties: {
+              'IcM.Title': '#$.labels.cluster#: #$.annotations.title#'
+              'IcM.CorrelationId': '#$.annotations.correlationId#'
+            }
+          }
+        ]
+        alert: 'userJourneyKubeApiserverAvailability1h5m'
+        enabled: true
+        labels: {
+          long_window: '1h'
+          severity: 'warning'
+          short_window: '5m'
+          slo: 'kas-availability'
+        }
+        annotations: {
+          correlationId: 'userJourneyKubeApiserverAvailability/{{ $labels.cluster }}/{{ $labels._id }}'
+          description: '''Resource ID: {{ $labels.resource_id }}
+OCM Cluster ID: {{ $labels._id }}
+Management Cluster: {{ $labels.cluster }}
+Namespace: {{ $labels.namespace }}
+'''
+          info: '''Resource ID: {{ $labels.resource_id }}
+OCM Cluster ID: {{ $labels._id }}
+Management Cluster: {{ $labels.cluster }}
+Namespace: {{ $labels.namespace }}
+'''
+          runbook_url: 'https://aka.ms/arohcp-runbook-ujkasavailable'
+          summary: '[HCPKASAvailableBurn] {{ $labels.namespace }} (1h/5m)'
+          title: '[HCPKASAvailableBurn] {{ $labels.namespace }} (1h/5m) resource_id:{{ $labels.resource_id }} _id:{{ $labels._id }} cluster:{{ $labels.cluster }}'
+        }
+        expression: '(1 - (hostedClusterAPI_kubeapiserver_available:sli_sum_5m / hostedClusterAPI_kubeapiserver_available:sli_count_5m) > (14.4 * (1 - 0.9995)) and on (name, namespace, _id, resource_id, cluster) hostedClusterAPI_kubeapiserver_available:sli_count_5m > 3) and on (name, namespace, _id, resource_id, cluster) (1 - (hostedClusterAPI_kubeapiserver_available:sli_sum_1h / hostedClusterAPI_kubeapiserver_available:sli_count_1h) > (14.4 * (1 - 0.9995)) and on (name, namespace, _id, resource_id, cluster) hostedClusterAPI_kubeapiserver_available:sli_count_1h > 54)'
+        for: 'PT10M'
+        severity: severityCeiling > 0 ? max(3, severityCeiling) : 3
+      }
+      {
+        actions: [
+          for g in actionGroups: {
+            actionGroupId: g
+            actionProperties: {
+              'IcM.Title': '#$.labels.cluster#: #$.annotations.title#'
+              'IcM.CorrelationId': '#$.annotations.correlationId#'
+            }
+          }
+        ]
+        alert: 'userJourneyKubeApiserverAvailability6h30m'
+        enabled: true
+        labels: {
+          long_window: '6h'
+          severity: 'warning'
+          short_window: '30m'
+          slo: 'kas-availability'
+        }
+        annotations: {
+          correlationId: 'userJourneyKubeApiserverAvailability/{{ $labels.cluster }}/{{ $labels._id }}'
+          description: '''Resource ID: {{ $labels.resource_id }}
+OCM Cluster ID: {{ $labels._id }}
+Management Cluster: {{ $labels.cluster }}
+Namespace: {{ $labels.namespace }}
+'''
+          info: '''Resource ID: {{ $labels.resource_id }}
+OCM Cluster ID: {{ $labels._id }}
+Management Cluster: {{ $labels.cluster }}
+Namespace: {{ $labels.namespace }}
+'''
+          runbook_url: 'https://aka.ms/arohcp-runbook-ujkasavailable'
+          summary: '[HCPKASAvailableBurn] {{ $labels.namespace }} (6h/30m)'
+          title: '[HCPKASAvailableBurn] {{ $labels.namespace }} (6h/30m) resource_id:{{ $labels.resource_id }} _id:{{ $labels._id }} cluster:{{ $labels.cluster }}'
+        }
+        expression: '(1 - (hostedClusterAPI_kubeapiserver_available:sli_sum_30m / hostedClusterAPI_kubeapiserver_available:sli_count_30m) > (6 * (1 - 0.9995)) and on (name, namespace, _id, resource_id, cluster) hostedClusterAPI_kubeapiserver_available:sli_count_30m > 27) and on (name, namespace, _id, resource_id, cluster) (1 - (hostedClusterAPI_kubeapiserver_available:sli_sum_6h / hostedClusterAPI_kubeapiserver_available:sli_count_6h) > (6 * (1 - 0.9995)) and on (name, namespace, _id, resource_id, cluster) hostedClusterAPI_kubeapiserver_available:sli_count_6h > 64)'
+        for: 'PT30M'
+        severity: severityCeiling > 0 ? max(3, severityCeiling) : 3
+      }
+      {
+        actions: [
+          for g in actionGroups: {
+            actionGroupId: g
+            actionProperties: {
+              'IcM.Title': '#$.labels.cluster#: #$.annotations.title#'
+              'IcM.CorrelationId': '#$.annotations.correlationId#'
+            }
+          }
+        ]
+        alert: 'userJourneyKubeApiserverAvailability3d6h'
+        enabled: true
+        labels: {
+          long_window: '3d'
+          severity: 'warning'
+          short_window: '6h'
+          slo: 'kas-availability'
+        }
+        annotations: {
+          correlationId: 'userJourneyKubeApiserverAvailability/{{ $labels.cluster }}/{{ $labels._id }}'
+          description: '''Resource ID: {{ $labels.resource_id }}
+OCM Cluster ID: {{ $labels._id }}
+Management Cluster: {{ $labels.cluster }}
+Namespace: {{ $labels.namespace }}
+'''
+          info: '''Resource ID: {{ $labels.resource_id }}
+OCM Cluster ID: {{ $labels._id }}
+Management Cluster: {{ $labels.cluster }}
+Namespace: {{ $labels.namespace }}
+'''
+          runbook_url: 'https://aka.ms/arohcp-runbook-ujkasavailable'
+          summary: '[HCPKASAvailableBurn] {{ $labels.namespace }} (3d/6h)'
+          title: '[HCPKASAvailableBurn] {{ $labels.namespace }} (3d/6h) resource_id:{{ $labels.resource_id }} _id:{{ $labels._id }} cluster:{{ $labels.cluster }}'
+        }
+        expression: '(1 - (hostedClusterAPI_kubeapiserver_available:sli_sum_6h / hostedClusterAPI_kubeapiserver_available:sli_count_6h) > (1 * (1 - 0.9995)) and on (name, namespace, _id, resource_id, cluster) hostedClusterAPI_kubeapiserver_available:sli_count_6h > 64) and on (name, namespace, _id, resource_id, cluster) (1 - (hostedClusterAPI_kubeapiserver_available:sli_sum_3d / hostedClusterAPI_kubeapiserver_available:sli_count_3d) > (1 * (1 - 0.9995)) and on (name, namespace, _id, resource_id, cluster) hostedClusterAPI_kubeapiserver_available:sli_count_3d > 130)'
+        for: 'PT3H'
+        severity: severityCeiling > 0 ? max(3, severityCeiling) : 3
+      }
+    ]
+    scopes: [
+      azureMonitoring
+    ]
+  }
+}

dev-infrastructure/templates/monitoring.bicep

@@ -206,6 +206,15 @@ module rpAlerts '../modules/metrics/rp-rules.bicep' = {
   }
 }
 
+module rpHcpAlerts '../modules/metrics/rp-hcp-rules.bicep' = {
+  name: 'rpHcpAlerts'
+  params: {
+    azureMonitoringWorkspaceId: hcpAzureMonitoringWorkspaceId
+    actionGroups: rpActionGroups
+    severityCeiling: alertSeverityCeiling
+  }
+}
+
 module msftAlerts '../modules/metrics/msft-rules.bicep' = {
   name: 'msftAlerts'
   params: {

observability/alerts-rp-hcps.yaml

@@ -0,0 +1,7 @@
+prometheusRules:
+  rulesFolders:
+  - alerts/UserJourneyKubeApiserverAvailabilityMonitor-prometheusRule-KSM.yaml
+  untestedRules: []
+  testDependencies:
+  - recording-rules-hcps.yaml
+  outputBicep: ../dev-infrastructure/modules/metrics/rules/generatedRPHCPPrometheusAlertingRules.bicep