feat(observability): add Kusto-only alert lanes

dev-infrastructure/modules/metrics/kusto-hcp-rules.bicep

@@ -0,0 +1,19 @@
+// Kusto-only alerts that apply to hosted control planes.
+
+@description('The Azure resource ID of the Azure Monitor Workspace (stores prometheus metrics for HCPs)')
+param azureMonitoringWorkspaceId string
+
+@description('Action group resource IDs to notify when alerts fire')
+param actionGroups array
+
+@description('The minimum IcM severity level (highest priority) that alerts can fire at. Alerts more critical than this ceiling will be degraded to this value. 0 means no ceiling.')
+param severityCeiling int = 0
+
+module generatedAlerts 'rules/generatedKustoOnlyHCPPrometheusAlertingRules.bicep' = {
+  name: 'generatedKustoOnlyHCPPrometheusAlertingRules'
+  params: {
+    azureMonitoring: azureMonitoringWorkspaceId
+    actionGroups: actionGroups
+    severityCeiling: severityCeiling
+  }
+}

dev-infrastructure/modules/metrics/svc-kube-rules.bicep → dev-infrastructure/modules/metrics/kusto-service-rules.bicep

@@ -1,17 +1,16 @@
-// Kubernetes control plane alerts scoped to the svc workspace.
-// These are the same upstream Kubernetes alerts also deployed to the hcp workspace,
-// duplicated here so that gather-observability can detect them on the svc workspace.
+// Kusto-only alerts that apply to service clusters and services.
 
 @description('The Azure resource ID of the Azure Monitor Workspace (stores prometheus metrics for services/aks level metrics)')
 param azureMonitoringWorkspaceId string
 
+@description('Action group resource IDs to notify when alerts fire')
 param actionGroups array
 
 @description('The minimum IcM severity level (highest priority) that alerts can fire at. Alerts more critical than this ceiling will be degraded to this value. 0 means no ceiling.')
 param severityCeiling int = 0
 
-module generatedAlerts 'rules/generatedSvcKubePrometheusAlertingRules.bicep' = {
-  name: 'generatedSvcKubePrometheusAlertingRules'
+module generatedAlerts 'rules/generatedKustoOnlyServicePrometheusAlertingRules.bicep' = {
+  name: 'generatedKustoOnlyServicePrometheusAlertingRules'
   params: {
     azureMonitoring: azureMonitoringWorkspaceId
     actionGroups: actionGroups

dev-infrastructure/modules/metrics/rules/generatedKustoOnlyHCPPrometheusAlertingRules.bicep

@@ -0,0 +1,11 @@
+#disable-next-line no-unused-params
+param azureMonitoring string
+
+#disable-next-line no-unused-params
+param actionGroups array
+
+@description('The minimum IcM severity level (highest priority) that alerts can fire at. Alerts more critical than this ceiling will be degraded to this value. 0 means no ceiling.')
+param severityCeiling int = 0
+
+#disable-next-line no-unused-params
+param location string = resourceGroup().location

dev-infrastructure/templates/monitoring.bicep

@@ -152,11 +152,11 @@ module serviceAlerts '../modules/metrics/service-rules.bicep' = {
   }
 }
 
-module svcKubeAlerts '../modules/metrics/svc-kube-rules.bicep' = {
-  name: 'svcKubeAlerts'
+module kustoServiceAlerts '../modules/metrics/kusto-service-rules.bicep' = {
+  name: 'kustoServiceAlerts'
   params: {
     azureMonitoringWorkspaceId: azureMonitoringWorkspaceId
-    actionGroups: slActionGroups
+    actionGroups: ehActionGroups
     severityCeiling: alertSeverityCeiling
   }
 }
@@ -179,6 +179,15 @@ module slHcpAlerts '../modules/metrics/sl-hcp-rules.bicep' = {
   }
 }
 
+module kustoHcpAlerts '../modules/metrics/kusto-hcp-rules.bicep' = {
+  name: 'kustoHcpAlerts'
+  params: {
+    azureMonitoringWorkspaceId: hcpAzureMonitoringWorkspaceId
+    actionGroups: ehActionGroups
+    severityCeiling: alertSeverityCeiling
+  }
+}
+
 module sreServiceAlerts '../modules/metrics/sre-service-rules.bicep' = {
   name: 'sreServiceAlerts'
   params: {

observability/alerts-kusto-hcps.yaml

@@ -0,0 +1,4 @@
+prometheusRules:
+  rulesFolders: []
+  untestedRules: []
+  outputBicep: ../dev-infrastructure/modules/metrics/rules/generatedKustoOnlyHCPPrometheusAlertingRules.bicep

observability/alerts-sl-svc-kube.yaml → observability/alerts-kusto-services.yaml

@@ -2,7 +2,7 @@ prometheusRules:
   rulesFolders: []
   untestedRules:
   - alerts/kubernetesControlPlane-prometheusRule.yaml
-  outputBicep: ../dev-infrastructure/modules/metrics/rules/generatedSvcKubePrometheusAlertingRules.bicep
+  outputBicep: ../dev-infrastructure/modules/metrics/rules/generatedKustoOnlyServicePrometheusAlertingRules.bicep
   groupNamePrefix: "svc-"
   outputReplacements:
   - from: 'job="kube-scheduler"'

tooling/prometheus-rules/Makefile

@@ -23,7 +23,7 @@ fmt-devinfra:
 run: alerts recording-rules
 .PHONY: run
 
-alerts: run-sl-services run-sl-svc-kube run-sl-hcps run-sre-hcps run-sre-services run-rp-services run-msft-services
+alerts: run-sl-services run-sl-hcps run-sre-hcps run-sre-services run-rp-services run-msft-services run-kusto-services run-kusto-hcps
 	$(MAKE) fmt-devinfra
 .PHONY: alerts
 
@@ -35,10 +35,6 @@ run-sl-services:
 	go run ./...  --config-file ../../observability/alerts-sl-services.yaml $(EXTRA_FLAGS)
 .PHONY: run-sl-services
 
-run-sl-svc-kube:
-	go run ./...  --config-file ../../observability/alerts-sl-svc-kube.yaml $(EXTRA_FLAGS)
-.PHONY: run-sl-svc-kube
-
 run-sl-hcps:
 	go run ./...  --config-file ../../observability/alerts-sl-hcps.yaml $(EXTRA_FLAGS)
 .PHONY: run-sl-hcps
@@ -59,6 +55,14 @@ run-msft-services:
 	go run ./...  --config-file ../../observability/alerts-msft-services.yaml $(EXTRA_FLAGS)
 .PHONY: run-msft-services
 
+run-kusto-services:
+	go run ./...  --config-file ../../observability/alerts-kusto-services.yaml $(EXTRA_FLAGS)
+.PHONY: run-kusto-services
+
+run-kusto-hcps:
+	go run ./...  --config-file ../../observability/alerts-kusto-hcps.yaml $(EXTRA_FLAGS)
+.PHONY: run-kusto-hcps
+
 run-recording-rules-services:
 	go run ./...  --config-file ../../observability/recording-rules-services.yaml $(EXTRA_FLAGS)
 .PHONY: run-recording-rules-services
@@ -69,12 +73,13 @@ run-recording-rules-hcps:
 
 ALERT_CONFIGS = \
 	../../observability/alerts-sl-services.yaml \
-	../../observability/alerts-sl-svc-kube.yaml \
 	../../observability/alerts-sl-hcps.yaml \
 	../../observability/alerts-sre-hcps.yaml \
 	../../observability/alerts-sre-services.yaml \
 	../../observability/alerts-rp-services.yaml \
-	../../observability/alerts-msft-services.yaml
+	../../observability/alerts-msft-services.yaml \
+	../../observability/alerts-kusto-services.yaml \
+	../../observability/alerts-kusto-hcps.yaml
 
 correlation-map:
 	go run ./... --correlation-map $(ALERT_CONFIGS) > $(OUTPUT)