diff --git a/dev-infrastructure/modules/metrics/kusto-hcp-rules.bicep b/dev-infrastructure/modules/metrics/kusto-hcp-rules.bicep new file mode 100644 index 00000000000..1f4cda0bbb0 --- /dev/null +++ b/dev-infrastructure/modules/metrics/kusto-hcp-rules.bicep @@ -0,0 +1,19 @@ +// Kusto-only alerts that apply to hosted control planes. + +@description('The Azure resource ID of the Azure Monitor Workspace (stores prometheus metrics for HCPs)') +param azureMonitoringWorkspaceId string + +@description('Action group resource IDs to notify when alerts fire') +param actionGroups array + +@description('The minimum IcM severity level (highest priority) that alerts can fire at. Alerts more critical than this ceiling will be degraded to this value. 0 means no ceiling.') +param severityCeiling int = 0 + +module generatedAlerts 'rules/generatedKustoOnlyHCPPrometheusAlertingRules.bicep' = { + name: 'generatedKustoOnlyHCPPrometheusAlertingRules' + params: { + azureMonitoring: azureMonitoringWorkspaceId + actionGroups: actionGroups + severityCeiling: severityCeiling + } +} diff --git a/dev-infrastructure/modules/metrics/svc-kube-rules.bicep b/dev-infrastructure/modules/metrics/kusto-service-rules.bicep similarity index 59% rename from dev-infrastructure/modules/metrics/svc-kube-rules.bicep rename to dev-infrastructure/modules/metrics/kusto-service-rules.bicep index cd9dd3294fa..f4e5f00d01d 100644 --- a/dev-infrastructure/modules/metrics/svc-kube-rules.bicep +++ b/dev-infrastructure/modules/metrics/kusto-service-rules.bicep @@ -1,17 +1,16 @@ -// Kubernetes control plane alerts scoped to the svc workspace. -// These are the same upstream Kubernetes alerts also deployed to the hcp workspace, -// duplicated here so that gather-observability can detect them on the svc workspace. +// Kusto-only alerts that apply to service clusters and services. @description('The Azure resource ID of the Azure Monitor Workspace (stores prometheus metrics for services/aks level metrics)') param azureMonitoringWorkspaceId string +@description('Action group resource IDs to notify when alerts fire') param actionGroups array @description('The minimum IcM severity level (highest priority) that alerts can fire at. Alerts more critical than this ceiling will be degraded to this value. 0 means no ceiling.') param severityCeiling int = 0 -module generatedAlerts 'rules/generatedSvcKubePrometheusAlertingRules.bicep' = { - name: 'generatedSvcKubePrometheusAlertingRules' +module generatedAlerts 'rules/generatedKustoOnlyServicePrometheusAlertingRules.bicep' = { + name: 'generatedKustoOnlyServicePrometheusAlertingRules' params: { azureMonitoring: azureMonitoringWorkspaceId actionGroups: actionGroups diff --git a/dev-infrastructure/modules/metrics/rules/generatedKustoOnlyHCPPrometheusAlertingRules.bicep b/dev-infrastructure/modules/metrics/rules/generatedKustoOnlyHCPPrometheusAlertingRules.bicep new file mode 100644 index 00000000000..74c11f5f18c --- /dev/null +++ b/dev-infrastructure/modules/metrics/rules/generatedKustoOnlyHCPPrometheusAlertingRules.bicep @@ -0,0 +1,11 @@ +#disable-next-line no-unused-params +param azureMonitoring string + +#disable-next-line no-unused-params +param actionGroups array + +@description('The minimum IcM severity level (highest priority) that alerts can fire at. Alerts more critical than this ceiling will be degraded to this value. 0 means no ceiling.') +param severityCeiling int = 0 + +#disable-next-line no-unused-params +param location string = resourceGroup().location diff --git a/dev-infrastructure/modules/metrics/rules/generatedSvcKubePrometheusAlertingRules.bicep b/dev-infrastructure/modules/metrics/rules/generatedKustoOnlyServicePrometheusAlertingRules.bicep similarity index 100% rename from dev-infrastructure/modules/metrics/rules/generatedSvcKubePrometheusAlertingRules.bicep rename to dev-infrastructure/modules/metrics/rules/generatedKustoOnlyServicePrometheusAlertingRules.bicep diff --git a/dev-infrastructure/templates/monitoring.bicep b/dev-infrastructure/templates/monitoring.bicep index 1c7a1aa62dd..f6ad497766b 100644 --- a/dev-infrastructure/templates/monitoring.bicep +++ b/dev-infrastructure/templates/monitoring.bicep @@ -152,11 +152,11 @@ module serviceAlerts '../modules/metrics/service-rules.bicep' = { } } -module svcKubeAlerts '../modules/metrics/svc-kube-rules.bicep' = { - name: 'svcKubeAlerts' +module kustoServiceAlerts '../modules/metrics/kusto-service-rules.bicep' = { + name: 'kustoServiceAlerts' params: { azureMonitoringWorkspaceId: azureMonitoringWorkspaceId - actionGroups: slActionGroups + actionGroups: ehActionGroups severityCeiling: alertSeverityCeiling } } @@ -179,6 +179,15 @@ module slHcpAlerts '../modules/metrics/sl-hcp-rules.bicep' = { } } +module kustoHcpAlerts '../modules/metrics/kusto-hcp-rules.bicep' = { + name: 'kustoHcpAlerts' + params: { + azureMonitoringWorkspaceId: hcpAzureMonitoringWorkspaceId + actionGroups: ehActionGroups + severityCeiling: alertSeverityCeiling + } +} + module sreServiceAlerts '../modules/metrics/sre-service-rules.bicep' = { name: 'sreServiceAlerts' params: { diff --git a/observability/alerts-kusto-hcps.yaml b/observability/alerts-kusto-hcps.yaml new file mode 100644 index 00000000000..3f5ac16ffcb --- /dev/null +++ b/observability/alerts-kusto-hcps.yaml @@ -0,0 +1,4 @@ +prometheusRules: + rulesFolders: [] + untestedRules: [] + outputBicep: ../dev-infrastructure/modules/metrics/rules/generatedKustoOnlyHCPPrometheusAlertingRules.bicep diff --git a/observability/alerts-sl-svc-kube.yaml b/observability/alerts-kusto-services.yaml similarity index 94% rename from observability/alerts-sl-svc-kube.yaml rename to observability/alerts-kusto-services.yaml index 665c7db45d7..8bda34b7184 100644 --- a/observability/alerts-sl-svc-kube.yaml +++ b/observability/alerts-kusto-services.yaml @@ -2,7 +2,7 @@ prometheusRules: rulesFolders: [] untestedRules: - alerts/kubernetesControlPlane-prometheusRule.yaml - outputBicep: ../dev-infrastructure/modules/metrics/rules/generatedSvcKubePrometheusAlertingRules.bicep + outputBicep: ../dev-infrastructure/modules/metrics/rules/generatedKustoOnlyServicePrometheusAlertingRules.bicep groupNamePrefix: "svc-" outputReplacements: - from: 'job="kube-scheduler"' diff --git a/tooling/prometheus-rules/Makefile b/tooling/prometheus-rules/Makefile index 6c15c1260c6..d4dd3d0457b 100644 --- a/tooling/prometheus-rules/Makefile +++ b/tooling/prometheus-rules/Makefile @@ -23,7 +23,7 @@ fmt-devinfra: run: alerts recording-rules .PHONY: run -alerts: run-sl-services run-sl-svc-kube run-sl-hcps run-sre-hcps run-sre-services run-rp-services run-msft-services +alerts: run-sl-services run-sl-hcps run-sre-hcps run-sre-services run-rp-services run-msft-services run-kusto-services run-kusto-hcps $(MAKE) fmt-devinfra .PHONY: alerts @@ -35,10 +35,6 @@ run-sl-services: go run ./... --config-file ../../observability/alerts-sl-services.yaml $(EXTRA_FLAGS) .PHONY: run-sl-services -run-sl-svc-kube: - go run ./... --config-file ../../observability/alerts-sl-svc-kube.yaml $(EXTRA_FLAGS) -.PHONY: run-sl-svc-kube - run-sl-hcps: go run ./... --config-file ../../observability/alerts-sl-hcps.yaml $(EXTRA_FLAGS) .PHONY: run-sl-hcps @@ -59,6 +55,14 @@ run-msft-services: go run ./... --config-file ../../observability/alerts-msft-services.yaml $(EXTRA_FLAGS) .PHONY: run-msft-services +run-kusto-services: + go run ./... --config-file ../../observability/alerts-kusto-services.yaml $(EXTRA_FLAGS) +.PHONY: run-kusto-services + +run-kusto-hcps: + go run ./... --config-file ../../observability/alerts-kusto-hcps.yaml $(EXTRA_FLAGS) +.PHONY: run-kusto-hcps + run-recording-rules-services: go run ./... --config-file ../../observability/recording-rules-services.yaml $(EXTRA_FLAGS) .PHONY: run-recording-rules-services @@ -69,12 +73,13 @@ run-recording-rules-hcps: ALERT_CONFIGS = \ ../../observability/alerts-sl-services.yaml \ - ../../observability/alerts-sl-svc-kube.yaml \ ../../observability/alerts-sl-hcps.yaml \ ../../observability/alerts-sre-hcps.yaml \ ../../observability/alerts-sre-services.yaml \ ../../observability/alerts-rp-services.yaml \ - ../../observability/alerts-msft-services.yaml + ../../observability/alerts-msft-services.yaml \ + ../../observability/alerts-kusto-services.yaml \ + ../../observability/alerts-kusto-hcps.yaml correlation-map: go run ./... --correlation-map $(ALERT_CONFIGS) > $(OUTPUT)