From ea0f2b2e4fcad8a6958a2e5245d5431f001fea67 Mon Sep 17 00:00:00 2001 From: Brendan Bergen Date: Fri, 10 Jul 2026 08:23:54 -0600 Subject: [PATCH 1/4] feat(observability): restore UJ KAS availability alert files Restore alert, recording rule, and lane config files to their pre-revert state (before PR #5754). Content is verbatim from commit 167ca21dc. Follow-up commits fix path prefixes, add severityCeiling, wire into deployment infrastructure, and address review feedback. Original work: PR #5527 (SudoBrendan) --- .../modules/metrics/rp-hcp-rules.bicep | 14 + observability/alerts-rp-hcps.yaml | 7 + .../HCPkasRecord-prometheusRule-KSM.yaml | 26 ++ ...vailabilityMonitor-prometheusRule-KSM.yaml | 96 ++++++ ...bilityMonitor-prometheusRule-KSM_test.yaml | 293 ++++++++++++++++++ ...AvailabilityRecord-prometheusRule-KSM.yaml | 87 ++++++ ...abilityRecord-prometheusRule-KSM_test.yaml | 23 ++ observability/recording-rules-hcps.yaml | 1 + 8 files changed, 547 insertions(+) create mode 100644 dev-infrastructure/modules/metrics/rp-hcp-rules.bicep create mode 100644 observability/alerts-rp-hcps.yaml create mode 100644 observability/alerts/UJKubeApiserverAvailabilityMonitor-prometheusRule-KSM.yaml create mode 100644 observability/alerts/UJKubeApiserverAvailabilityMonitor-prometheusRule-KSM_test.yaml create mode 100644 observability/alerts/UJKubeApiserverAvailabilityRecord-prometheusRule-KSM.yaml create mode 100644 observability/alerts/UJKubeApiserverAvailabilityRecord-prometheusRule-KSM_test.yaml diff --git a/dev-infrastructure/modules/metrics/rp-hcp-rules.bicep b/dev-infrastructure/modules/metrics/rp-hcp-rules.bicep new file mode 100644 index 00000000000..a2800216a40 --- /dev/null +++ b/dev-infrastructure/modules/metrics/rp-hcp-rules.bicep @@ -0,0 +1,14 @@ +// Alerts for HCP KubeAPIServer availability, routed to the RP ICM queue. + +@description('The Azure resource ID of the Azure Monitor Workspace (stores prometheus metrics for hosted control planes)') +param azureMonitoringWorkspaceId string + +param actionGroups array + +module generatedAlerts 'rules/generatedRPHCPPrometheusAlertingRules.bicep' = { + name: 'generatedRPHCPPrometheusAlertingRules' + params: { + azureMonitoring: azureMonitoringWorkspaceId + actionGroups: actionGroups + } +} diff --git a/observability/alerts-rp-hcps.yaml b/observability/alerts-rp-hcps.yaml new file mode 100644 index 00000000000..dbc4c8d236a --- /dev/null +++ b/observability/alerts-rp-hcps.yaml @@ -0,0 +1,7 @@ +prometheusRules: + rulesFolders: + - ../observability/alerts/UJKubeApiserverAvailabilityMonitor-prometheusRule-KSM.yaml + untestedRules: [] + testDependencies: + - recording-rules-hcps.yaml + outputBicep: ../dev-infrastructure/modules/metrics/rules/generatedRPHCPPrometheusAlertingRules.bicep diff --git a/observability/alerts/HCPkasRecord-prometheusRule-KSM.yaml b/observability/alerts/HCPkasRecord-prometheusRule-KSM.yaml index 9f92f5c1bad..aee197cf443 100644 --- a/observability/alerts/HCPkasRecord-prometheusRule-KSM.yaml +++ b/observability/alerts/HCPkasRecord-prometheusRule-KSM.yaml @@ -141,3 +141,29 @@ spec: ) and on (name, namespace, _id, cluster) count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + - record: hostedClusterAPI_kubeapiserver_available:sli_sum_3d + expr: | + sum by (name, namespace, _id, cluster) ( + sum_over_time( + ( + hostedClusterAPI_kubeapiserver_available{status="True"} + and on (name, namespace, _id, cluster) + max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) + )[3d:30m] + ) + ) + and on (name, namespace, _id, cluster) + count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + - record: hostedClusterAPI_kubeapiserver_available:sli_count_3d + expr: | + sum by (name, namespace, _id, cluster) ( + count_over_time( + ( + max by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + and on (name, namespace, _id, cluster) + max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) + )[3d:30m] + ) + ) + and on (name, namespace, _id, cluster) + count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) diff --git a/observability/alerts/UJKubeApiserverAvailabilityMonitor-prometheusRule-KSM.yaml b/observability/alerts/UJKubeApiserverAvailabilityMonitor-prometheusRule-KSM.yaml new file mode 100644 index 00000000000..022b4876fac --- /dev/null +++ b/observability/alerts/UJKubeApiserverAvailabilityMonitor-prometheusRule-KSM.yaml @@ -0,0 +1,96 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: uj-kubeapiserver-availability-alerts + namespace: prometheus +spec: + groups: + - name: rp-uj-kas-availability-monitor-rules + rules: + - alert: UJKubeApiserverAvailability1h5m + expr: | + ( + 1 - (hostedClusterAPI_kubeapiserver_available:sli_sum_5m / hostedClusterAPI_kubeapiserver_available:sli_count_5m) + > (14.4 * (1 - 0.9995)) + and on (name, namespace, _id, cluster) + hostedClusterAPI_kubeapiserver_available:sli_count_5m > 3 + ) + and on (name, namespace, _id, cluster) + ( + 1 - (hostedClusterAPI_kubeapiserver_available:sli_sum_1h / hostedClusterAPI_kubeapiserver_available:sli_count_1h) + > (14.4 * (1 - 0.9995)) + and on (name, namespace, _id, cluster) + hostedClusterAPI_kubeapiserver_available:sli_count_1h > 54 + ) + for: 10m + labels: + long_window: 1h + short_window: 5m + severity: critical + slo: kas-availability + annotations: + summary: "[HCP] [{{ $labels.name }}] KubeAPIServer Fast Error Budget Burn (1h/5m)" + description: | + CID: {{ $labels._id }} + Management Cluster: {{ $labels.cluster }}/{{ $labels.namespace }} + runbook_url: https://aka.ms/arohcp-runbook-ujkasavailable + correlationId: "UJKubeApiserverAvailability/{{ $labels.cluster }}/{{ $labels._id }}" + - alert: UJKubeApiserverAvailability6h30m + # sli_sum_6h and sli_count_6h are recording rules defined in HCPkasRecord-prometheusRule-KSM.yaml + expr: | + ( + 1 - (hostedClusterAPI_kubeapiserver_available:sli_sum_30m / hostedClusterAPI_kubeapiserver_available:sli_count_30m) + > (6 * (1 - 0.9995)) + and on (name, namespace, _id, cluster) + hostedClusterAPI_kubeapiserver_available:sli_count_30m > 27 + ) + and on (name, namespace, _id, cluster) + ( + 1 - (hostedClusterAPI_kubeapiserver_available:sli_sum_6h / hostedClusterAPI_kubeapiserver_available:sli_count_6h) + > (6 * (1 - 0.9995)) + and on (name, namespace, _id, cluster) + hostedClusterAPI_kubeapiserver_available:sli_count_6h > 64 + ) + for: 30m + labels: + long_window: 6h + short_window: 30m + severity: critical + slo: kas-availability + annotations: + summary: "[HCP] [{{ $labels.name }}] KubeAPIServer Medium Error Budget Burn (6h/30m)" + description: | + CID: {{ $labels._id }} + Management Cluster: {{ $labels.cluster }}/{{ $labels.namespace }} + runbook_url: https://aka.ms/arohcp-runbook-ujkasavailable + correlationId: "UJKubeApiserverAvailability/{{ $labels.cluster }}/{{ $labels._id }}" + - alert: UJKubeApiserverAvailability3d6h + # sli_sum_3d, sli_count_3d (long window) and sli_sum_6h, sli_count_6h (short window) + # are recording rules defined in HCPkasRecord-prometheusRule-KSM.yaml + expr: | + ( + 1 - (hostedClusterAPI_kubeapiserver_available:sli_sum_6h / hostedClusterAPI_kubeapiserver_available:sli_count_6h) + > (1 * (1 - 0.9995)) + and on (name, namespace, _id, cluster) + hostedClusterAPI_kubeapiserver_available:sli_count_6h > 64 + ) + and on (name, namespace, _id, cluster) + ( + 1 - (hostedClusterAPI_kubeapiserver_available:sli_sum_3d / hostedClusterAPI_kubeapiserver_available:sli_count_3d) + > (1 * (1 - 0.9995)) + and on (name, namespace, _id, cluster) + hostedClusterAPI_kubeapiserver_available:sli_count_3d > 130 + ) + for: 3h + labels: + long_window: 3d + short_window: 6h + severity: warning + slo: kas-availability + annotations: + summary: "[HCP] [{{ $labels.name }}] KubeAPIServer Slow Error Budget Burn (3d/6h)" + description: | + CID: {{ $labels._id }} + Management Cluster: {{ $labels.cluster }}/{{ $labels.namespace }} + runbook_url: https://aka.ms/arohcp-runbook-ujkasavailable + correlationId: "UJKubeApiserverAvailability/{{ $labels.cluster }}/{{ $labels._id }}" diff --git a/observability/alerts/UJKubeApiserverAvailabilityMonitor-prometheusRule-KSM_test.yaml b/observability/alerts/UJKubeApiserverAvailabilityMonitor-prometheusRule-KSM_test.yaml new file mode 100644 index 00000000000..74e60b5bcb7 --- /dev/null +++ b/observability/alerts/UJKubeApiserverAvailabilityMonitor-prometheusRule-KSM_test.yaml @@ -0,0 +1,293 @@ +rule_files: +- HCPkasRecord-prometheusRule-KSM.yaml +- UJKubeApiserverAvailabilityRecord-prometheusRule-KSM.yaml +- UJKubeApiserverAvailabilityMonitor-prometheusRule-KSM.yaml +group_eval_order: +- hcp-kms-recording-rules +- uj-kubeapiserver-availability-recording-rules +- rp-uj-kas-availability-monitor-rules +evaluation_interval: 1m +tests: +# Test 1: Normal operation - KubeAPIServer always available, no alerts should fire +- interval: 1m + name: "Test 1: Normal operation - perfect availability, no alerts" + input_series: + - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster", namespace="clusters", _id="abc-123", cluster="mgmt-cluster"}' + values: "1+0x600" # 10 hours of perfect availability + alert_rule_test: + - eval_time: 600m + alertname: UJKubeApiserverAvailability1h5m + exp_alerts: [] + - eval_time: 600m + alertname: UJKubeApiserverAvailability6h30m + exp_alerts: [] + - eval_time: 600m + alertname: UJKubeApiserverAvailability3d6h + exp_alerts: [] # sli_count_3d uses [3d:30m]; at 600m only ~20 steps pass offset gate — far below 130 guard +# Test 2: Fast burn - complete failure triggers UJKubeApiserverAvailability1h5m +# SLI rules require the series to have existed 15m ago (offset blackout). The binding +# constraint is the sli_count_1h > 54 guard: with [1h:1m] subquery and data from t=0, +# the count of gate-passing steps is T-14, which first exceeds 54 at T=69m (55 steps). +# With for:10m, the fast-burn alert first fires at T=79m. +- interval: 30s + name: "Test 2: Fast burn - complete failure triggers UJKubeApiserverAvailability1h5m" + input_series: + - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster", namespace="clusters", _id="abc-123", cluster="mgmt-cluster"}' + values: "0+0x200" # 100 minutes of complete failure (200 * 30s) + alert_rule_test: + - eval_time: 10m + alertname: UJKubeApiserverAvailability1h5m + exp_alerts: [] # Offset blackout: series has not existed for 15m yet + - eval_time: 10m + alertname: UJKubeApiserverAvailability6h30m + exp_alerts: [] # Offset blackout: series has not existed for 15m yet + - eval_time: 78m + alertname: UJKubeApiserverAvailability1h5m + exp_alerts: [] # for: timer = 9m, not yet 10m — alert not yet firing + - eval_time: 80m # T=79m: for: timer reaches 10m; T=80m: still firing + alertname: UJKubeApiserverAvailability1h5m + exp_alerts: + - exp_labels: + severity: critical + slo: kas-availability + long_window: 1h + short_window: 5m + name: test-cluster + namespace: clusters + _id: abc-123 + cluster: mgmt-cluster + exp_annotations: + summary: "[HCP] [test-cluster] KubeAPIServer Fast Error Budget Burn (1h/5m)" + description: "CID: abc-123\nManagement Cluster: mgmt-cluster/clusters\n" + runbook_url: https://aka.ms/arohcp-runbook-ujkasavailable + correlationId: "UJKubeApiserverAvailability/mgmt-cluster/abc-123" +# Test 3: Medium burn - complete failure triggers both tiers after 6h+ +# The fast-burn (1h/5m) alert fires at T=79m (same binding constraint as Test 2). +# The medium-burn (6h/30m) alert requires sli_count_6h > 64 guard: with [6h:5m] +# subquery at 5m step, gate-passing steps first exceed 64 at T=335m (65 steps), +# and with for:30m fires at T=365m. At eval_time=420m (7 hours), both tiers fire. +- interval: 30s + name: "Test 3: Medium burn - complete failure triggers both tiers after 6h+" + input_series: + - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster", namespace="clusters", _id="abc-123", cluster="mgmt-cluster"}' + values: "0+0x900" # 450 minutes = 7.5 hours of complete failure (900 * 30s) + alert_rule_test: + - eval_time: 80m + alertname: UJKubeApiserverAvailability1h5m + exp_alerts: + - exp_labels: + severity: critical + slo: kas-availability + long_window: 1h + short_window: 5m + name: test-cluster + namespace: clusters + _id: abc-123 + cluster: mgmt-cluster + exp_annotations: + summary: "[HCP] [test-cluster] KubeAPIServer Fast Error Budget Burn (1h/5m)" + description: "CID: abc-123\nManagement Cluster: mgmt-cluster/clusters\n" + runbook_url: https://aka.ms/arohcp-runbook-ujkasavailable + correlationId: "UJKubeApiserverAvailability/mgmt-cluster/abc-123" + - eval_time: 364m + alertname: UJKubeApiserverAvailability6h30m + exp_alerts: [] # for: timer = 29m at T=364m, not yet 30m — alert not yet firing + - eval_time: 366m # T=365m: for: timer reaches 30m; T=366m: still firing + alertname: UJKubeApiserverAvailability6h30m + exp_alerts: + - exp_labels: + severity: critical + slo: kas-availability + long_window: 6h + short_window: 30m + name: test-cluster + namespace: clusters + _id: abc-123 + cluster: mgmt-cluster + exp_annotations: + summary: "[HCP] [test-cluster] KubeAPIServer Medium Error Budget Burn (6h/30m)" + description: "CID: abc-123\nManagement Cluster: mgmt-cluster/clusters\n" + runbook_url: https://aka.ms/arohcp-runbook-ujkasavailable + correlationId: "UJKubeApiserverAvailability/mgmt-cluster/abc-123" + - eval_time: 420m # 7 hours — both tiers firing + alertname: UJKubeApiserverAvailability1h5m + exp_alerts: + - exp_labels: + severity: critical + slo: kas-availability + long_window: 1h + short_window: 5m + name: test-cluster + namespace: clusters + _id: abc-123 + cluster: mgmt-cluster + exp_annotations: + summary: "[HCP] [test-cluster] KubeAPIServer Fast Error Budget Burn (1h/5m)" + description: "CID: abc-123\nManagement Cluster: mgmt-cluster/clusters\n" + runbook_url: https://aka.ms/arohcp-runbook-ujkasavailable + correlationId: "UJKubeApiserverAvailability/mgmt-cluster/abc-123" + - eval_time: 420m + alertname: UJKubeApiserverAvailability6h30m + exp_alerts: + - exp_labels: + severity: critical + slo: kas-availability + long_window: 6h + short_window: 30m + name: test-cluster + namespace: clusters + _id: abc-123 + cluster: mgmt-cluster + exp_annotations: + summary: "[HCP] [test-cluster] KubeAPIServer Medium Error Budget Burn (6h/30m)" + description: "CID: abc-123\nManagement Cluster: mgmt-cluster/clusters\n" + runbook_url: https://aka.ms/arohcp-runbook-ujkasavailable + correlationId: "UJKubeApiserverAvailability/mgmt-cluster/abc-123" + - eval_time: 420m + alertname: UJKubeApiserverAvailability3d6h + exp_alerts: [] # sli_count_3d guard: only 14 steps at T=420m, need > 130; 3d alert not triggered by 7.5h failure +# Test 4: Provisioning grace - newly created cluster (< 15m) does NOT trigger alert +# The offset blackout (hostedClusterAPI_kubeapiserver_available offset 15m) requires +# the series to have existed 15 minutes ago. A brand-new cluster has no offset data, +# so the SLI recording rules produce no output and no alert can fire. +- interval: 1m + name: "Test 4: Provisioning grace - new cluster excluded from alerting" + input_series: + - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="new-cluster", namespace="clusters", _id="new-123", cluster="mgmt-cluster"}' + values: "0+0x10" # 10 minutes of complete failure (new cluster) + alert_rule_test: + - eval_time: 10m + alertname: UJKubeApiserverAvailability1h5m + exp_alerts: [] # Offset blackout: no data 15m ago, SLI rules produce no output + - eval_time: 10m + alertname: UJKubeApiserverAvailability6h30m + exp_alerts: [] + - eval_time: 10m + alertname: UJKubeApiserverAvailability3d6h + exp_alerts: [] # sli_count_3d far below 130 guard at 10m +# Test 5: Partial failure below threshold - no alert fires +# One failure at t=21m; at eval_time=400m this falls outside both the 5m (t=395-400m) +# and 1h (t=340-400m) windows — no burn in either window. +- interval: 1m + name: "Test 5: Transient failure outside evaluation windows - no alert fires" + input_series: + - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster", namespace="clusters", _id="abc-123", cluster="mgmt-cluster"}' + values: "1+0x20 0 1+0x580" # One failure in ~603 samples + alert_rule_test: + - eval_time: 400m + alertname: UJKubeApiserverAvailability1h5m + exp_alerts: [] + - eval_time: 400m + alertname: UJKubeApiserverAvailability6h30m + exp_alerts: [] +# Test 6: Sample guard behavior - short burst suppressed by minimum-sample guards +# Failure burst from t=20m to t=41m (22 samples: 0+0x21 = 22 samples). +# At eval_time=35m: +# - sli_count_1h uses [1h:1m] with offset 15m gate; gate passes samples from t>=15m. +# At t=35m, only t=15..35m = 21 samples are visible → below the > 54 guard. +# - Fast-burn alert is suppressed by the sli_count_1h > 54 minimum-sample guard. +# At eval_time=65m: +# - sli_count_6h uses [6h:5m] with offset 15m gate; at t=65m only t=15,20,...,65m +# = 11 samples at 5m step → far below the > 64 guard. +# - Medium-burn alert is suppressed by the sli_count_6h > 64 minimum-sample guard, +# even though the 30m error rate is elevated. +# - This is the intended guard behavior: prevent sparse-window false pages. +# Note: Test 3 covers the scenario where both alerts fire with sufficient data. +- interval: 1m + name: "Test 6: Sample guards suppress sparse-window alerts during short burst" + input_series: + - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster", namespace="clusters", _id="abc-123", cluster="mgmt-cluster"}' + values: "1+0x19 0+0x21 1+0x800" # 20m healthy, 22m failure (t=20..t=41), then recovery + alert_rule_test: + - eval_time: 35m + alertname: UJKubeApiserverAvailability1h5m + exp_alerts: [] # sli_count_1h guard: only ~21 samples visible at t=35m, need > 54 + - eval_time: 35m + alertname: UJKubeApiserverAvailability6h30m + exp_alerts: [] # for:30m not yet satisfied; sli_count_6h guard also blocks (~5 samples at 5m step, need > 64) + - eval_time: 65m + alertname: UJKubeApiserverAvailability1h5m + exp_alerts: [] # Fast tier: 5m window (t=61-65) clean; sli_count_1h guard still blocks (~51 samples) + - eval_time: 65m + alertname: UJKubeApiserverAvailability6h30m + exp_alerts: [] # sli_count_6h guard blocks: only ~11 samples at 5m step, need > 64 +# Test 7: Near-threshold - minimal failure burst validates for:10m boundary +# 80m of success (t=0..79m), then 7 failures (t=80..86m), then success. +# The [5m:1m] subquery at eval time T covers steps (T-5m, T] (exclusive start, 5 steps). +# At T=80m: 5m window (75,80]={76..80m} has failure at t=80m → error_rate_5m=1/5=20% ✓ +# sli_count_1h = 60 steps in (20,80], all > 54 ✓, error_rate_1h=1/60≈1.7%>threshold ✓ +# Condition first becomes true at T=80m (start of for: timer). +# At T=89m: for: timer = 9m → NOT firing. +# At T=90m: for: timer = 10m (80+10=90). 5m window (85,90]={86..90m} has failure at +# t=86m → error_rate_5m=1/5=20% → condition still true → FIRES. +# At T=91m: 5m window (86,91]={87..91m} all success → error_rate_5m=0 → condition +# false → RESOLVES. +- interval: 1m + name: "Test 7: Near-threshold - minimal burst validates for:10m boundary" + input_series: + - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster", namespace="clusters", _id="abc-123", cluster="mgmt-cluster"}' + values: "1+0x79 0+0x6 1+0x100" # 80 successes (t=0..79m), 7 failures (t=80..86m), then success + alert_rule_test: + - eval_time: 89m + alertname: UJKubeApiserverAvailability1h5m + exp_alerts: [] # for: timer = 9m, not yet 10m — alert not yet firing + - eval_time: 90m + alertname: UJKubeApiserverAvailability1h5m + exp_alerts: + - exp_labels: + severity: critical + slo: kas-availability + long_window: 1h + short_window: 5m + name: test-cluster + namespace: clusters + _id: abc-123 + cluster: mgmt-cluster + exp_annotations: + summary: "[HCP] [test-cluster] KubeAPIServer Fast Error Budget Burn (1h/5m)" + description: "CID: abc-123\nManagement Cluster: mgmt-cluster/clusters\n" + runbook_url: https://aka.ms/arohcp-runbook-ujkasavailable + correlationId: "UJKubeApiserverAvailability/mgmt-cluster/abc-123" + - eval_time: 91m + alertname: UJKubeApiserverAvailability1h5m + exp_alerts: [] # 5m window (86,91]={87..91m} all success → error_rate_5m=0 → alert resolves +# Test 8: Slow burn - sustained complete failure triggers UJKubeApiserverAvailability3d6h +# The slow-burn (3d/6h) alert uses sli_count_3d with [3d:30m] subquery (30m step). +# With interval:1m and data from t=0, the offset gate (offset 15m) requires data at t-15m. +# At subquery step t=30m: needs data at 15m (exists with 1m data density) → passes. +# At evaluation T: valid steps in [3d:30m] = T/30 (steps at 30m, 60m, ..., T). +# sli_count_3d > 130: first satisfied when T/30 > 130 → T > 3900 → T=3930m (131 steps). +# sli_count_6h > 64: satisfied at T=335m (steps at 5m from 15m to T; floor((T-15)/5)+1 > 64). +# for:3h timer starts at T=3930m, fires at T=3930m+180m=4110m. +# Boundary: eval_time=4080m → for:timer=150m → NOT firing. +# eval_time=4109m → for:timer=179m → NOT firing (1-minute before threshold). +# eval_time=4110m → for:timer=180m=3h → FIRES. +- interval: 1m + name: "Test 8: Slow burn - sustained failure triggers UJKubeApiserverAvailability3d6h" + input_series: + - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster", namespace="clusters", _id="abc-123", cluster="mgmt-cluster"}' + values: "0+0x4110" # 4111 samples at 1m interval, t=0 to t=4110m, complete failure + alert_rule_test: + - eval_time: 4080m + alertname: UJKubeApiserverAvailability3d6h + exp_alerts: [] # for: timer = 150m at T=4080m, not yet 3h — alert not yet firing + - eval_time: 4109m + alertname: UJKubeApiserverAvailability3d6h + exp_alerts: [] # for: timer = 179m at T=4109m, not yet 3h — alert not yet firing + - eval_time: 4110m + alertname: UJKubeApiserverAvailability3d6h + exp_alerts: + - exp_labels: + severity: warning + slo: kas-availability + long_window: 3d + short_window: 6h + name: test-cluster + namespace: clusters + _id: abc-123 + cluster: mgmt-cluster + exp_annotations: + summary: "[HCP] [test-cluster] KubeAPIServer Slow Error Budget Burn (3d/6h)" + description: "CID: abc-123\nManagement Cluster: mgmt-cluster/clusters\n" + runbook_url: https://aka.ms/arohcp-runbook-ujkasavailable + correlationId: "UJKubeApiserverAvailability/mgmt-cluster/abc-123" diff --git a/observability/alerts/UJKubeApiserverAvailabilityRecord-prometheusRule-KSM.yaml b/observability/alerts/UJKubeApiserverAvailabilityRecord-prometheusRule-KSM.yaml new file mode 100644 index 00000000000..83d9c212939 --- /dev/null +++ b/observability/alerts/UJKubeApiserverAvailabilityRecord-prometheusRule-KSM.yaml @@ -0,0 +1,87 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: uj-kubeapiserver-availability-recording-rules + namespace: prometheus +spec: + groups: + - name: uj-kubeapiserver-availability-recording-rules + rules: + - record: hostedClusterAPI_kubeapiserver_available:sli_sum_5m + expr: | + sum by (name, namespace, _id, cluster) ( + sum_over_time( + ( + hostedClusterAPI_kubeapiserver_available{status="True"} + and on (name, namespace, _id, cluster) + max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) + )[5m:1m] + ) + ) + and on (name, namespace, _id, cluster) + count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + - record: hostedClusterAPI_kubeapiserver_available:sli_count_5m + expr: | + sum by (name, namespace, _id, cluster) ( + count_over_time( + ( + max by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + and on (name, namespace, _id, cluster) + max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) + )[5m:1m] + ) + ) + and on (name, namespace, _id, cluster) + count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + - record: hostedClusterAPI_kubeapiserver_available:sli_sum_1h + expr: | + sum by (name, namespace, _id, cluster) ( + sum_over_time( + ( + hostedClusterAPI_kubeapiserver_available{status="True"} + and on (name, namespace, _id, cluster) + max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) + )[1h:1m] + ) + ) + and on (name, namespace, _id, cluster) + count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + - record: hostedClusterAPI_kubeapiserver_available:sli_count_1h + expr: | + sum by (name, namespace, _id, cluster) ( + count_over_time( + ( + max by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + and on (name, namespace, _id, cluster) + max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) + )[1h:1m] + ) + ) + and on (name, namespace, _id, cluster) + count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + - record: hostedClusterAPI_kubeapiserver_available:sli_sum_30m + expr: | + sum by (name, namespace, _id, cluster) ( + sum_over_time( + ( + hostedClusterAPI_kubeapiserver_available{status="True"} + and on (name, namespace, _id, cluster) + max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) + )[30m:1m] + ) + ) + and on (name, namespace, _id, cluster) + count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + - record: hostedClusterAPI_kubeapiserver_available:sli_count_30m + expr: | + sum by (name, namespace, _id, cluster) ( + count_over_time( + ( + max by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + and on (name, namespace, _id, cluster) + max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) + )[30m:1m] + ) + ) + and on (name, namespace, _id, cluster) + count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) diff --git a/observability/alerts/UJKubeApiserverAvailabilityRecord-prometheusRule-KSM_test.yaml b/observability/alerts/UJKubeApiserverAvailabilityRecord-prometheusRule-KSM_test.yaml new file mode 100644 index 00000000000..94feb9639ae --- /dev/null +++ b/observability/alerts/UJKubeApiserverAvailabilityRecord-prometheusRule-KSM_test.yaml @@ -0,0 +1,23 @@ +rule_files: +- HCPkasRecord-prometheusRule-KSM.yaml +- UJKubeApiserverAvailabilityRecord-prometheusRule-KSM.yaml +evaluation_interval: 1m +tests: +# Recording rules are exercised end-to-end by UJKubeApiserverAvailabilityMonitor-prometheusRule-KSM_test.yaml +# (6 alert scenarios covering all 6 SLI rules). This file satisfies the generator's +# test-file requirement with a basic smoke test. +- interval: 1m + input_series: + - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster", namespace="clusters", _id="abc-123", cluster="mgmt-cluster"}' + values: "1+0x30" + promql_expr_test: + - eval_time: 20m + expr: hostedClusterAPI_kubeapiserver_available:sli_sum_5m{name="test-cluster"} + exp_samples: + - labels: 'hostedClusterAPI_kubeapiserver_available:sli_sum_5m{_id="abc-123", cluster="mgmt-cluster", name="test-cluster", namespace="clusters"}' + value: 5 + - eval_time: 20m + expr: hostedClusterAPI_kubeapiserver_available:sli_count_5m{name="test-cluster"} + exp_samples: + - labels: 'hostedClusterAPI_kubeapiserver_available:sli_count_5m{_id="abc-123", cluster="mgmt-cluster", name="test-cluster", namespace="clusters"}' + value: 5 diff --git a/observability/recording-rules-hcps.yaml b/observability/recording-rules-hcps.yaml index 9a86a9ff4a4..7879ae97d49 100644 --- a/observability/recording-rules-hcps.yaml +++ b/observability/recording-rules-hcps.yaml @@ -3,5 +3,6 @@ prometheusRules: - alerts/HCPkasRecord-prometheusRule-KSM.yaml - alerts/HCPkasRecord-prometheusRule-apiserver-requests.yaml - alerts/HCPkasRecord-prometheusRule-latency.yaml + - alerts/UJKubeApiserverAvailabilityRecord-prometheusRule-KSM.yaml untestedRules: [] outputBicep: ../dev-infrastructure/modules/metrics/rules/generatedHCPRecordingRules.bicep From fb9e844aba270e6efb8b4574db75c16c75272cb3 Mon Sep 17 00:00:00 2001 From: Brendan Bergen Date: Mon, 13 Jul 2026 16:16:37 -0600 Subject: [PATCH 2/4] refactor(observability): rename UJ to UserJourney in KAS availability alert files Just to keep the github UI clean for review, we rename premptively. Co-Authored-By: Claude Opus 4.6 --- ...urneyKubeApiserverAvailabilityMonitor-prometheusRule-KSM.yaml} | 0 ...KubeApiserverAvailabilityMonitor-prometheusRule-KSM_test.yaml} | 0 ...ourneyKubeApiserverAvailabilityRecord-prometheusRule-KSM.yaml} | 0 ...yKubeApiserverAvailabilityRecord-prometheusRule-KSM_test.yaml} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename observability/alerts/{UJKubeApiserverAvailabilityMonitor-prometheusRule-KSM.yaml => UserJourneyKubeApiserverAvailabilityMonitor-prometheusRule-KSM.yaml} (100%) rename observability/alerts/{UJKubeApiserverAvailabilityMonitor-prometheusRule-KSM_test.yaml => UserJourneyKubeApiserverAvailabilityMonitor-prometheusRule-KSM_test.yaml} (100%) rename observability/alerts/{UJKubeApiserverAvailabilityRecord-prometheusRule-KSM.yaml => UserJourneyKubeApiserverAvailabilityRecord-prometheusRule-KSM.yaml} (100%) rename observability/alerts/{UJKubeApiserverAvailabilityRecord-prometheusRule-KSM_test.yaml => UserJourneyKubeApiserverAvailabilityRecord-prometheusRule-KSM_test.yaml} (100%) diff --git a/observability/alerts/UJKubeApiserverAvailabilityMonitor-prometheusRule-KSM.yaml b/observability/alerts/UserJourneyKubeApiserverAvailabilityMonitor-prometheusRule-KSM.yaml similarity index 100% rename from observability/alerts/UJKubeApiserverAvailabilityMonitor-prometheusRule-KSM.yaml rename to observability/alerts/UserJourneyKubeApiserverAvailabilityMonitor-prometheusRule-KSM.yaml diff --git a/observability/alerts/UJKubeApiserverAvailabilityMonitor-prometheusRule-KSM_test.yaml b/observability/alerts/UserJourneyKubeApiserverAvailabilityMonitor-prometheusRule-KSM_test.yaml similarity index 100% rename from observability/alerts/UJKubeApiserverAvailabilityMonitor-prometheusRule-KSM_test.yaml rename to observability/alerts/UserJourneyKubeApiserverAvailabilityMonitor-prometheusRule-KSM_test.yaml diff --git a/observability/alerts/UJKubeApiserverAvailabilityRecord-prometheusRule-KSM.yaml b/observability/alerts/UserJourneyKubeApiserverAvailabilityRecord-prometheusRule-KSM.yaml similarity index 100% rename from observability/alerts/UJKubeApiserverAvailabilityRecord-prometheusRule-KSM.yaml rename to observability/alerts/UserJourneyKubeApiserverAvailabilityRecord-prometheusRule-KSM.yaml diff --git a/observability/alerts/UJKubeApiserverAvailabilityRecord-prometheusRule-KSM_test.yaml b/observability/alerts/UserJourneyKubeApiserverAvailabilityRecord-prometheusRule-KSM_test.yaml similarity index 100% rename from observability/alerts/UJKubeApiserverAvailabilityRecord-prometheusRule-KSM_test.yaml rename to observability/alerts/UserJourneyKubeApiserverAvailabilityRecord-prometheusRule-KSM_test.yaml From ef3d6548217c8bd76631110d189c5f6ab0d1df08 Mon Sep 17 00:00:00 2001 From: Brendan Bergen Date: Fri, 10 Jul 2026 15:31:08 -0600 Subject: [PATCH 3/4] fix(observability): update KAS availability alerts for review feedback - Rename UJ prefix to userJourney (camelCase) per ADR naming convention - Downgrade all alert severities to warning for initial rollout, use CEN - Add severityCeiling parameter to rp-hcp-rules.bicep - Wire rpHcpAlerts module into monitoring.bicep - Add run-rp-hcps build target to prometheus-rules Makefile - Fix alerts-rp-hcps.yaml path prefix to match main conventions - Rename CID to OCM Cluster ID in descriptions - Separate Management Cluster and Namespace into distinct lines - Add Environment and Region to descriptions (external labels) - Change summary format to [HCPKASAvailableBurn] with namespace - Add resource_id label to recording rule aggregation chains - Update all test expectations to match Co-Authored-By: Simon Wiencki Co-Authored-By: Claude Opus 4.6 --- .../modules/metrics/rp-hcp-rules.bicep | 4 + dev-infrastructure/templates/monitoring.bicep | 9 ++ observability/alerts-rp-hcps.yaml | 2 +- .../HCPkasRecord-prometheusRule-KSM.yaml | 150 ++++++++--------- .../HCPkasRecord-prometheusRule-KSM_test.yaml | 48 +++--- ...vailabilityMonitor-prometheusRule-KSM.yaml | 67 ++++---- ...bilityMonitor-prometheusRule-KSM_test.yaml | 151 +++++++++--------- ...AvailabilityRecord-prometheusRule-KSM.yaml | 70 ++++---- ...abilityRecord-prometheusRule-KSM_test.yaml | 10 +- observability/recording-rules-hcps.yaml | 2 +- tooling/prometheus-rules/Makefile | 7 +- 11 files changed, 276 insertions(+), 244 deletions(-) diff --git a/dev-infrastructure/modules/metrics/rp-hcp-rules.bicep b/dev-infrastructure/modules/metrics/rp-hcp-rules.bicep index a2800216a40..53353deb5db 100644 --- a/dev-infrastructure/modules/metrics/rp-hcp-rules.bicep +++ b/dev-infrastructure/modules/metrics/rp-hcp-rules.bicep @@ -5,10 +5,14 @@ param azureMonitoringWorkspaceId string param actionGroups array +@description('The minimum IcM severity level (highest priority) that alerts can fire at. Alerts more critical than this ceiling will be degraded to this value. 0 means no ceiling.') +param severityCeiling int = 0 + module generatedAlerts 'rules/generatedRPHCPPrometheusAlertingRules.bicep' = { name: 'generatedRPHCPPrometheusAlertingRules' params: { azureMonitoring: azureMonitoringWorkspaceId actionGroups: actionGroups + severityCeiling: severityCeiling } } diff --git a/dev-infrastructure/templates/monitoring.bicep b/dev-infrastructure/templates/monitoring.bicep index f6ad497766b..a7e5e98a882 100644 --- a/dev-infrastructure/templates/monitoring.bicep +++ b/dev-infrastructure/templates/monitoring.bicep @@ -206,6 +206,15 @@ module rpAlerts '../modules/metrics/rp-rules.bicep' = { } } +module rpHcpAlerts '../modules/metrics/rp-hcp-rules.bicep' = { + name: 'rpHcpAlerts' + params: { + azureMonitoringWorkspaceId: hcpAzureMonitoringWorkspaceId + actionGroups: rpActionGroups + severityCeiling: alertSeverityCeiling + } +} + module msftAlerts '../modules/metrics/msft-rules.bicep' = { name: 'msftAlerts' params: { diff --git a/observability/alerts-rp-hcps.yaml b/observability/alerts-rp-hcps.yaml index dbc4c8d236a..2bef020471e 100644 --- a/observability/alerts-rp-hcps.yaml +++ b/observability/alerts-rp-hcps.yaml @@ -1,6 +1,6 @@ prometheusRules: rulesFolders: - - ../observability/alerts/UJKubeApiserverAvailabilityMonitor-prometheusRule-KSM.yaml + - alerts/UserJourneyKubeApiserverAvailabilityMonitor-prometheusRule-KSM.yaml untestedRules: [] testDependencies: - recording-rules-hcps.yaml diff --git a/observability/alerts/HCPkasRecord-prometheusRule-KSM.yaml b/observability/alerts/HCPkasRecord-prometheusRule-KSM.yaml index aee197cf443..f74e2e3894b 100644 --- a/observability/alerts/HCPkasRecord-prometheusRule-KSM.yaml +++ b/observability/alerts/HCPkasRecord-prometheusRule-KSM.yaml @@ -13,77 +13,77 @@ spec: # Long-window averages for slow-burn alerts - record: hostedClusterAPI_kubeapiserver_available:ratio_avg_30d expr: | - avg by (name, namespace, _id, cluster) (avg_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[30d])) - and on (name, namespace, _id, cluster) - count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + avg by (name, namespace, _id, resource_id, cluster) (avg_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[30d])) + and on (name, namespace, _id, resource_id, cluster) + count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) - record: hostedClusterAPI_kubeapiserver_available:ratio_avg_7d expr: | - avg by (name, namespace, _id, cluster) (avg_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[7d])) - and on (name, namespace, _id, cluster) - count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + avg by (name, namespace, _id, resource_id, cluster) (avg_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[7d])) + and on (name, namespace, _id, resource_id, cluster) + count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) - record: hostedClusterAPI_kubeapiserver_available:ratio_avg_1d expr: | - avg by (name, namespace, _id, cluster) (avg_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[1d])) - and on (name, namespace, _id, cluster) - count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + avg by (name, namespace, _id, resource_id, cluster) (avg_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[1d])) + and on (name, namespace, _id, resource_id, cluster) + count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) - record: hostedClusterAPI_kubeapiserver_available:ratio_avg_3d expr: | - avg by (name, namespace, _id, cluster) (avg_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[3d])) - and on (name, namespace, _id, cluster) - count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + avg by (name, namespace, _id, resource_id, cluster) (avg_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[3d])) + and on (name, namespace, _id, resource_id, cluster) + count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) # Short-window sum_over_time for burn-rate calculations - record: hostedClusterAPI_kubeapiserver_available:sum_over_time_30m expr: | - sum by (name, namespace, _id, cluster) (sum_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[30m])) - and on (name, namespace, _id, cluster) - count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + sum by (name, namespace, _id, resource_id, cluster) (sum_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[30m])) + and on (name, namespace, _id, resource_id, cluster) + count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) - record: hostedClusterAPI_kubeapiserver_available:sum_over_time_1h expr: | - sum by (name, namespace, _id, cluster) (sum_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[1h])) - and on (name, namespace, _id, cluster) - count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + sum by (name, namespace, _id, resource_id, cluster) (sum_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[1h])) + and on (name, namespace, _id, resource_id, cluster) + count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) - record: hostedClusterAPI_kubeapiserver_available:sum_over_time_2h expr: | - sum by (name, namespace, _id, cluster) (sum_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[2h])) - and on (name, namespace, _id, cluster) - count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + sum by (name, namespace, _id, resource_id, cluster) (sum_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[2h])) + and on (name, namespace, _id, resource_id, cluster) + count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) - record: hostedClusterAPI_kubeapiserver_available:sum_over_time_6h expr: | - sum by (name, namespace, _id, cluster) (sum_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[6h])) - and on (name, namespace, _id, cluster) - count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + sum by (name, namespace, _id, resource_id, cluster) (sum_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[6h])) + and on (name, namespace, _id, resource_id, cluster) + count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) # Short-window count_over_time for burn-rate calculations - record: hostedClusterAPI_kubeapiserver_available:count_over_time_30m expr: | - sum by (name, namespace, _id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[30m])) - and on (name, namespace, _id, cluster) - count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + sum by (name, namespace, _id, resource_id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[30m])) + and on (name, namespace, _id, resource_id, cluster) + count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) - record: hostedClusterAPI_kubeapiserver_available:count_over_time_1h expr: | - sum by (name, namespace, _id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[1h])) - and on (name, namespace, _id, cluster) - count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + sum by (name, namespace, _id, resource_id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[1h])) + and on (name, namespace, _id, resource_id, cluster) + count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) - record: hostedClusterAPI_kubeapiserver_available:count_over_time_2h expr: | - sum by (name, namespace, _id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[2h])) - and on (name, namespace, _id, cluster) - count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + sum by (name, namespace, _id, resource_id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[2h])) + and on (name, namespace, _id, resource_id, cluster) + count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) - record: hostedClusterAPI_kubeapiserver_available:count_over_time_6h expr: | - sum by (name, namespace, _id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[6h])) - and on (name, namespace, _id, cluster) - count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + sum by (name, namespace, _id, resource_id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[6h])) + and on (name, namespace, _id, resource_id, cluster) + count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) # Long-window count_over_time for sample-count guards - record: hostedClusterAPI_kubeapiserver_available:count_over_time_1d expr: | - sum by (name, namespace, _id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[1d])) - and on (name, namespace, _id, cluster) - count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + sum by (name, namespace, _id, resource_id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[1d])) + and on (name, namespace, _id, resource_id, cluster) + count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) - record: hostedClusterAPI_kubeapiserver_available:count_over_time_3d expr: | - sum by (name, namespace, _id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[3d])) - and on (name, namespace, _id, cluster) - count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + sum by (name, namespace, _id, resource_id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[3d])) + and on (name, namespace, _id, resource_id, cluster) + count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) # SLI rules using 15m offset blackout to exclude newly-provisioned clusters. # The offset blackout ensures a cluster existed at least 15 minutes ago before # contributing to SLI calculations, preventing provisioning noise from skewing @@ -91,79 +91,79 @@ spec: # all samples (any status via max by) so downtime is counted as 0, not skipped. - record: hostedClusterAPI_kubeapiserver_available:sli_sum_15m expr: | - sum by (name, namespace, _id, cluster) ( + sum by (name, namespace, _id, resource_id, cluster) ( sum_over_time( ( hostedClusterAPI_kubeapiserver_available{status="True"} - and on (name, namespace, _id, cluster) - max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) + and on (name, namespace, _id, resource_id, cluster) + max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) )[15m:1m] ) ) - and on (name, namespace, _id, cluster) - count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + and on (name, namespace, _id, resource_id, cluster) + count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) - record: hostedClusterAPI_kubeapiserver_available:sli_count_15m expr: | - sum by (name, namespace, _id, cluster) ( + sum by (name, namespace, _id, resource_id, cluster) ( count_over_time( ( - max by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) - and on (name, namespace, _id, cluster) - max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) + max by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) + and on (name, namespace, _id, resource_id, cluster) + max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) )[15m:1m] ) ) - and on (name, namespace, _id, cluster) - count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + and on (name, namespace, _id, resource_id, cluster) + count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) - record: hostedClusterAPI_kubeapiserver_available:sli_sum_6h expr: | - sum by (name, namespace, _id, cluster) ( + sum by (name, namespace, _id, resource_id, cluster) ( sum_over_time( ( hostedClusterAPI_kubeapiserver_available{status="True"} - and on (name, namespace, _id, cluster) - max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) + and on (name, namespace, _id, resource_id, cluster) + max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) )[6h:5m] ) ) - and on (name, namespace, _id, cluster) - count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + and on (name, namespace, _id, resource_id, cluster) + count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) - record: hostedClusterAPI_kubeapiserver_available:sli_count_6h expr: | - sum by (name, namespace, _id, cluster) ( + sum by (name, namespace, _id, resource_id, cluster) ( count_over_time( ( - max by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) - and on (name, namespace, _id, cluster) - max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) + max by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) + and on (name, namespace, _id, resource_id, cluster) + max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) )[6h:5m] ) ) - and on (name, namespace, _id, cluster) - count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + and on (name, namespace, _id, resource_id, cluster) + count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) - record: hostedClusterAPI_kubeapiserver_available:sli_sum_3d expr: | - sum by (name, namespace, _id, cluster) ( + sum by (name, namespace, _id, resource_id, cluster) ( sum_over_time( ( hostedClusterAPI_kubeapiserver_available{status="True"} - and on (name, namespace, _id, cluster) - max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) + and on (name, namespace, _id, resource_id, cluster) + max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) )[3d:30m] ) ) - and on (name, namespace, _id, cluster) - count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + and on (name, namespace, _id, resource_id, cluster) + count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) - record: hostedClusterAPI_kubeapiserver_available:sli_count_3d expr: | - sum by (name, namespace, _id, cluster) ( + sum by (name, namespace, _id, resource_id, cluster) ( count_over_time( ( - max by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) - and on (name, namespace, _id, cluster) - max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) + max by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) + and on (name, namespace, _id, resource_id, cluster) + max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) )[3d:30m] ) ) - and on (name, namespace, _id, cluster) - count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + and on (name, namespace, _id, resource_id, cluster) + count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) diff --git a/observability/alerts/HCPkasRecord-prometheusRule-KSM_test.yaml b/observability/alerts/HCPkasRecord-prometheusRule-KSM_test.yaml index fbf9be60da7..0c704b58955 100644 --- a/observability/alerts/HCPkasRecord-prometheusRule-KSM_test.yaml +++ b/observability/alerts/HCPkasRecord-prometheusRule-KSM_test.yaml @@ -5,7 +5,7 @@ tests: # Test 1: 100% Availability (Perfect Success) - interval: 1m input_series: - - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster-1", namespace="clusters", _id="abc-123", cluster="mgmt-cluster"}' + - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster-1", namespace="clusters", _id="abc-123", cluster="mgmt-cluster", resource_id="test-resource-1"}' # Generate 10 data points with a value of 1, over 10 minutes. values: "1+0x9" promql_expr_test: @@ -13,91 +13,91 @@ tests: - expr: hostedClusterAPI_kubeapiserver_available:ratio_avg_30d{name="test-cluster-1", namespace="clusters"} eval_time: 10m exp_samples: - - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:ratio_avg_30d", name="test-cluster-1", namespace="clusters", _id="abc-123", cluster="mgmt-cluster"}' + - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:ratio_avg_30d", name="test-cluster-1", namespace="clusters", _id="abc-123", cluster="mgmt-cluster", resource_id="test-resource-1"}' value: 1 # Test the 7d rule - expr: hostedClusterAPI_kubeapiserver_available:ratio_avg_7d{name="test-cluster-1", namespace="clusters"} eval_time: 10m exp_samples: - - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:ratio_avg_7d", name="test-cluster-1", namespace="clusters", _id="abc-123", cluster="mgmt-cluster"}' + - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:ratio_avg_7d", name="test-cluster-1", namespace="clusters", _id="abc-123", cluster="mgmt-cluster", resource_id="test-resource-1"}' value: 1 # Test the 1d rule - expr: hostedClusterAPI_kubeapiserver_available:ratio_avg_1d{name="test-cluster-1", namespace="clusters"} eval_time: 10m exp_samples: - - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:ratio_avg_1d", name="test-cluster-1", namespace="clusters", _id="abc-123", cluster="mgmt-cluster"}' + - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:ratio_avg_1d", name="test-cluster-1", namespace="clusters", _id="abc-123", cluster="mgmt-cluster", resource_id="test-resource-1"}' value: 1 # Test the 3d rule - expr: hostedClusterAPI_kubeapiserver_available:ratio_avg_3d{name="test-cluster-1", namespace="clusters"} eval_time: 10m exp_samples: - - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:ratio_avg_3d", name="test-cluster-1", namespace="clusters", _id="abc-123", cluster="mgmt-cluster"}' + - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:ratio_avg_3d", name="test-cluster-1", namespace="clusters", _id="abc-123", cluster="mgmt-cluster", resource_id="test-resource-1"}' value: 1 # Test 2: 50% Availability (Intermittent Failures) - interval: 1m input_series: - - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster-2", namespace="clusters", _id="def-456", cluster="mgmt-cluster"}' + - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster-2", namespace="clusters", _id="def-456", cluster="mgmt-cluster", resource_id="test-resource-2"}' values: "1 0 1 0 1 0 1 0 1 0" promql_expr_test: # Test the 30d rule - expr: hostedClusterAPI_kubeapiserver_available:ratio_avg_30d{name="test-cluster-2", namespace="clusters"} eval_time: 10m exp_samples: - - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:ratio_avg_30d", name="test-cluster-2", namespace="clusters", _id="def-456", cluster="mgmt-cluster"}' + - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:ratio_avg_30d", name="test-cluster-2", namespace="clusters", _id="def-456", cluster="mgmt-cluster", resource_id="test-resource-2"}' value: 0.5 # Test the 7d rule - expr: hostedClusterAPI_kubeapiserver_available:ratio_avg_7d{name="test-cluster-2", namespace="clusters"} eval_time: 10m exp_samples: - - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:ratio_avg_7d", name="test-cluster-2", namespace="clusters", _id="def-456", cluster="mgmt-cluster"}' + - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:ratio_avg_7d", name="test-cluster-2", namespace="clusters", _id="def-456", cluster="mgmt-cluster", resource_id="test-resource-2"}' value: 0.5 # Test the 1d rule - expr: hostedClusterAPI_kubeapiserver_available:ratio_avg_1d{name="test-cluster-2", namespace="clusters"} eval_time: 10m exp_samples: - - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:ratio_avg_1d", name="test-cluster-2", namespace="clusters", _id="def-456", cluster="mgmt-cluster"}' + - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:ratio_avg_1d", name="test-cluster-2", namespace="clusters", _id="def-456", cluster="mgmt-cluster", resource_id="test-resource-2"}' value: 0.5 # Test the 3d rule - expr: hostedClusterAPI_kubeapiserver_available:ratio_avg_3d{name="test-cluster-2", namespace="clusters"} eval_time: 10m exp_samples: - - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:ratio_avg_3d", name="test-cluster-2", namespace="clusters", _id="def-456", cluster="mgmt-cluster"}' + - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:ratio_avg_3d", name="test-cluster-2", namespace="clusters", _id="def-456", cluster="mgmt-cluster", resource_id="test-resource-2"}' value: 0.5 # Test 3: 0% Availability (Complete Failure) - interval: 1m input_series: - - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster-3", namespace="clusters", _id="ghi-789", cluster="mgmt-cluster"}' + - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster-3", namespace="clusters", _id="ghi-789", cluster="mgmt-cluster", resource_id="test-resource-3"}' values: "0+0x9" promql_expr_test: # Test the 30d rule - expr: hostedClusterAPI_kubeapiserver_available:ratio_avg_30d{name="test-cluster-3", namespace="clusters"} eval_time: 10m exp_samples: - - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:ratio_avg_30d", name="test-cluster-3", namespace="clusters", _id="ghi-789", cluster="mgmt-cluster"}' + - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:ratio_avg_30d", name="test-cluster-3", namespace="clusters", _id="ghi-789", cluster="mgmt-cluster", resource_id="test-resource-3"}' value: 0 # Test the 7d rule - expr: hostedClusterAPI_kubeapiserver_available:ratio_avg_7d{name="test-cluster-3", namespace="clusters"} eval_time: 10m exp_samples: - - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:ratio_avg_7d", name="test-cluster-3", namespace="clusters", _id="ghi-789", cluster="mgmt-cluster"}' + - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:ratio_avg_7d", name="test-cluster-3", namespace="clusters", _id="ghi-789", cluster="mgmt-cluster", resource_id="test-resource-3"}' value: 0 # Test the 1d rule - expr: hostedClusterAPI_kubeapiserver_available:ratio_avg_1d{name="test-cluster-3", namespace="clusters"} eval_time: 10m exp_samples: - - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:ratio_avg_1d", name="test-cluster-3", namespace="clusters", _id="ghi-789", cluster="mgmt-cluster"}' + - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:ratio_avg_1d", name="test-cluster-3", namespace="clusters", _id="ghi-789", cluster="mgmt-cluster", resource_id="test-resource-3"}' value: 0 # Test the 3d rule - expr: hostedClusterAPI_kubeapiserver_available:ratio_avg_3d{name="test-cluster-3", namespace="clusters"} eval_time: 10m exp_samples: - - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:ratio_avg_3d", name="test-cluster-3", namespace="clusters", _id="ghi-789", cluster="mgmt-cluster"}' + - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:ratio_avg_3d", name="test-cluster-3", namespace="clusters", _id="ghi-789", cluster="mgmt-cluster", resource_id="test-resource-3"}' value: 0 # Test 4: Burn-rate recording rules (sum_over_time and count_over_time) - interval: 1m name: "Test 4: Burn-rate recording rules - all successes" input_series: - - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster-4", namespace="clusters", _id="xyz-789", cluster="mgmt-cluster"}' + - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster-4", namespace="clusters", _id="xyz-789", cluster="mgmt-cluster", resource_id="test-resource-4"}' # Perfect availability for 2+ hours values: "1+0x150" promql_expr_test: @@ -105,25 +105,25 @@ tests: - expr: hostedClusterAPI_kubeapiserver_available:sum_over_time_30m{name="test-cluster-4"} eval_time: 90m exp_samples: - - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:sum_over_time_30m", name="test-cluster-4", namespace="clusters", _id="xyz-789", cluster="mgmt-cluster"}' + - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:sum_over_time_30m", name="test-cluster-4", namespace="clusters", _id="xyz-789", cluster="mgmt-cluster", resource_id="test-resource-4"}' value: 30 # Test count_over_time_30m - expr: hostedClusterAPI_kubeapiserver_available:count_over_time_30m{name="test-cluster-4"} eval_time: 90m exp_samples: - - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:count_over_time_30m", name="test-cluster-4", namespace="clusters", _id="xyz-789", cluster="mgmt-cluster"}' + - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:count_over_time_30m", name="test-cluster-4", namespace="clusters", _id="xyz-789", cluster="mgmt-cluster", resource_id="test-resource-4"}' value: 30 # Test sum_over_time_1h (1h window is left-open: (30m, 90m] = 60 samples) - expr: hostedClusterAPI_kubeapiserver_available:sum_over_time_1h{name="test-cluster-4"} eval_time: 90m exp_samples: - - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:sum_over_time_1h", name="test-cluster-4", namespace="clusters", _id="xyz-789", cluster="mgmt-cluster"}' + - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:sum_over_time_1h", name="test-cluster-4", namespace="clusters", _id="xyz-789", cluster="mgmt-cluster", resource_id="test-resource-4"}' value: 60 # Test count_over_time_1h - expr: hostedClusterAPI_kubeapiserver_available:count_over_time_1h{name="test-cluster-4"} eval_time: 90m exp_samples: - - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:count_over_time_1h", name="test-cluster-4", namespace="clusters", _id="xyz-789", cluster="mgmt-cluster"}' + - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:count_over_time_1h", name="test-cluster-4", namespace="clusters", _id="xyz-789", cluster="mgmt-cluster", resource_id="test-resource-4"}' value: 60 # Test 5: SLI recording rules - perfect availability # The sli_* rules use a 15m offset blackout gate and [15m:1m] subquery. @@ -132,18 +132,18 @@ tests: - interval: 1m name: "Test 5: SLI recording rules - perfect availability" input_series: - - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster-5", namespace="clusters", _id="sli-100", cluster="mgmt-cluster"}' + - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster-5", namespace="clusters", _id="sli-100", cluster="mgmt-cluster", resource_id="test-resource-5"}' values: "1+0x60" promql_expr_test: - expr: hostedClusterAPI_kubeapiserver_available:sli_sum_15m{name="test-cluster-5"} eval_time: 35m exp_samples: - - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:sli_sum_15m", name="test-cluster-5", namespace="clusters", _id="sli-100", cluster="mgmt-cluster"}' + - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:sli_sum_15m", name="test-cluster-5", namespace="clusters", _id="sli-100", cluster="mgmt-cluster", resource_id="test-resource-5"}' value: 15 - expr: hostedClusterAPI_kubeapiserver_available:sli_count_15m{name="test-cluster-5"} eval_time: 35m exp_samples: - - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:sli_count_15m", name="test-cluster-5", namespace="clusters", _id="sli-100", cluster="mgmt-cluster"}' + - labels: '{__name__="hostedClusterAPI_kubeapiserver_available:sli_count_15m", name="test-cluster-5", namespace="clusters", _id="sli-100", cluster="mgmt-cluster", resource_id="test-resource-5"}' value: 15 # Test 6: SLI offset blackout - newly provisioned cluster excluded # A cluster with only 10m of data should produce no SLI values because @@ -151,7 +151,7 @@ tests: - interval: 1m name: "Test 6: SLI offset blackout - new cluster excluded" input_series: - - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster-6", namespace="clusters", _id="sli-new", cluster="mgmt-cluster"}' + - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster-6", namespace="clusters", _id="sli-new", cluster="mgmt-cluster", resource_id="test-resource-6"}' values: "1+0x10" promql_expr_test: - expr: hostedClusterAPI_kubeapiserver_available:sli_sum_15m{name="test-cluster-6"} diff --git a/observability/alerts/UserJourneyKubeApiserverAvailabilityMonitor-prometheusRule-KSM.yaml b/observability/alerts/UserJourneyKubeApiserverAvailabilityMonitor-prometheusRule-KSM.yaml index 022b4876fac..87c2785c2df 100644 --- a/observability/alerts/UserJourneyKubeApiserverAvailabilityMonitor-prometheusRule-KSM.yaml +++ b/observability/alerts/UserJourneyKubeApiserverAvailabilityMonitor-prometheusRule-KSM.yaml @@ -1,96 +1,103 @@ apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: - name: uj-kubeapiserver-availability-alerts + name: userjourney-kubeapiserver-availability-alerts namespace: prometheus spec: groups: - - name: rp-uj-kas-availability-monitor-rules + - name: rp-userjourney-kas-availability-monitor-rules rules: - - alert: UJKubeApiserverAvailability1h5m + - alert: userJourneyKubeApiserverAvailability1h5m expr: | ( 1 - (hostedClusterAPI_kubeapiserver_available:sli_sum_5m / hostedClusterAPI_kubeapiserver_available:sli_count_5m) > (14.4 * (1 - 0.9995)) - and on (name, namespace, _id, cluster) + and on (name, namespace, _id, resource_id, cluster) hostedClusterAPI_kubeapiserver_available:sli_count_5m > 3 ) - and on (name, namespace, _id, cluster) + and on (name, namespace, _id, resource_id, cluster) ( 1 - (hostedClusterAPI_kubeapiserver_available:sli_sum_1h / hostedClusterAPI_kubeapiserver_available:sli_count_1h) > (14.4 * (1 - 0.9995)) - and on (name, namespace, _id, cluster) + and on (name, namespace, _id, resource_id, cluster) hostedClusterAPI_kubeapiserver_available:sli_count_1h > 54 ) for: 10m labels: long_window: 1h short_window: 5m - severity: critical + severity: "3" slo: kas-availability annotations: - summary: "[HCP] [{{ $labels.name }}] KubeAPIServer Fast Error Budget Burn (1h/5m)" + summary: "[HCPKASAvailableBurn] {{ $labels.cluster }} / {{ $labels.namespace }} (1h/5m)" description: | - CID: {{ $labels._id }} - Management Cluster: {{ $labels.cluster }}/{{ $labels.namespace }} + Resource ID: {{ $labels.resource_id }} + OCM Cluster ID: {{ $labels._id }} + Management Cluster: {{ $labels.cluster }} + Namespace: {{ $labels.namespace }} runbook_url: https://aka.ms/arohcp-runbook-ujkasavailable - correlationId: "UJKubeApiserverAvailability/{{ $labels.cluster }}/{{ $labels._id }}" - - alert: UJKubeApiserverAvailability6h30m - # sli_sum_6h and sli_count_6h are recording rules defined in HCPkasRecord-prometheusRule-KSM.yaml + correlationId: "userJourneyKubeApiserverAvailability/{{ $labels.cluster }}/{{ $labels._id }}" + - alert: userJourneyKubeApiserverAvailability6h30m + # sli_sum_30m, sli_count_30m are in userJourneyKubeApiserverAvailabilityRecord-prometheusRule-KSM.yaml + # sli_sum_6h, sli_count_6h are in HCPkasRecord-prometheusRule-KSM.yaml expr: | ( 1 - (hostedClusterAPI_kubeapiserver_available:sli_sum_30m / hostedClusterAPI_kubeapiserver_available:sli_count_30m) > (6 * (1 - 0.9995)) - and on (name, namespace, _id, cluster) + and on (name, namespace, _id, resource_id, cluster) hostedClusterAPI_kubeapiserver_available:sli_count_30m > 27 ) - and on (name, namespace, _id, cluster) + and on (name, namespace, _id, resource_id, cluster) ( 1 - (hostedClusterAPI_kubeapiserver_available:sli_sum_6h / hostedClusterAPI_kubeapiserver_available:sli_count_6h) > (6 * (1 - 0.9995)) - and on (name, namespace, _id, cluster) + and on (name, namespace, _id, resource_id, cluster) hostedClusterAPI_kubeapiserver_available:sli_count_6h > 64 ) for: 30m labels: long_window: 6h short_window: 30m - severity: critical + severity: "3" slo: kas-availability annotations: - summary: "[HCP] [{{ $labels.name }}] KubeAPIServer Medium Error Budget Burn (6h/30m)" + summary: "[HCPKASAvailableBurn] {{ $labels.cluster }} / {{ $labels.namespace }} (6h/30m)" description: | - CID: {{ $labels._id }} - Management Cluster: {{ $labels.cluster }}/{{ $labels.namespace }} + Resource ID: {{ $labels.resource_id }} + OCM Cluster ID: {{ $labels._id }} + Management Cluster: {{ $labels.cluster }} + Namespace: {{ $labels.namespace }} runbook_url: https://aka.ms/arohcp-runbook-ujkasavailable - correlationId: "UJKubeApiserverAvailability/{{ $labels.cluster }}/{{ $labels._id }}" - - alert: UJKubeApiserverAvailability3d6h + correlationId: "userJourneyKubeApiserverAvailability/{{ $labels.cluster }}/{{ $labels._id }}" + - alert: userJourneyKubeApiserverAvailability3d6h # sli_sum_3d, sli_count_3d (long window) and sli_sum_6h, sli_count_6h (short window) # are recording rules defined in HCPkasRecord-prometheusRule-KSM.yaml expr: | ( 1 - (hostedClusterAPI_kubeapiserver_available:sli_sum_6h / hostedClusterAPI_kubeapiserver_available:sli_count_6h) > (1 * (1 - 0.9995)) - and on (name, namespace, _id, cluster) + and on (name, namespace, _id, resource_id, cluster) hostedClusterAPI_kubeapiserver_available:sli_count_6h > 64 ) - and on (name, namespace, _id, cluster) + and on (name, namespace, _id, resource_id, cluster) ( 1 - (hostedClusterAPI_kubeapiserver_available:sli_sum_3d / hostedClusterAPI_kubeapiserver_available:sli_count_3d) > (1 * (1 - 0.9995)) - and on (name, namespace, _id, cluster) + and on (name, namespace, _id, resource_id, cluster) hostedClusterAPI_kubeapiserver_available:sli_count_3d > 130 ) for: 3h labels: long_window: 3d short_window: 6h - severity: warning + severity: "3" slo: kas-availability annotations: - summary: "[HCP] [{{ $labels.name }}] KubeAPIServer Slow Error Budget Burn (3d/6h)" + summary: "[HCPKASAvailableBurn] {{ $labels.cluster }} / {{ $labels.namespace }} (3d/6h)" description: | - CID: {{ $labels._id }} - Management Cluster: {{ $labels.cluster }}/{{ $labels.namespace }} + Resource ID: {{ $labels.resource_id }} + OCM Cluster ID: {{ $labels._id }} + Management Cluster: {{ $labels.cluster }} + Namespace: {{ $labels.namespace }} runbook_url: https://aka.ms/arohcp-runbook-ujkasavailable - correlationId: "UJKubeApiserverAvailability/{{ $labels.cluster }}/{{ $labels._id }}" + correlationId: "userJourneyKubeApiserverAvailability/{{ $labels.cluster }}/{{ $labels._id }}" diff --git a/observability/alerts/UserJourneyKubeApiserverAvailabilityMonitor-prometheusRule-KSM_test.yaml b/observability/alerts/UserJourneyKubeApiserverAvailabilityMonitor-prometheusRule-KSM_test.yaml index 74e60b5bcb7..98d7f931cbf 100644 --- a/observability/alerts/UserJourneyKubeApiserverAvailabilityMonitor-prometheusRule-KSM_test.yaml +++ b/observability/alerts/UserJourneyKubeApiserverAvailabilityMonitor-prometheusRule-KSM_test.yaml @@ -1,54 +1,54 @@ rule_files: - HCPkasRecord-prometheusRule-KSM.yaml -- UJKubeApiserverAvailabilityRecord-prometheusRule-KSM.yaml -- UJKubeApiserverAvailabilityMonitor-prometheusRule-KSM.yaml +- UserJourneyKubeApiserverAvailabilityRecord-prometheusRule-KSM.yaml +- UserJourneyKubeApiserverAvailabilityMonitor-prometheusRule-KSM.yaml group_eval_order: - hcp-kms-recording-rules -- uj-kubeapiserver-availability-recording-rules -- rp-uj-kas-availability-monitor-rules +- userjourney-kubeapiserver-availability-recording-rules +- rp-userjourney-kas-availability-monitor-rules evaluation_interval: 1m tests: # Test 1: Normal operation - KubeAPIServer always available, no alerts should fire - interval: 1m name: "Test 1: Normal operation - perfect availability, no alerts" input_series: - - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster", namespace="clusters", _id="abc-123", cluster="mgmt-cluster"}' + - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster", namespace="clusters", _id="abc-123", cluster="mgmt-cluster", resource_id="/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster"}' values: "1+0x600" # 10 hours of perfect availability alert_rule_test: - eval_time: 600m - alertname: UJKubeApiserverAvailability1h5m + alertname: userJourneyKubeApiserverAvailability1h5m exp_alerts: [] - eval_time: 600m - alertname: UJKubeApiserverAvailability6h30m + alertname: userJourneyKubeApiserverAvailability6h30m exp_alerts: [] - eval_time: 600m - alertname: UJKubeApiserverAvailability3d6h + alertname: userJourneyKubeApiserverAvailability3d6h exp_alerts: [] # sli_count_3d uses [3d:30m]; at 600m only ~20 steps pass offset gate — far below 130 guard -# Test 2: Fast burn - complete failure triggers UJKubeApiserverAvailability1h5m +# Test 2: Fast burn - complete failure triggers userJourneyKubeApiserverAvailability1h5m # SLI rules require the series to have existed 15m ago (offset blackout). The binding # constraint is the sli_count_1h > 54 guard: with [1h:1m] subquery and data from t=0, # the count of gate-passing steps is T-14, which first exceeds 54 at T=69m (55 steps). # With for:10m, the fast-burn alert first fires at T=79m. - interval: 30s - name: "Test 2: Fast burn - complete failure triggers UJKubeApiserverAvailability1h5m" + name: "Test 2: Fast burn - complete failure triggers userJourneyKubeApiserverAvailability1h5m" input_series: - - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster", namespace="clusters", _id="abc-123", cluster="mgmt-cluster"}' + - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster", namespace="clusters", _id="abc-123", cluster="mgmt-cluster", resource_id="/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster"}' values: "0+0x200" # 100 minutes of complete failure (200 * 30s) alert_rule_test: - eval_time: 10m - alertname: UJKubeApiserverAvailability1h5m + alertname: userJourneyKubeApiserverAvailability1h5m exp_alerts: [] # Offset blackout: series has not existed for 15m yet - eval_time: 10m - alertname: UJKubeApiserverAvailability6h30m + alertname: userJourneyKubeApiserverAvailability6h30m exp_alerts: [] # Offset blackout: series has not existed for 15m yet - eval_time: 78m - alertname: UJKubeApiserverAvailability1h5m + alertname: userJourneyKubeApiserverAvailability1h5m exp_alerts: [] # for: timer = 9m, not yet 10m — alert not yet firing - eval_time: 80m # T=79m: for: timer reaches 10m; T=80m: still firing - alertname: UJKubeApiserverAvailability1h5m + alertname: userJourneyKubeApiserverAvailability1h5m exp_alerts: - exp_labels: - severity: critical + severity: "3" slo: kas-availability long_window: 1h short_window: 5m @@ -56,11 +56,12 @@ tests: namespace: clusters _id: abc-123 cluster: mgmt-cluster + resource_id: /subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster exp_annotations: - summary: "[HCP] [test-cluster] KubeAPIServer Fast Error Budget Burn (1h/5m)" - description: "CID: abc-123\nManagement Cluster: mgmt-cluster/clusters\n" + summary: "[HCPKASAvailableBurn] mgmt-cluster / clusters (1h/5m)" + description: "Resource ID: /subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster\nOCM Cluster ID: abc-123\nManagement Cluster: mgmt-cluster\nNamespace: clusters\n" runbook_url: https://aka.ms/arohcp-runbook-ujkasavailable - correlationId: "UJKubeApiserverAvailability/mgmt-cluster/abc-123" + correlationId: "userJourneyKubeApiserverAvailability/mgmt-cluster/abc-123" # Test 3: Medium burn - complete failure triggers both tiers after 6h+ # The fast-burn (1h/5m) alert fires at T=79m (same binding constraint as Test 2). # The medium-burn (6h/30m) alert requires sli_count_6h > 64 guard: with [6h:5m] @@ -69,14 +70,14 @@ tests: - interval: 30s name: "Test 3: Medium burn - complete failure triggers both tiers after 6h+" input_series: - - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster", namespace="clusters", _id="abc-123", cluster="mgmt-cluster"}' + - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster", namespace="clusters", _id="abc-123", cluster="mgmt-cluster", resource_id="/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster"}' values: "0+0x900" # 450 minutes = 7.5 hours of complete failure (900 * 30s) alert_rule_test: - eval_time: 80m - alertname: UJKubeApiserverAvailability1h5m + alertname: userJourneyKubeApiserverAvailability1h5m exp_alerts: - exp_labels: - severity: critical + severity: "3" slo: kas-availability long_window: 1h short_window: 5m @@ -84,19 +85,20 @@ tests: namespace: clusters _id: abc-123 cluster: mgmt-cluster + resource_id: /subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster exp_annotations: - summary: "[HCP] [test-cluster] KubeAPIServer Fast Error Budget Burn (1h/5m)" - description: "CID: abc-123\nManagement Cluster: mgmt-cluster/clusters\n" + summary: "[HCPKASAvailableBurn] mgmt-cluster / clusters (1h/5m)" + description: "Resource ID: /subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster\nOCM Cluster ID: abc-123\nManagement Cluster: mgmt-cluster\nNamespace: clusters\n" runbook_url: https://aka.ms/arohcp-runbook-ujkasavailable - correlationId: "UJKubeApiserverAvailability/mgmt-cluster/abc-123" + correlationId: "userJourneyKubeApiserverAvailability/mgmt-cluster/abc-123" - eval_time: 364m - alertname: UJKubeApiserverAvailability6h30m + alertname: userJourneyKubeApiserverAvailability6h30m exp_alerts: [] # for: timer = 29m at T=364m, not yet 30m — alert not yet firing - eval_time: 366m # T=365m: for: timer reaches 30m; T=366m: still firing - alertname: UJKubeApiserverAvailability6h30m + alertname: userJourneyKubeApiserverAvailability6h30m exp_alerts: - exp_labels: - severity: critical + severity: "3" slo: kas-availability long_window: 6h short_window: 30m @@ -104,16 +106,17 @@ tests: namespace: clusters _id: abc-123 cluster: mgmt-cluster + resource_id: /subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster exp_annotations: - summary: "[HCP] [test-cluster] KubeAPIServer Medium Error Budget Burn (6h/30m)" - description: "CID: abc-123\nManagement Cluster: mgmt-cluster/clusters\n" + summary: "[HCPKASAvailableBurn] mgmt-cluster / clusters (6h/30m)" + description: "Resource ID: /subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster\nOCM Cluster ID: abc-123\nManagement Cluster: mgmt-cluster\nNamespace: clusters\n" runbook_url: https://aka.ms/arohcp-runbook-ujkasavailable - correlationId: "UJKubeApiserverAvailability/mgmt-cluster/abc-123" + correlationId: "userJourneyKubeApiserverAvailability/mgmt-cluster/abc-123" - eval_time: 420m # 7 hours — both tiers firing - alertname: UJKubeApiserverAvailability1h5m + alertname: userJourneyKubeApiserverAvailability1h5m exp_alerts: - exp_labels: - severity: critical + severity: "3" slo: kas-availability long_window: 1h short_window: 5m @@ -121,16 +124,17 @@ tests: namespace: clusters _id: abc-123 cluster: mgmt-cluster + resource_id: /subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster exp_annotations: - summary: "[HCP] [test-cluster] KubeAPIServer Fast Error Budget Burn (1h/5m)" - description: "CID: abc-123\nManagement Cluster: mgmt-cluster/clusters\n" + summary: "[HCPKASAvailableBurn] mgmt-cluster / clusters (1h/5m)" + description: "Resource ID: /subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster\nOCM Cluster ID: abc-123\nManagement Cluster: mgmt-cluster\nNamespace: clusters\n" runbook_url: https://aka.ms/arohcp-runbook-ujkasavailable - correlationId: "UJKubeApiserverAvailability/mgmt-cluster/abc-123" + correlationId: "userJourneyKubeApiserverAvailability/mgmt-cluster/abc-123" - eval_time: 420m - alertname: UJKubeApiserverAvailability6h30m + alertname: userJourneyKubeApiserverAvailability6h30m exp_alerts: - exp_labels: - severity: critical + severity: "3" slo: kas-availability long_window: 6h short_window: 30m @@ -138,13 +142,14 @@ tests: namespace: clusters _id: abc-123 cluster: mgmt-cluster + resource_id: /subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster exp_annotations: - summary: "[HCP] [test-cluster] KubeAPIServer Medium Error Budget Burn (6h/30m)" - description: "CID: abc-123\nManagement Cluster: mgmt-cluster/clusters\n" + summary: "[HCPKASAvailableBurn] mgmt-cluster / clusters (6h/30m)" + description: "Resource ID: /subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster\nOCM Cluster ID: abc-123\nManagement Cluster: mgmt-cluster\nNamespace: clusters\n" runbook_url: https://aka.ms/arohcp-runbook-ujkasavailable - correlationId: "UJKubeApiserverAvailability/mgmt-cluster/abc-123" + correlationId: "userJourneyKubeApiserverAvailability/mgmt-cluster/abc-123" - eval_time: 420m - alertname: UJKubeApiserverAvailability3d6h + alertname: userJourneyKubeApiserverAvailability3d6h exp_alerts: [] # sli_count_3d guard: only 14 steps at T=420m, need > 130; 3d alert not triggered by 7.5h failure # Test 4: Provisioning grace - newly created cluster (< 15m) does NOT trigger alert # The offset blackout (hostedClusterAPI_kubeapiserver_available offset 15m) requires @@ -153,17 +158,17 @@ tests: - interval: 1m name: "Test 4: Provisioning grace - new cluster excluded from alerting" input_series: - - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="new-cluster", namespace="clusters", _id="new-123", cluster="mgmt-cluster"}' + - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="new-cluster", namespace="clusters", _id="new-123", cluster="mgmt-cluster", resource_id="/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/new-cluster"}' values: "0+0x10" # 10 minutes of complete failure (new cluster) alert_rule_test: - eval_time: 10m - alertname: UJKubeApiserverAvailability1h5m + alertname: userJourneyKubeApiserverAvailability1h5m exp_alerts: [] # Offset blackout: no data 15m ago, SLI rules produce no output - eval_time: 10m - alertname: UJKubeApiserverAvailability6h30m + alertname: userJourneyKubeApiserverAvailability6h30m exp_alerts: [] - eval_time: 10m - alertname: UJKubeApiserverAvailability3d6h + alertname: userJourneyKubeApiserverAvailability3d6h exp_alerts: [] # sli_count_3d far below 130 guard at 10m # Test 5: Partial failure below threshold - no alert fires # One failure at t=21m; at eval_time=400m this falls outside both the 5m (t=395-400m) @@ -171,14 +176,14 @@ tests: - interval: 1m name: "Test 5: Transient failure outside evaluation windows - no alert fires" input_series: - - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster", namespace="clusters", _id="abc-123", cluster="mgmt-cluster"}' + - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster", namespace="clusters", _id="abc-123", cluster="mgmt-cluster", resource_id="/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster"}' values: "1+0x20 0 1+0x580" # One failure in ~603 samples alert_rule_test: - eval_time: 400m - alertname: UJKubeApiserverAvailability1h5m + alertname: userJourneyKubeApiserverAvailability1h5m exp_alerts: [] - eval_time: 400m - alertname: UJKubeApiserverAvailability6h30m + alertname: userJourneyKubeApiserverAvailability6h30m exp_alerts: [] # Test 6: Sample guard behavior - short burst suppressed by minimum-sample guards # Failure burst from t=20m to t=41m (22 samples: 0+0x21 = 22 samples). @@ -196,20 +201,20 @@ tests: - interval: 1m name: "Test 6: Sample guards suppress sparse-window alerts during short burst" input_series: - - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster", namespace="clusters", _id="abc-123", cluster="mgmt-cluster"}' + - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster", namespace="clusters", _id="abc-123", cluster="mgmt-cluster", resource_id="/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster"}' values: "1+0x19 0+0x21 1+0x800" # 20m healthy, 22m failure (t=20..t=41), then recovery alert_rule_test: - eval_time: 35m - alertname: UJKubeApiserverAvailability1h5m + alertname: userJourneyKubeApiserverAvailability1h5m exp_alerts: [] # sli_count_1h guard: only ~21 samples visible at t=35m, need > 54 - eval_time: 35m - alertname: UJKubeApiserverAvailability6h30m + alertname: userJourneyKubeApiserverAvailability6h30m exp_alerts: [] # for:30m not yet satisfied; sli_count_6h guard also blocks (~5 samples at 5m step, need > 64) - eval_time: 65m - alertname: UJKubeApiserverAvailability1h5m + alertname: userJourneyKubeApiserverAvailability1h5m exp_alerts: [] # Fast tier: 5m window (t=61-65) clean; sli_count_1h guard still blocks (~51 samples) - eval_time: 65m - alertname: UJKubeApiserverAvailability6h30m + alertname: userJourneyKubeApiserverAvailability6h30m exp_alerts: [] # sli_count_6h guard blocks: only ~11 samples at 5m step, need > 64 # Test 7: Near-threshold - minimal failure burst validates for:10m boundary # 80m of success (t=0..79m), then 7 failures (t=80..86m), then success. @@ -225,17 +230,17 @@ tests: - interval: 1m name: "Test 7: Near-threshold - minimal burst validates for:10m boundary" input_series: - - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster", namespace="clusters", _id="abc-123", cluster="mgmt-cluster"}' + - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster", namespace="clusters", _id="abc-123", cluster="mgmt-cluster", resource_id="/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster"}' values: "1+0x79 0+0x6 1+0x100" # 80 successes (t=0..79m), 7 failures (t=80..86m), then success alert_rule_test: - eval_time: 89m - alertname: UJKubeApiserverAvailability1h5m + alertname: userJourneyKubeApiserverAvailability1h5m exp_alerts: [] # for: timer = 9m, not yet 10m — alert not yet firing - eval_time: 90m - alertname: UJKubeApiserverAvailability1h5m + alertname: userJourneyKubeApiserverAvailability1h5m exp_alerts: - exp_labels: - severity: critical + severity: "3" slo: kas-availability long_window: 1h short_window: 5m @@ -243,15 +248,16 @@ tests: namespace: clusters _id: abc-123 cluster: mgmt-cluster + resource_id: /subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster exp_annotations: - summary: "[HCP] [test-cluster] KubeAPIServer Fast Error Budget Burn (1h/5m)" - description: "CID: abc-123\nManagement Cluster: mgmt-cluster/clusters\n" + summary: "[HCPKASAvailableBurn] mgmt-cluster / clusters (1h/5m)" + description: "Resource ID: /subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster\nOCM Cluster ID: abc-123\nManagement Cluster: mgmt-cluster\nNamespace: clusters\n" runbook_url: https://aka.ms/arohcp-runbook-ujkasavailable - correlationId: "UJKubeApiserverAvailability/mgmt-cluster/abc-123" + correlationId: "userJourneyKubeApiserverAvailability/mgmt-cluster/abc-123" - eval_time: 91m - alertname: UJKubeApiserverAvailability1h5m + alertname: userJourneyKubeApiserverAvailability1h5m exp_alerts: [] # 5m window (86,91]={87..91m} all success → error_rate_5m=0 → alert resolves -# Test 8: Slow burn - sustained complete failure triggers UJKubeApiserverAvailability3d6h +# Test 8: Slow burn - sustained complete failure triggers userJourneyKubeApiserverAvailability3d6h # The slow-burn (3d/6h) alert uses sli_count_3d with [3d:30m] subquery (30m step). # With interval:1m and data from t=0, the offset gate (offset 15m) requires data at t-15m. # At subquery step t=30m: needs data at 15m (exists with 1m data density) → passes. @@ -263,22 +269,22 @@ tests: # eval_time=4109m → for:timer=179m → NOT firing (1-minute before threshold). # eval_time=4110m → for:timer=180m=3h → FIRES. - interval: 1m - name: "Test 8: Slow burn - sustained failure triggers UJKubeApiserverAvailability3d6h" + name: "Test 8: Slow burn - sustained failure triggers userJourneyKubeApiserverAvailability3d6h" input_series: - - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster", namespace="clusters", _id="abc-123", cluster="mgmt-cluster"}' + - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster", namespace="clusters", _id="abc-123", cluster="mgmt-cluster", resource_id="/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster"}' values: "0+0x4110" # 4111 samples at 1m interval, t=0 to t=4110m, complete failure alert_rule_test: - eval_time: 4080m - alertname: UJKubeApiserverAvailability3d6h + alertname: userJourneyKubeApiserverAvailability3d6h exp_alerts: [] # for: timer = 150m at T=4080m, not yet 3h — alert not yet firing - eval_time: 4109m - alertname: UJKubeApiserverAvailability3d6h + alertname: userJourneyKubeApiserverAvailability3d6h exp_alerts: [] # for: timer = 179m at T=4109m, not yet 3h — alert not yet firing - eval_time: 4110m - alertname: UJKubeApiserverAvailability3d6h + alertname: userJourneyKubeApiserverAvailability3d6h exp_alerts: - exp_labels: - severity: warning + severity: "3" slo: kas-availability long_window: 3d short_window: 6h @@ -286,8 +292,9 @@ tests: namespace: clusters _id: abc-123 cluster: mgmt-cluster + resource_id: /subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster exp_annotations: - summary: "[HCP] [test-cluster] KubeAPIServer Slow Error Budget Burn (3d/6h)" - description: "CID: abc-123\nManagement Cluster: mgmt-cluster/clusters\n" + summary: "[HCPKASAvailableBurn] mgmt-cluster / clusters (3d/6h)" + description: "Resource ID: /subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster\nOCM Cluster ID: abc-123\nManagement Cluster: mgmt-cluster\nNamespace: clusters\n" runbook_url: https://aka.ms/arohcp-runbook-ujkasavailable - correlationId: "UJKubeApiserverAvailability/mgmt-cluster/abc-123" + correlationId: "userJourneyKubeApiserverAvailability/mgmt-cluster/abc-123" diff --git a/observability/alerts/UserJourneyKubeApiserverAvailabilityRecord-prometheusRule-KSM.yaml b/observability/alerts/UserJourneyKubeApiserverAvailabilityRecord-prometheusRule-KSM.yaml index 83d9c212939..3e120c750a0 100644 --- a/observability/alerts/UserJourneyKubeApiserverAvailabilityRecord-prometheusRule-KSM.yaml +++ b/observability/alerts/UserJourneyKubeApiserverAvailabilityRecord-prometheusRule-KSM.yaml @@ -1,87 +1,87 @@ apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: - name: uj-kubeapiserver-availability-recording-rules + name: userjourney-kubeapiserver-availability-recording-rules namespace: prometheus spec: groups: - - name: uj-kubeapiserver-availability-recording-rules + - name: userjourney-kubeapiserver-availability-recording-rules rules: - record: hostedClusterAPI_kubeapiserver_available:sli_sum_5m expr: | - sum by (name, namespace, _id, cluster) ( + sum by (name, namespace, _id, resource_id, cluster) ( sum_over_time( ( hostedClusterAPI_kubeapiserver_available{status="True"} - and on (name, namespace, _id, cluster) - max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) + and on (name, namespace, _id, resource_id, cluster) + max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) )[5m:1m] ) ) - and on (name, namespace, _id, cluster) - count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + and on (name, namespace, _id, resource_id, cluster) + count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) - record: hostedClusterAPI_kubeapiserver_available:sli_count_5m expr: | - sum by (name, namespace, _id, cluster) ( + sum by (name, namespace, _id, resource_id, cluster) ( count_over_time( ( - max by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) - and on (name, namespace, _id, cluster) - max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) + max by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) + and on (name, namespace, _id, resource_id, cluster) + max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) )[5m:1m] ) ) - and on (name, namespace, _id, cluster) - count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + and on (name, namespace, _id, resource_id, cluster) + count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) - record: hostedClusterAPI_kubeapiserver_available:sli_sum_1h expr: | - sum by (name, namespace, _id, cluster) ( + sum by (name, namespace, _id, resource_id, cluster) ( sum_over_time( ( hostedClusterAPI_kubeapiserver_available{status="True"} - and on (name, namespace, _id, cluster) - max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) + and on (name, namespace, _id, resource_id, cluster) + max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) )[1h:1m] ) ) - and on (name, namespace, _id, cluster) - count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + and on (name, namespace, _id, resource_id, cluster) + count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) - record: hostedClusterAPI_kubeapiserver_available:sli_count_1h expr: | - sum by (name, namespace, _id, cluster) ( + sum by (name, namespace, _id, resource_id, cluster) ( count_over_time( ( - max by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) - and on (name, namespace, _id, cluster) - max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) + max by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) + and on (name, namespace, _id, resource_id, cluster) + max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) )[1h:1m] ) ) - and on (name, namespace, _id, cluster) - count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + and on (name, namespace, _id, resource_id, cluster) + count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) - record: hostedClusterAPI_kubeapiserver_available:sli_sum_30m expr: | - sum by (name, namespace, _id, cluster) ( + sum by (name, namespace, _id, resource_id, cluster) ( sum_over_time( ( hostedClusterAPI_kubeapiserver_available{status="True"} - and on (name, namespace, _id, cluster) - max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) + and on (name, namespace, _id, resource_id, cluster) + max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) )[30m:1m] ) ) - and on (name, namespace, _id, cluster) - count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + and on (name, namespace, _id, resource_id, cluster) + count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) - record: hostedClusterAPI_kubeapiserver_available:sli_count_30m expr: | - sum by (name, namespace, _id, cluster) ( + sum by (name, namespace, _id, resource_id, cluster) ( count_over_time( ( - max by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) - and on (name, namespace, _id, cluster) - max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) + max by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) + and on (name, namespace, _id, resource_id, cluster) + max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0) )[30m:1m] ) ) - and on (name, namespace, _id, cluster) - count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) + and on (name, namespace, _id, resource_id, cluster) + count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) diff --git a/observability/alerts/UserJourneyKubeApiserverAvailabilityRecord-prometheusRule-KSM_test.yaml b/observability/alerts/UserJourneyKubeApiserverAvailabilityRecord-prometheusRule-KSM_test.yaml index 94feb9639ae..c098136ea8f 100644 --- a/observability/alerts/UserJourneyKubeApiserverAvailabilityRecord-prometheusRule-KSM_test.yaml +++ b/observability/alerts/UserJourneyKubeApiserverAvailabilityRecord-prometheusRule-KSM_test.yaml @@ -1,23 +1,23 @@ rule_files: - HCPkasRecord-prometheusRule-KSM.yaml -- UJKubeApiserverAvailabilityRecord-prometheusRule-KSM.yaml +- UserJourneyKubeApiserverAvailabilityRecord-prometheusRule-KSM.yaml evaluation_interval: 1m tests: -# Recording rules are exercised end-to-end by UJKubeApiserverAvailabilityMonitor-prometheusRule-KSM_test.yaml +# Recording rules are exercised end-to-end by UserJourneyKubeApiserverAvailabilityMonitor-prometheusRule-KSM_test.yaml # (6 alert scenarios covering all 6 SLI rules). This file satisfies the generator's # test-file requirement with a basic smoke test. - interval: 1m input_series: - - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster", namespace="clusters", _id="abc-123", cluster="mgmt-cluster"}' + - series: 'hostedClusterAPI_kubeapiserver_available{status="True", name="test-cluster", namespace="clusters", _id="abc-123", cluster="mgmt-cluster", resource_id="/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster"}' values: "1+0x30" promql_expr_test: - eval_time: 20m expr: hostedClusterAPI_kubeapiserver_available:sli_sum_5m{name="test-cluster"} exp_samples: - - labels: 'hostedClusterAPI_kubeapiserver_available:sli_sum_5m{_id="abc-123", cluster="mgmt-cluster", name="test-cluster", namespace="clusters"}' + - labels: 'hostedClusterAPI_kubeapiserver_available:sli_sum_5m{_id="abc-123", cluster="mgmt-cluster", name="test-cluster", namespace="clusters", resource_id="/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster"}' value: 5 - eval_time: 20m expr: hostedClusterAPI_kubeapiserver_available:sli_count_5m{name="test-cluster"} exp_samples: - - labels: 'hostedClusterAPI_kubeapiserver_available:sli_count_5m{_id="abc-123", cluster="mgmt-cluster", name="test-cluster", namespace="clusters"}' + - labels: 'hostedClusterAPI_kubeapiserver_available:sli_count_5m{_id="abc-123", cluster="mgmt-cluster", name="test-cluster", namespace="clusters", resource_id="/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster"}' value: 5 diff --git a/observability/recording-rules-hcps.yaml b/observability/recording-rules-hcps.yaml index 7879ae97d49..21d373251f8 100644 --- a/observability/recording-rules-hcps.yaml +++ b/observability/recording-rules-hcps.yaml @@ -3,6 +3,6 @@ prometheusRules: - alerts/HCPkasRecord-prometheusRule-KSM.yaml - alerts/HCPkasRecord-prometheusRule-apiserver-requests.yaml - alerts/HCPkasRecord-prometheusRule-latency.yaml - - alerts/UJKubeApiserverAvailabilityRecord-prometheusRule-KSM.yaml + - alerts/UserJourneyKubeApiserverAvailabilityRecord-prometheusRule-KSM.yaml untestedRules: [] outputBicep: ../dev-infrastructure/modules/metrics/rules/generatedHCPRecordingRules.bicep diff --git a/tooling/prometheus-rules/Makefile b/tooling/prometheus-rules/Makefile index d4dd3d0457b..afe7cd31637 100644 --- a/tooling/prometheus-rules/Makefile +++ b/tooling/prometheus-rules/Makefile @@ -23,7 +23,7 @@ fmt-devinfra: run: alerts recording-rules .PHONY: run -alerts: run-sl-services run-sl-hcps run-sre-hcps run-sre-services run-rp-services run-msft-services run-kusto-services run-kusto-hcps +alerts: run-sl-services run-sl-hcps run-sre-hcps run-sre-services run-rp-services run-rp-hcps run-msft-services run-kusto-services run-kusto-hcps $(MAKE) fmt-devinfra .PHONY: alerts @@ -51,6 +51,10 @@ run-rp-services: go run ./... --config-file ../../observability/alerts-rp-services.yaml $(EXTRA_FLAGS) .PHONY: run-rp-services +run-rp-hcps: + go run ./... --config-file ../../observability/alerts-rp-hcps.yaml $(EXTRA_FLAGS) +.PHONY: run-rp-hcps + run-msft-services: go run ./... --config-file ../../observability/alerts-msft-services.yaml $(EXTRA_FLAGS) .PHONY: run-msft-services @@ -77,6 +81,7 @@ ALERT_CONFIGS = \ ../../observability/alerts-sre-hcps.yaml \ ../../observability/alerts-sre-services.yaml \ ../../observability/alerts-rp-services.yaml \ + ../../observability/alerts-rp-hcps.yaml \ ../../observability/alerts-msft-services.yaml \ ../../observability/alerts-kusto-services.yaml \ ../../observability/alerts-kusto-hcps.yaml From 33b153f72c6855d8127a438d1ee9e24fba02db2f Mon Sep 17 00:00:00 2001 From: Brendan Bergen Date: Fri, 10 Jul 2026 15:34:27 -0600 Subject: [PATCH 4/4] chore(observability): regenerate bicep output Generated from `make alerts recording-rules` in observability/. Co-Authored-By: Claude Opus 4.6 --- .../rules/generatedHCPRecordingRules.bicep | 82 ++++++++--- ...eneratedRPHCPPrometheusAlertingRules.bicep | 138 ++++++++++++++++++ 2 files changed, 202 insertions(+), 18 deletions(-) create mode 100644 dev-infrastructure/modules/metrics/rules/generatedRPHCPPrometheusAlertingRules.bicep diff --git a/dev-infrastructure/modules/metrics/rules/generatedHCPRecordingRules.bicep b/dev-infrastructure/modules/metrics/rules/generatedHCPRecordingRules.bicep index 0cf7772a7f9..60f1c97c613 100644 --- a/dev-infrastructure/modules/metrics/rules/generatedHCPRecordingRules.bicep +++ b/dev-infrastructure/modules/metrics/rules/generatedHCPRecordingRules.bicep @@ -14,75 +14,83 @@ resource hcpKmsRecordingRules 'Microsoft.AlertsManagement/prometheusRuleGroups@2 rules: [ { record: 'hostedClusterAPI_kubeapiserver_available:ratio_avg_30d' - expression: 'avg by (name, namespace, _id, cluster) (avg_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[30d])) and on (name, namespace, _id, cluster) count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)' + expression: 'avg by (name, namespace, _id, resource_id, cluster) (avg_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[30d])) and on (name, namespace, _id, resource_id, cluster) count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)' } { record: 'hostedClusterAPI_kubeapiserver_available:ratio_avg_7d' - expression: 'avg by (name, namespace, _id, cluster) (avg_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[1w])) and on (name, namespace, _id, cluster) count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)' + expression: 'avg by (name, namespace, _id, resource_id, cluster) (avg_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[1w])) and on (name, namespace, _id, resource_id, cluster) count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)' } { record: 'hostedClusterAPI_kubeapiserver_available:ratio_avg_1d' - expression: 'avg by (name, namespace, _id, cluster) (avg_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[1d])) and on (name, namespace, _id, cluster) count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)' + expression: 'avg by (name, namespace, _id, resource_id, cluster) (avg_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[1d])) and on (name, namespace, _id, resource_id, cluster) count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)' } { record: 'hostedClusterAPI_kubeapiserver_available:ratio_avg_3d' - expression: 'avg by (name, namespace, _id, cluster) (avg_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[3d])) and on (name, namespace, _id, cluster) count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)' + expression: 'avg by (name, namespace, _id, resource_id, cluster) (avg_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[3d])) and on (name, namespace, _id, resource_id, cluster) count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)' } { record: 'hostedClusterAPI_kubeapiserver_available:sum_over_time_30m' - expression: 'sum by (name, namespace, _id, cluster) (sum_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[30m])) and on (name, namespace, _id, cluster) count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)' + expression: 'sum by (name, namespace, _id, resource_id, cluster) (sum_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[30m])) and on (name, namespace, _id, resource_id, cluster) count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)' } { record: 'hostedClusterAPI_kubeapiserver_available:sum_over_time_1h' - expression: 'sum by (name, namespace, _id, cluster) (sum_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[1h])) and on (name, namespace, _id, cluster) count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)' + expression: 'sum by (name, namespace, _id, resource_id, cluster) (sum_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[1h])) and on (name, namespace, _id, resource_id, cluster) count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)' } { record: 'hostedClusterAPI_kubeapiserver_available:sum_over_time_2h' - expression: 'sum by (name, namespace, _id, cluster) (sum_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[2h])) and on (name, namespace, _id, cluster) count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)' + expression: 'sum by (name, namespace, _id, resource_id, cluster) (sum_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[2h])) and on (name, namespace, _id, resource_id, cluster) count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)' } { record: 'hostedClusterAPI_kubeapiserver_available:sum_over_time_6h' - expression: 'sum by (name, namespace, _id, cluster) (sum_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[6h])) and on (name, namespace, _id, cluster) count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)' + expression: 'sum by (name, namespace, _id, resource_id, cluster) (sum_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[6h])) and on (name, namespace, _id, resource_id, cluster) count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)' } { record: 'hostedClusterAPI_kubeapiserver_available:count_over_time_30m' - expression: 'sum by (name, namespace, _id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[30m])) and on (name, namespace, _id, cluster) count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)' + expression: 'sum by (name, namespace, _id, resource_id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[30m])) and on (name, namespace, _id, resource_id, cluster) count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)' } { record: 'hostedClusterAPI_kubeapiserver_available:count_over_time_1h' - expression: 'sum by (name, namespace, _id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[1h])) and on (name, namespace, _id, cluster) count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)' + expression: 'sum by (name, namespace, _id, resource_id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[1h])) and on (name, namespace, _id, resource_id, cluster) count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)' } { record: 'hostedClusterAPI_kubeapiserver_available:count_over_time_2h' - expression: 'sum by (name, namespace, _id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[2h])) and on (name, namespace, _id, cluster) count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)' + expression: 'sum by (name, namespace, _id, resource_id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[2h])) and on (name, namespace, _id, resource_id, cluster) count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)' } { record: 'hostedClusterAPI_kubeapiserver_available:count_over_time_6h' - expression: 'sum by (name, namespace, _id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[6h])) and on (name, namespace, _id, cluster) count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)' + expression: 'sum by (name, namespace, _id, resource_id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[6h])) and on (name, namespace, _id, resource_id, cluster) count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)' } { record: 'hostedClusterAPI_kubeapiserver_available:count_over_time_1d' - expression: 'sum by (name, namespace, _id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[1d])) and on (name, namespace, _id, cluster) count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)' + expression: 'sum by (name, namespace, _id, resource_id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[1d])) and on (name, namespace, _id, resource_id, cluster) count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)' } { record: 'hostedClusterAPI_kubeapiserver_available:count_over_time_3d' - expression: 'sum by (name, namespace, _id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[3d])) and on (name, namespace, _id, cluster) count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)' + expression: 'sum by (name, namespace, _id, resource_id, cluster) (count_over_time(hostedClusterAPI_kubeapiserver_available{status="True"}[3d])) and on (name, namespace, _id, resource_id, cluster) count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)' } { record: 'hostedClusterAPI_kubeapiserver_available:sli_sum_15m' - expression: 'sum by (name, namespace, _id, cluster) (sum_over_time((hostedClusterAPI_kubeapiserver_available{status="True"} and on (name, namespace, _id, cluster) max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0))[15m:1m])) and on (name, namespace, _id, cluster) count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)' + expression: 'sum by (name, namespace, _id, resource_id, cluster) (sum_over_time((hostedClusterAPI_kubeapiserver_available{status="True"} and on (name, namespace, _id, resource_id, cluster) max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0))[15m:1m])) and on (name, namespace, _id, resource_id, cluster) count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)' } { record: 'hostedClusterAPI_kubeapiserver_available:sli_count_15m' - expression: 'sum by (name, namespace, _id, cluster) (count_over_time((max by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) and on (name, namespace, _id, cluster) max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0))[15m:1m])) and on (name, namespace, _id, cluster) count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)' + expression: 'sum by (name, namespace, _id, resource_id, cluster) (count_over_time((max by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) and on (name, namespace, _id, resource_id, cluster) max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0))[15m:1m])) and on (name, namespace, _id, resource_id, cluster) count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)' } { record: 'hostedClusterAPI_kubeapiserver_available:sli_sum_6h' - expression: 'sum by (name, namespace, _id, cluster) (sum_over_time((hostedClusterAPI_kubeapiserver_available{status="True"} and on (name, namespace, _id, cluster) max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0))[6h:5m])) and on (name, namespace, _id, cluster) count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)' + expression: 'sum by (name, namespace, _id, resource_id, cluster) (sum_over_time((hostedClusterAPI_kubeapiserver_available{status="True"} and on (name, namespace, _id, resource_id, cluster) max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0))[6h:5m])) and on (name, namespace, _id, resource_id, cluster) count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)' } { record: 'hostedClusterAPI_kubeapiserver_available:sli_count_6h' - expression: 'sum by (name, namespace, _id, cluster) (count_over_time((max by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available) and on (name, namespace, _id, cluster) max by (name, namespace, _id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0))[6h:5m])) and on (name, namespace, _id, cluster) count by (name, namespace, _id, cluster) (hostedClusterAPI_kubeapiserver_available)' + expression: 'sum by (name, namespace, _id, resource_id, cluster) (count_over_time((max by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) and on (name, namespace, _id, resource_id, cluster) max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0))[6h:5m])) and on (name, namespace, _id, resource_id, cluster) count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)' + } + { + record: 'hostedClusterAPI_kubeapiserver_available:sli_sum_3d' + expression: 'sum by (name, namespace, _id, resource_id, cluster) (sum_over_time((hostedClusterAPI_kubeapiserver_available{status="True"} and on (name, namespace, _id, resource_id, cluster) max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0))[3d:30m])) and on (name, namespace, _id, resource_id, cluster) count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)' + } + { + record: 'hostedClusterAPI_kubeapiserver_available:sli_count_3d' + expression: 'sum by (name, namespace, _id, resource_id, cluster) (count_over_time((max by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) and on (name, namespace, _id, resource_id, cluster) max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0))[3d:30m])) and on (name, namespace, _id, resource_id, cluster) count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)' } ] } @@ -195,3 +203,41 @@ resource hcpKasLatencyRecordingRules 'Microsoft.AlertsManagement/prometheusRuleG ] } } + +resource userjourneyKubeapiserverAvailabilityRecordingRules 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'userjourney-kubeapiserver-availability-recording-rules' + location: location + properties: { + scopes: [ + azureMonitoring + ] + enabled: true + interval: 'PT1M' + rules: [ + { + record: 'hostedClusterAPI_kubeapiserver_available:sli_sum_5m' + expression: 'sum by (name, namespace, _id, resource_id, cluster) (sum_over_time((hostedClusterAPI_kubeapiserver_available{status="True"} and on (name, namespace, _id, resource_id, cluster) max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0))[5m:1m])) and on (name, namespace, _id, resource_id, cluster) count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)' + } + { + record: 'hostedClusterAPI_kubeapiserver_available:sli_count_5m' + expression: 'sum by (name, namespace, _id, resource_id, cluster) (count_over_time((max by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) and on (name, namespace, _id, resource_id, cluster) max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0))[5m:1m])) and on (name, namespace, _id, resource_id, cluster) count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)' + } + { + record: 'hostedClusterAPI_kubeapiserver_available:sli_sum_1h' + expression: 'sum by (name, namespace, _id, resource_id, cluster) (sum_over_time((hostedClusterAPI_kubeapiserver_available{status="True"} and on (name, namespace, _id, resource_id, cluster) max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0))[1h:1m])) and on (name, namespace, _id, resource_id, cluster) count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)' + } + { + record: 'hostedClusterAPI_kubeapiserver_available:sli_count_1h' + expression: 'sum by (name, namespace, _id, resource_id, cluster) (count_over_time((max by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) and on (name, namespace, _id, resource_id, cluster) max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0))[1h:1m])) and on (name, namespace, _id, resource_id, cluster) count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)' + } + { + record: 'hostedClusterAPI_kubeapiserver_available:sli_sum_30m' + expression: 'sum by (name, namespace, _id, resource_id, cluster) (sum_over_time((hostedClusterAPI_kubeapiserver_available{status="True"} and on (name, namespace, _id, resource_id, cluster) max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0))[30m:1m])) and on (name, namespace, _id, resource_id, cluster) count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)' + } + { + record: 'hostedClusterAPI_kubeapiserver_available:sli_count_30m' + expression: 'sum by (name, namespace, _id, resource_id, cluster) (count_over_time((max by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available) and on (name, namespace, _id, resource_id, cluster) max by (name, namespace, _id, resource_id, cluster) ((hostedClusterAPI_kubeapiserver_available offset 15m) >= 0))[30m:1m])) and on (name, namespace, _id, resource_id, cluster) count by (name, namespace, _id, resource_id, cluster) (hostedClusterAPI_kubeapiserver_available)' + } + ] + } +} diff --git a/dev-infrastructure/modules/metrics/rules/generatedRPHCPPrometheusAlertingRules.bicep b/dev-infrastructure/modules/metrics/rules/generatedRPHCPPrometheusAlertingRules.bicep new file mode 100644 index 00000000000..3423c2fab43 --- /dev/null +++ b/dev-infrastructure/modules/metrics/rules/generatedRPHCPPrometheusAlertingRules.bicep @@ -0,0 +1,138 @@ +#disable-next-line no-unused-params +param azureMonitoring string + +#disable-next-line no-unused-params +param actionGroups array + +@description('The minimum IcM severity level (highest priority) that alerts can fire at. Alerts more critical than this ceiling will be degraded to this value. 0 means no ceiling.') +param severityCeiling int = 0 + +#disable-next-line no-unused-params +param location string = resourceGroup().location + +resource rpUserjourneyKasAvailabilityMonitorRules 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'rp-userjourney-kas-availability-monitor-rules' + location: location + properties: { + interval: 'PT1M' + rules: [ + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'userJourneyKubeApiserverAvailability1h5m' + enabled: true + labels: { + long_window: '1h' + severity: '3' + short_window: '5m' + slo: 'kas-availability' + } + annotations: { + correlationId: 'userJourneyKubeApiserverAvailability/{{ $labels.cluster }}/{{ $labels._id }}' + description: '''Resource ID: {{ $labels.resource_id }} +OCM Cluster ID: {{ $labels._id }} +Management Cluster: {{ $labels.cluster }} +Namespace: {{ $labels.namespace }} +''' + info: '''Resource ID: {{ $labels.resource_id }} +OCM Cluster ID: {{ $labels._id }} +Management Cluster: {{ $labels.cluster }} +Namespace: {{ $labels.namespace }} +''' + runbook_url: 'https://aka.ms/arohcp-runbook-ujkasavailable' + summary: '[HCPKASAvailableBurn] {{ $labels.cluster }} / {{ $labels.namespace }} (1h/5m)' + title: '[HCPKASAvailableBurn] {{ $labels.cluster }} / {{ $labels.namespace }} (1h/5m) resource_id:{{ $labels.resource_id }} _id:{{ $labels._id }}' + } + expression: '(1 - (hostedClusterAPI_kubeapiserver_available:sli_sum_5m / hostedClusterAPI_kubeapiserver_available:sli_count_5m) > (14.4 * (1 - 0.9995)) and on (name, namespace, _id, resource_id, cluster) hostedClusterAPI_kubeapiserver_available:sli_count_5m > 3) and on (name, namespace, _id, resource_id, cluster) (1 - (hostedClusterAPI_kubeapiserver_available:sli_sum_1h / hostedClusterAPI_kubeapiserver_available:sli_count_1h) > (14.4 * (1 - 0.9995)) and on (name, namespace, _id, resource_id, cluster) hostedClusterAPI_kubeapiserver_available:sli_count_1h > 54)' + for: 'PT10M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'userJourneyKubeApiserverAvailability6h30m' + enabled: true + labels: { + long_window: '6h' + severity: '3' + short_window: '30m' + slo: 'kas-availability' + } + annotations: { + correlationId: 'userJourneyKubeApiserverAvailability/{{ $labels.cluster }}/{{ $labels._id }}' + description: '''Resource ID: {{ $labels.resource_id }} +OCM Cluster ID: {{ $labels._id }} +Management Cluster: {{ $labels.cluster }} +Namespace: {{ $labels.namespace }} +''' + info: '''Resource ID: {{ $labels.resource_id }} +OCM Cluster ID: {{ $labels._id }} +Management Cluster: {{ $labels.cluster }} +Namespace: {{ $labels.namespace }} +''' + runbook_url: 'https://aka.ms/arohcp-runbook-ujkasavailable' + summary: '[HCPKASAvailableBurn] {{ $labels.cluster }} / {{ $labels.namespace }} (6h/30m)' + title: '[HCPKASAvailableBurn] {{ $labels.cluster }} / {{ $labels.namespace }} (6h/30m) resource_id:{{ $labels.resource_id }} _id:{{ $labels._id }}' + } + expression: '(1 - (hostedClusterAPI_kubeapiserver_available:sli_sum_30m / hostedClusterAPI_kubeapiserver_available:sli_count_30m) > (6 * (1 - 0.9995)) and on (name, namespace, _id, resource_id, cluster) hostedClusterAPI_kubeapiserver_available:sli_count_30m > 27) and on (name, namespace, _id, resource_id, cluster) (1 - (hostedClusterAPI_kubeapiserver_available:sli_sum_6h / hostedClusterAPI_kubeapiserver_available:sli_count_6h) > (6 * (1 - 0.9995)) and on (name, namespace, _id, resource_id, cluster) hostedClusterAPI_kubeapiserver_available:sli_count_6h > 64)' + for: 'PT30M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'userJourneyKubeApiserverAvailability3d6h' + enabled: true + labels: { + long_window: '3d' + severity: '3' + short_window: '6h' + slo: 'kas-availability' + } + annotations: { + correlationId: 'userJourneyKubeApiserverAvailability/{{ $labels.cluster }}/{{ $labels._id }}' + description: '''Resource ID: {{ $labels.resource_id }} +OCM Cluster ID: {{ $labels._id }} +Management Cluster: {{ $labels.cluster }} +Namespace: {{ $labels.namespace }} +''' + info: '''Resource ID: {{ $labels.resource_id }} +OCM Cluster ID: {{ $labels._id }} +Management Cluster: {{ $labels.cluster }} +Namespace: {{ $labels.namespace }} +''' + runbook_url: 'https://aka.ms/arohcp-runbook-ujkasavailable' + summary: '[HCPKASAvailableBurn] {{ $labels.cluster }} / {{ $labels.namespace }} (3d/6h)' + title: '[HCPKASAvailableBurn] {{ $labels.cluster }} / {{ $labels.namespace }} (3d/6h) resource_id:{{ $labels.resource_id }} _id:{{ $labels._id }}' + } + expression: '(1 - (hostedClusterAPI_kubeapiserver_available:sli_sum_6h / hostedClusterAPI_kubeapiserver_available:sli_count_6h) > (1 * (1 - 0.9995)) and on (name, namespace, _id, resource_id, cluster) hostedClusterAPI_kubeapiserver_available:sli_count_6h > 64) and on (name, namespace, _id, resource_id, cluster) (1 - (hostedClusterAPI_kubeapiserver_available:sli_sum_3d / hostedClusterAPI_kubeapiserver_available:sli_count_3d) > (1 * (1 - 0.9995)) and on (name, namespace, _id, resource_id, cluster) hostedClusterAPI_kubeapiserver_available:sli_count_3d > 130)' + for: 'PT3H' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + ] + scopes: [ + azureMonitoring + ] + } +}