From e15883c1a614059149a63dba5d87de8fb5f6b048 Mon Sep 17 00:00:00 2001 From: Manyanda Chitimbo Date: Tue, 23 Jun 2026 15:05:20 +0200 Subject: [PATCH 1/4] feat: add cluster version selection-to-active SLO rules and alerts Track control plane version upgrade activation against a 99.95% / 30m SLO from backend_cluster_version_info. Register recording rules for Azure Monitor and add multi-window burn-rate alerts at Sev 4. Co-authored-by: Cursor feat: add cluster version selection-to-active SLO Grafana dashboard Fleet-wide SLI view for upgrade activation latency, stuck rate, and error budget consumption. Co-authored-by: Cursor feat: clarify cluster upgrade naming in version selection-to-active SLO Rename alerts, summaries, and dashboard copy to explicitly describe cluster upgrade version selection-to-active so on-call context is clearer. Co-authored-by: Cursor feat: add UJ cluster upgrade partial and complete SLO observability Introduce user-journey recording rules, burn-rate alerts, promtool tests, and Grafana dashboards for selection-to-partial (20m) and upgrade-complete SLOs, replacing the earlier selection-to-active prototype with UJClusterUpgrade naming aligned to cluster-upgrade dashboards and slo labels. Co-authored-by: Cursor refactor: make runbook as TBD refactor: rename shared cluster version state_first_seen_timestamp metric Drop selection_to_partial from the first-seen recording rule name since it tracks desired, partial, and completed states and is shared by both upgrade SLOs. Co-authored-by: Cursor refactor: align cluster upgrade SLO alerts with RP lane conventions Use long-window-first alert tier suffixes and aka.ms/arohcp-runbook-cluster-upgrade runbook URLs so RP lane monitors match team naming and review requirements. Co-authored-by: Cursor refactor: rename the recording rules following PR reviews Uses https://prometheus.io/docs/practices/rules/#naming for guidance Co-authored-by: Cursor --- .../generatedRPPrometheusAlertingRules.bicep | 242 +++ .../rules/generatedRecordingRules.bicep | 100 ++ observability/alerts-rp-services.yaml | 2 + ...UpgradeCompleteMonitor-prometheusRule.yaml | 82 + ...deCompleteMonitor-prometheusRule_test.yaml | 772 +++++++++ ...adeReachPartialMonitor-prometheusRule.yaml | 72 + ...achPartialMonitor-prometheusRule_test.yaml | 778 +++++++++ ...eyClusterUpgradeRecord-prometheusRule.yaml | 197 +++ ...sterUpgradeRecord-prometheusRule_test.yaml | 135 ++ .../cluster-upgrade-slo-dashboard.json | 1490 +++++++++++++++++ observability/recording-rules-services.yaml | 1 + 11 files changed, 3871 insertions(+) create mode 100644 observability/alerts/userJourneyClusterUpgradeCompleteMonitor-prometheusRule.yaml create mode 100644 observability/alerts/userJourneyClusterUpgradeCompleteMonitor-prometheusRule_test.yaml create mode 100644 observability/alerts/userJourneyClusterUpgradeReachPartialMonitor-prometheusRule.yaml create mode 100644 observability/alerts/userJourneyClusterUpgradeReachPartialMonitor-prometheusRule_test.yaml create mode 100644 observability/alerts/userJourneyClusterUpgradeRecord-prometheusRule.yaml create mode 100644 observability/alerts/userJourneyClusterUpgradeRecord-prometheusRule_test.yaml create mode 100644 observability/grafana-dashboards/sre/user-journey/cluster-upgrade-slo-dashboard.json diff --git a/dev-infrastructure/modules/metrics/rules/generatedRPPrometheusAlertingRules.bicep b/dev-infrastructure/modules/metrics/rules/generatedRPPrometheusAlertingRules.bicep index 57dd23e379d..4b81bebb931 100644 --- a/dev-infrastructure/modules/metrics/rules/generatedRPPrometheusAlertingRules.bicep +++ b/dev-infrastructure/modules/metrics/rules/generatedRPPrometheusAlertingRules.bicep @@ -234,6 +234,248 @@ resource arohcpAccessClusterSaturationAlerts 'Microsoft.AlertsManagement/prometh } } +resource rpUserJourneyClusterUpgradeReachPartialMonitorRules 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'rp-user-journey-cluster-upgrade-reach-partial-monitor-rules' + location: location + properties: { + interval: 'PT1M' + rules: [ + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'userJourneyClusterUpgradeReachPartial1h5m' + enabled: true + labels: { + long_window: '1h' + severity: 'info' + short_window: '5m' + slo: 'cluster-upgrade-reach-partial' + } + annotations: { + correlationId: 'userJourneyClusterUpgradeReachPartial/{{ $labels.cluster }}' + description: '''More than 14.4x of the 99.95% cluster upgrade reach-partial error budget is being consumed. Cluster upgrades are stuck for over 20 minutes without the target version reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. +Window: 1h/5m +Management Cluster: {{ $labels.cluster }} +''' + info: '''More than 14.4x of the 99.95% cluster upgrade reach-partial error budget is being consumed. Cluster upgrades are stuck for over 20 minutes without the target version reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. +Window: 1h/5m +Management Cluster: {{ $labels.cluster }} +''' + runbook_url: 'https://aka.ms/arohcp-runbook-cluster-upgrade' + summary: '[HCPUpgradeReachPartialBurn] {{ $labels.cluster }}' + title: '[HCPUpgradeReachPartialBurn] {{ $labels.cluster }}' + } + expression: 'hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate5m > 14.4 and hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate1h > 14.4' + for: 'PT5M' + severity: severityCeiling > 0 ? max(4, severityCeiling) : 4 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'userJourneyClusterUpgradeReachPartial6h30m' + enabled: true + labels: { + long_window: '6h' + severity: 'info' + short_window: '30m' + slo: 'cluster-upgrade-reach-partial' + } + annotations: { + correlationId: 'userJourneyClusterUpgradeReachPartial/{{ $labels.cluster }}' + description: '''More than 6x of the 99.95% cluster upgrade reach-partial error budget is being consumed. Cluster upgrades are stuck for over 20 minutes without the target version reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. +Window: 6h/30m +Management Cluster: {{ $labels.cluster }} +''' + info: '''More than 6x of the 99.95% cluster upgrade reach-partial error budget is being consumed. Cluster upgrades are stuck for over 20 minutes without the target version reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. +Window: 6h/30m +Management Cluster: {{ $labels.cluster }} +''' + runbook_url: 'https://aka.ms/arohcp-runbook-cluster-upgrade' + summary: '[HCPUpgradeReachPartialBurn] {{ $labels.cluster }}' + title: '[HCPUpgradeReachPartialBurn] {{ $labels.cluster }}' + } + expression: 'hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate30m > 6 and hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate6h > 6' + for: 'PT30M' + severity: severityCeiling > 0 ? max(4, severityCeiling) : 4 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'userJourneyClusterUpgradeReachPartial3d6h' + enabled: true + labels: { + long_window: '3d' + severity: 'info' + short_window: '6h' + slo: 'cluster-upgrade-reach-partial' + } + annotations: { + correlationId: 'userJourneyClusterUpgradeReachPartial/{{ $labels.cluster }}' + description: '''Cluster upgrade reach-partial has been steadily consuming the 99.95% SLO error budget (1x burn). Cluster upgrades are stuck for over 20 minutes without reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. +Window: 3d/6h +Management Cluster: {{ $labels.cluster }} +''' + info: '''Cluster upgrade reach-partial has been steadily consuming the 99.95% SLO error budget (1x burn). Cluster upgrades are stuck for over 20 minutes without reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. +Window: 3d/6h +Management Cluster: {{ $labels.cluster }} +''' + runbook_url: 'https://aka.ms/arohcp-runbook-cluster-upgrade' + summary: '[HCPUpgradeReachPartialBurn] {{ $labels.cluster }}' + title: '[HCPUpgradeReachPartialBurn] {{ $labels.cluster }}' + } + expression: 'hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate6h > 1 and hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate3d > 1' + for: 'PT30M' + severity: severityCeiling > 0 ? max(4, severityCeiling) : 4 + } + ] + scopes: [ + azureMonitoring + ] + } +} + +resource rpUserJourneyClusterUpgradeCompleteMonitorRules 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'rp-user-journey-cluster-upgrade-complete-monitor-rules' + location: location + properties: { + interval: 'PT1M' + rules: [ + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'userJourneyClusterUpgradeComplete1h5m' + enabled: true + labels: { + long_window: '1h' + severity: 'info' + short_window: '5m' + slo: 'cluster-upgrade-complete' + } + annotations: { + correlationId: 'userJourneyClusterUpgradeComplete/{{ $labels.cluster }}' + description: '''More than 14.4x of the 99.95% cluster upgrade complete error budget is being consumed. Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. +Window: 1h/5m +Management Cluster: {{ $labels.cluster }} +''' + info: '''More than 14.4x of the 99.95% cluster upgrade complete error budget is being consumed. Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. +Window: 1h/5m +Management Cluster: {{ $labels.cluster }} +''' + runbook_url: 'https://aka.ms/arohcp-runbook-cluster-upgrade' + summary: '[HCPUpgradeCompleteBurn] {{ $labels.cluster }}' + title: '[HCPUpgradeCompleteBurn] {{ $labels.cluster }}' + } + expression: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate5m > 14.4 and hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate1h > 14.4' + for: 'PT5M' + severity: severityCeiling > 0 ? max(4, severityCeiling) : 4 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'userJourneyClusterUpgradeComplete6h30m' + enabled: true + labels: { + long_window: '6h' + severity: 'info' + short_window: '30m' + slo: 'cluster-upgrade-complete' + } + annotations: { + correlationId: 'userJourneyClusterUpgradeComplete/{{ $labels.cluster }}' + description: '''More than 6x of the 99.95% cluster upgrade complete error budget is being consumed. Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. +Window: 6h/30m +Management Cluster: {{ $labels.cluster }} +''' + info: '''More than 6x of the 99.95% cluster upgrade complete error budget is being consumed. Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. +Window: 6h/30m +Management Cluster: {{ $labels.cluster }} +''' + runbook_url: 'https://aka.ms/arohcp-runbook-cluster-upgrade' + summary: '[HCPUpgradeCompleteBurn] {{ $labels.cluster }}' + title: '[HCPUpgradeCompleteBurn] {{ $labels.cluster }}' + } + expression: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate30m > 6 and hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate6h > 6' + for: 'PT30M' + severity: severityCeiling > 0 ? max(4, severityCeiling) : 4 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'userJourneyClusterUpgradeComplete3d6h' + enabled: true + labels: { + long_window: '3d' + severity: 'info' + short_window: '6h' + slo: 'cluster-upgrade-complete' + } + annotations: { + correlationId: 'userJourneyClusterUpgradeComplete/{{ $labels.cluster }}' + description: '''Cluster upgrade complete has been steadily consuming the 99.95% SLO error budget (1x burn). Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. +Window: 3d/6h +Management Cluster: {{ $labels.cluster }} +''' + info: '''Cluster upgrade complete has been steadily consuming the 99.95% SLO error budget (1x burn). Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. +Window: 3d/6h +Management Cluster: {{ $labels.cluster }} +''' + runbook_url: 'https://aka.ms/arohcp-runbook-cluster-upgrade' + summary: '[HCPUpgradeCompleteBurn] {{ $labels.cluster }}' + title: '[HCPUpgradeCompleteBurn] {{ $labels.cluster }}' + } + expression: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate6h > 1 and hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate3d > 1' + for: 'PT30M' + severity: severityCeiling > 0 ? max(4, severityCeiling) : 4 + } + ] + scopes: [ + azureMonitoring + ] + } +} + resource arohcpNodepoolSloErrorAlerts 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { name: 'arohcp_nodepool_slo_error_alerts' location: location diff --git a/dev-infrastructure/modules/metrics/rules/generatedRecordingRules.bicep b/dev-infrastructure/modules/metrics/rules/generatedRecordingRules.bicep index dfc1823f76c..03583f38862 100644 --- a/dev-infrastructure/modules/metrics/rules/generatedRecordingRules.bicep +++ b/dev-infrastructure/modules/metrics/rules/generatedRecordingRules.bicep @@ -27,3 +27,103 @@ resource arohcpAccessClusterSloRecordingRules 'Microsoft.AlertsManagement/promet ] } } + +resource userJourneyClusterUpgradeReachPartialRecordingRules 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'user-journey-cluster-upgrade-reach-partial-recording-rules' + location: location + properties: { + scopes: [ + azureMonitoring + ] + enabled: true + interval: 'PT1M' + rules: [ + { + record: 'hosted_control_plane_upgrade:version_state_first_seen:timestamp' + expression: 'min without (prometheus_replica) (min by (cluster, resource_id, subscription_id, cluster_uuid, version, state) (hosted_control_plane_upgrade:version_state_first_seen:timestamp or (timestamp(backend_cluster_version_info{state=~"desired|partial"} == 1) and on (cluster, resource_id) (count by (cluster, resource_id) (backend_cluster_version_info{state="completed"} == 1 unless on (cluster, resource_id, version) backend_cluster_version_info{state=~"desired|partial"} == 1) >= 1)) or timestamp(backend_cluster_version_info{state="completed"} == 1)))' + } + { + record: 'hosted_control_plane_upgrade:in_progress:count' + expression: 'count by (cluster) (count by (cluster, resource_id) (backend_cluster_version_info{state=~"desired|partial"} == 1) >= 1 and on (cluster, resource_id) (count by (cluster, resource_id) (backend_cluster_version_info{state="completed"} == 1 unless on (cluster, resource_id, version) backend_cluster_version_info{state=~"desired|partial"} == 1) >= 1)) or 0 * count by (cluster) (backend_cluster_version_info)' + } + { + record: 'hosted_control_plane_upgrade:duration_in_desired:seconds' + expression: '(time() - hosted_control_plane_upgrade:version_state_first_seen:timestamp{state="desired"}) and on (cluster, resource_id, subscription_id, cluster_uuid, version) (backend_cluster_version_info{state="desired"} == 1) and on (cluster, resource_id) (count by (cluster, resource_id) (backend_cluster_version_info{state="completed"} == 1 unless on (cluster, resource_id, version) backend_cluster_version_info{state=~"desired|partial"} == 1) >= 1)' + } + { + record: 'hosted_control_plane_upgrade:desired:stuck_over_20m:count' + expression: 'count by (cluster) (hosted_control_plane_upgrade:duration_in_desired:seconds > 1200) or 0 * hosted_control_plane_upgrade:in_progress:count' + } + { + record: 'hosted_control_plane_upgrade:desired:stuck_over_20m:ratio' + expression: 'hosted_control_plane_upgrade:desired:stuck_over_20m:count / clamp_min(hosted_control_plane_upgrade:in_progress:count, 1)' + } + { + record: 'hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate5m' + expression: 'avg_over_time(hosted_control_plane_upgrade:desired:stuck_over_20m:ratio[5m]) / (1 - 0.9995)' + } + { + record: 'hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate30m' + expression: 'avg_over_time(hosted_control_plane_upgrade:desired:stuck_over_20m:ratio[30m]) / (1 - 0.9995)' + } + { + record: 'hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate1h' + expression: 'avg_over_time(hosted_control_plane_upgrade:desired:stuck_over_20m:ratio[1h]) / (1 - 0.9995)' + } + { + record: 'hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate6h' + expression: 'avg_over_time(hosted_control_plane_upgrade:desired:stuck_over_20m:ratio[6h]) / (1 - 0.9995)' + } + { + record: 'hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate3d' + expression: 'avg_over_time(hosted_control_plane_upgrade:desired:stuck_over_20m:ratio[3d]) / (1 - 0.9995)' + } + ] + } +} + +resource userJourneyClusterUpgradeCompleteRecordingRules 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'user-journey-cluster-upgrade-complete-recording-rules' + location: location + properties: { + scopes: [ + azureMonitoring + ] + enabled: true + interval: 'PT1M' + rules: [ + { + record: 'hosted_control_plane_upgrade:duration_in_progress:seconds' + expression: '(time() - hosted_control_plane_upgrade:version_state_first_seen:timestamp{state="desired"}) unless on (cluster, resource_id, subscription_id, cluster_uuid, version) (backend_cluster_version_info{state="completed"} == 1) and on (cluster, resource_id, subscription_id, cluster_uuid, version) (backend_cluster_version_info{state=~"desired|partial"} == 1) and on (cluster, resource_id) (count by (cluster, resource_id) (backend_cluster_version_info{state="completed"} == 1 unless on (cluster, resource_id, version) backend_cluster_version_info{state=~"desired|partial"} == 1) >= 1)' + } + { + record: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:count' + expression: 'count by (cluster) (hosted_control_plane_upgrade:duration_in_progress:seconds > 1800) or 0 * hosted_control_plane_upgrade:in_progress:count' + } + { + record: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio' + expression: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:count / clamp_min(hosted_control_plane_upgrade:in_progress:count, 1)' + } + { + record: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate5m' + expression: 'avg_over_time(hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio[5m]) / (1 - 0.9995)' + } + { + record: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate30m' + expression: 'avg_over_time(hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio[30m]) / (1 - 0.9995)' + } + { + record: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate1h' + expression: 'avg_over_time(hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio[1h]) / (1 - 0.9995)' + } + { + record: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate6h' + expression: 'avg_over_time(hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio[6h]) / (1 - 0.9995)' + } + { + record: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate3d' + expression: 'avg_over_time(hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio[3d]) / (1 - 0.9995)' + } + ] + } +} diff --git a/observability/alerts-rp-services.yaml b/observability/alerts-rp-services.yaml index 514243ecdac..ea83912dfa5 100644 --- a/observability/alerts-rp-services.yaml +++ b/observability/alerts-rp-services.yaml @@ -1,6 +1,8 @@ prometheusRules: rulesFolders: - alerts/access-cluster-slo-prometheusRule.yaml + - alerts/userJourneyClusterUpgradeReachPartialMonitor-prometheusRule.yaml + - alerts/userJourneyClusterUpgradeCompleteMonitor-prometheusRule.yaml - alerts/nodepool-slo-prometheusRule.yaml untestedRules: [] testDependencies: diff --git a/observability/alerts/userJourneyClusterUpgradeCompleteMonitor-prometheusRule.yaml b/observability/alerts/userJourneyClusterUpgradeCompleteMonitor-prometheusRule.yaml new file mode 100644 index 00000000000..eaf69b8e1a0 --- /dev/null +++ b/observability/alerts/userJourneyClusterUpgradeCompleteMonitor-prometheusRule.yaml @@ -0,0 +1,82 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + app.kubernetes.io/name: kube-prometheus + app.kubernetes.io/part-of: kube-prometheus + prometheus: k8s + role: alert-rules + name: user-journey-cluster-upgrade-complete-alerts + namespace: monitoring +spec: + groups: + # ======================================== + # Cluster Upgrade Complete SLO Monitor Alerts + # ======================================== + # Multi-window burn-rate alerts. Sev 4 (info). + # + # In absence of https://issues.redhat.com/browse/ARO-22686 (reporting of control plane + # version history for versions lower than 4.22), cluster upgrade completeness is directly + # impacted by data plane state — e.g. no worker nodes means the upgrade will not reach + # completed; missing console component in external auth prevents CVO from marking the + # upgrade complete. Alerts may therefore fire due to missing data plane configuration + # rather than control plane failure. The runbook will be written to accommodate this. + - name: rp-user-journey-cluster-upgrade-complete-monitor-rules + rules: + - alert: userJourneyClusterUpgradeComplete1h5m + expr: | + hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate5m > 14.4 + and + hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate1h > 14.4 + for: 5m + labels: + severity: info + slo: cluster-upgrade-complete + short_window: 5m + long_window: 1h + annotations: + summary: "[HCPUpgradeCompleteBurn] {{ $labels.cluster }}" + description: | + More than 14.4x of the 99.95% cluster upgrade complete error budget is being consumed. Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. + Window: 1h/5m + Management Cluster: {{ $labels.cluster }} + runbook_url: https://aka.ms/arohcp-runbook-cluster-upgrade + correlationId: "userJourneyClusterUpgradeComplete/{{ $labels.cluster }}" + - alert: userJourneyClusterUpgradeComplete6h30m + expr: | + hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate30m > 6 + and + hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate6h > 6 + for: 30m + labels: + severity: info + slo: cluster-upgrade-complete + short_window: 30m + long_window: 6h + annotations: + summary: "[HCPUpgradeCompleteBurn] {{ $labels.cluster }}" + description: | + More than 6x of the 99.95% cluster upgrade complete error budget is being consumed. Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. + Window: 6h/30m + Management Cluster: {{ $labels.cluster }} + runbook_url: https://aka.ms/arohcp-runbook-cluster-upgrade + correlationId: "userJourneyClusterUpgradeComplete/{{ $labels.cluster }}" + - alert: userJourneyClusterUpgradeComplete3d6h + expr: | + hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate6h > 1 + and + hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate3d > 1 + for: 30m + labels: + severity: info + slo: cluster-upgrade-complete + short_window: 6h + long_window: 3d + annotations: + summary: "[HCPUpgradeCompleteBurn] {{ $labels.cluster }}" + description: | + Cluster upgrade complete has been steadily consuming the 99.95% SLO error budget (1x burn). Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. + Window: 3d/6h + Management Cluster: {{ $labels.cluster }} + runbook_url: https://aka.ms/arohcp-runbook-cluster-upgrade + correlationId: "userJourneyClusterUpgradeComplete/{{ $labels.cluster }}" diff --git a/observability/alerts/userJourneyClusterUpgradeCompleteMonitor-prometheusRule_test.yaml b/observability/alerts/userJourneyClusterUpgradeCompleteMonitor-prometheusRule_test.yaml new file mode 100644 index 00000000000..6c740d2dda8 --- /dev/null +++ b/observability/alerts/userJourneyClusterUpgradeCompleteMonitor-prometheusRule_test.yaml @@ -0,0 +1,772 @@ +rule_files: +- userJourneyClusterUpgradeRecord-prometheusRule.yaml +- userJourneyClusterUpgradeCompleteMonitor-prometheusRule.yaml +evaluation_interval: 1m +tests: +# ======================================== +# Test 1: Successful upgrade complete — no alert +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed"}' + values: "1+0x40" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="desired"}' + values: "1x10 0x25" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="completed"}' + values: "0x10 1x25" + alert_rule_test: + - eval_time: 40m + alertname: userJourneyClusterUpgradeComplete1h5m + - eval_time: 40m + alertname: userJourneyClusterUpgradeComplete6h30m + - eval_time: 40m + alertname: userJourneyClusterUpgradeComplete3d6h +# ======================================== +# Test 2: Fast burn — 2 of 10 in-flight upgrades stuck in progress >30m → 5m+1h alert fires +# a/b stuck from t=0; c–j join at minute 50 with <30m in progress at eval (healthy in-flight). +# Eval at 80m: ratio 0.2 stable since minute 50. burnrate1h omitted (1h window spans mixed fleet phases). +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/a", subscription_id="sub-3", cluster_uuid="uuid-a", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/a", subscription_id="sub-3", cluster_uuid="uuid-a", version="4.19.22", state="desired"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/b", subscription_id="sub-3", cluster_uuid="uuid-b", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/b", subscription_id="sub-3", cluster_uuid="uuid-b", version="4.19.22", state="desired"}' + values: "1x1 0x89" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/b", subscription_id="sub-3", cluster_uuid="uuid-b", version="4.19.22", state="partial"}' + values: "0x1 1x89" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/c", subscription_id="sub-3", cluster_uuid="uuid-c", version="4.19.19", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/c", subscription_id="sub-3", cluster_uuid="uuid-c", version="4.19.20", state="desired"}' + values: "0x50 1x40" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/d", subscription_id="sub-3", cluster_uuid="uuid-d", version="4.19.19", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/d", subscription_id="sub-3", cluster_uuid="uuid-d", version="4.19.20", state="desired"}' + values: "0x50 1x40" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/e", subscription_id="sub-3", cluster_uuid="uuid-e", version="4.19.19", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/e", subscription_id="sub-3", cluster_uuid="uuid-e", version="4.19.20", state="desired"}' + values: "0x50 1x40" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/f", subscription_id="sub-3", cluster_uuid="uuid-f", version="4.19.19", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/f", subscription_id="sub-3", cluster_uuid="uuid-f", version="4.19.20", state="desired"}' + values: "0x50 1x40" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/g", subscription_id="sub-3", cluster_uuid="uuid-g", version="4.19.19", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/g", subscription_id="sub-3", cluster_uuid="uuid-g", version="4.19.20", state="desired"}' + values: "0x50 1x40" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/h", subscription_id="sub-3", cluster_uuid="uuid-h", version="4.19.19", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/h", subscription_id="sub-3", cluster_uuid="uuid-h", version="4.19.20", state="desired"}' + values: "0x50 1x40" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/i", subscription_id="sub-3", cluster_uuid="uuid-i", version="4.19.19", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/i", subscription_id="sub-3", cluster_uuid="uuid-i", version="4.19.20", state="desired"}' + values: "0x50 1x40" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/j", subscription_id="sub-3", cluster_uuid="uuid-j", version="4.19.19", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/j", subscription_id="sub-3", cluster_uuid="uuid-j", version="4.19.20", state="desired"}' + values: "0x50 1x40" + promql_expr_test: + - expr: hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio{cluster="svc-1"} + eval_time: 80m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio{cluster="svc-1"}' + value: 0.2 + - expr: hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate5m{cluster="svc-1"} + eval_time: 80m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate5m{cluster="svc-1"}' + value: 4.0000000000004405e+02 + alert_rule_test: + - eval_time: 80m + alertname: userJourneyClusterUpgradeComplete1h5m + exp_alerts: + - exp_labels: + cluster: "svc-1" + severity: info + slo: cluster-upgrade-complete + short_window: 5m + long_window: 1h + exp_annotations: + summary: "[HCPUpgradeCompleteBurn] svc-1" + description: | + More than 14.4x of the 99.95% cluster upgrade complete error budget is being consumed. Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. + Window: 1h/5m + Management Cluster: svc-1 + runbook_url: "https://aka.ms/arohcp-runbook-cluster-upgrade" + correlationId: "userJourneyClusterUpgradeComplete/svc-1" + - eval_time: 80m + alertname: userJourneyClusterUpgradeComplete6h30m + exp_alerts: + - exp_labels: + cluster: "svc-1" + severity: info + slo: cluster-upgrade-complete + short_window: 30m + long_window: 6h + exp_annotations: + summary: "[HCPUpgradeCompleteBurn] svc-1" + description: | + More than 6x of the 99.95% cluster upgrade complete error budget is being consumed. Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. + Window: 6h/30m + Management Cluster: svc-1 + runbook_url: "https://aka.ms/arohcp-runbook-cluster-upgrade" + correlationId: "userJourneyClusterUpgradeComplete/svc-1" + - eval_time: 80m + alertname: userJourneyClusterUpgradeComplete3d6h + exp_alerts: + - exp_labels: + cluster: "svc-1" + severity: info + slo: cluster-upgrade-complete + short_window: 6h + long_window: 3d + exp_annotations: + summary: "[HCPUpgradeCompleteBurn] svc-1" + description: | + Cluster upgrade complete has been steadily consuming the 99.95% SLO error budget (1x burn). Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. + Window: 3d/6h + Management Cluster: svc-1 + runbook_url: "https://aka.ms/arohcp-runbook-cluster-upgrade" + correlationId: "userJourneyClusterUpgradeComplete/svc-1" +# ======================================== +# Test 3: Fleet healthy — no SLO alert +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/4/rg/a", subscription_id="sub-4", cluster_uuid="uuid-a", version="4.19.20", state="completed"}' + values: "1+0x40" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/4/rg/b", subscription_id="sub-4", cluster_uuid="uuid-b", version="4.19.20", state="completed"}' + values: "1+0x40" + promql_expr_test: + - expr: hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio{cluster="svc-1"} + eval_time: 10m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio{cluster="svc-1"}' + value: 0 + alert_rule_test: + - eval_time: 40m + alertname: userJourneyClusterUpgradeComplete1h5m + - eval_time: 40m + alertname: userJourneyClusterUpgradeComplete6h30m + - eval_time: 40m + alertname: userJourneyClusterUpgradeComplete3d6h +# ======================================== +# Test 4: Medium burn — 1 of 140 stuck >30m → 6h/30m alert fires, 1h/5m does not +# Fleet stable from t=0: 1 stuck in desired, 139 healthy in partial (not complete-SLO-stuck). +# Ratio 1/140 ≈ 0.71% from minute 31; eval at 90m satisfies 6h30m for:30m once burnrate6h > 6. +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/stuck", subscription_id="sub-5", cluster_uuid="uuid-stuck", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/stuck", subscription_id="sub-5", cluster_uuid="uuid-stuck", version="4.19.22", state="desired"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h000", subscription_id="sub-5", cluster_uuid="uuid-h000", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h000", subscription_id="sub-5", cluster_uuid="uuid-h000", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h001", subscription_id="sub-5", cluster_uuid="uuid-h001", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h001", subscription_id="sub-5", cluster_uuid="uuid-h001", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h002", subscription_id="sub-5", cluster_uuid="uuid-h002", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h002", subscription_id="sub-5", cluster_uuid="uuid-h002", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h003", subscription_id="sub-5", cluster_uuid="uuid-h003", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h003", subscription_id="sub-5", cluster_uuid="uuid-h003", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h004", subscription_id="sub-5", cluster_uuid="uuid-h004", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h004", subscription_id="sub-5", cluster_uuid="uuid-h004", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h005", subscription_id="sub-5", cluster_uuid="uuid-h005", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h005", subscription_id="sub-5", cluster_uuid="uuid-h005", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h006", subscription_id="sub-5", cluster_uuid="uuid-h006", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h006", subscription_id="sub-5", cluster_uuid="uuid-h006", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h007", subscription_id="sub-5", cluster_uuid="uuid-h007", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h007", subscription_id="sub-5", cluster_uuid="uuid-h007", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h008", subscription_id="sub-5", cluster_uuid="uuid-h008", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h008", subscription_id="sub-5", cluster_uuid="uuid-h008", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h009", subscription_id="sub-5", cluster_uuid="uuid-h009", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h009", subscription_id="sub-5", cluster_uuid="uuid-h009", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h010", subscription_id="sub-5", cluster_uuid="uuid-h010", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h010", subscription_id="sub-5", cluster_uuid="uuid-h010", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h011", subscription_id="sub-5", cluster_uuid="uuid-h011", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h011", subscription_id="sub-5", cluster_uuid="uuid-h011", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h012", subscription_id="sub-5", cluster_uuid="uuid-h012", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h012", subscription_id="sub-5", cluster_uuid="uuid-h012", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h013", subscription_id="sub-5", cluster_uuid="uuid-h013", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h013", subscription_id="sub-5", cluster_uuid="uuid-h013", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h014", subscription_id="sub-5", cluster_uuid="uuid-h014", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h014", subscription_id="sub-5", cluster_uuid="uuid-h014", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h015", subscription_id="sub-5", cluster_uuid="uuid-h015", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h015", subscription_id="sub-5", cluster_uuid="uuid-h015", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h016", subscription_id="sub-5", cluster_uuid="uuid-h016", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h016", subscription_id="sub-5", cluster_uuid="uuid-h016", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h017", subscription_id="sub-5", cluster_uuid="uuid-h017", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h017", subscription_id="sub-5", cluster_uuid="uuid-h017", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h018", subscription_id="sub-5", cluster_uuid="uuid-h018", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h018", subscription_id="sub-5", cluster_uuid="uuid-h018", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h019", subscription_id="sub-5", cluster_uuid="uuid-h019", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h019", subscription_id="sub-5", cluster_uuid="uuid-h019", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h020", subscription_id="sub-5", cluster_uuid="uuid-h020", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h020", subscription_id="sub-5", cluster_uuid="uuid-h020", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h021", subscription_id="sub-5", cluster_uuid="uuid-h021", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h021", subscription_id="sub-5", cluster_uuid="uuid-h021", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h022", subscription_id="sub-5", cluster_uuid="uuid-h022", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h022", subscription_id="sub-5", cluster_uuid="uuid-h022", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h023", subscription_id="sub-5", cluster_uuid="uuid-h023", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h023", subscription_id="sub-5", cluster_uuid="uuid-h023", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h024", subscription_id="sub-5", cluster_uuid="uuid-h024", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h024", subscription_id="sub-5", cluster_uuid="uuid-h024", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h025", subscription_id="sub-5", cluster_uuid="uuid-h025", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h025", subscription_id="sub-5", cluster_uuid="uuid-h025", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h026", subscription_id="sub-5", cluster_uuid="uuid-h026", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h026", subscription_id="sub-5", cluster_uuid="uuid-h026", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h027", subscription_id="sub-5", cluster_uuid="uuid-h027", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h027", subscription_id="sub-5", cluster_uuid="uuid-h027", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h028", subscription_id="sub-5", cluster_uuid="uuid-h028", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h028", subscription_id="sub-5", cluster_uuid="uuid-h028", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h029", subscription_id="sub-5", cluster_uuid="uuid-h029", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h029", subscription_id="sub-5", cluster_uuid="uuid-h029", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h030", subscription_id="sub-5", cluster_uuid="uuid-h030", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h030", subscription_id="sub-5", cluster_uuid="uuid-h030", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h031", subscription_id="sub-5", cluster_uuid="uuid-h031", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h031", subscription_id="sub-5", cluster_uuid="uuid-h031", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h032", subscription_id="sub-5", cluster_uuid="uuid-h032", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h032", subscription_id="sub-5", cluster_uuid="uuid-h032", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h033", subscription_id="sub-5", cluster_uuid="uuid-h033", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h033", subscription_id="sub-5", cluster_uuid="uuid-h033", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h034", subscription_id="sub-5", cluster_uuid="uuid-h034", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h034", subscription_id="sub-5", cluster_uuid="uuid-h034", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h035", subscription_id="sub-5", cluster_uuid="uuid-h035", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h035", subscription_id="sub-5", cluster_uuid="uuid-h035", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h036", subscription_id="sub-5", cluster_uuid="uuid-h036", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h036", subscription_id="sub-5", cluster_uuid="uuid-h036", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h037", subscription_id="sub-5", cluster_uuid="uuid-h037", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h037", subscription_id="sub-5", cluster_uuid="uuid-h037", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h038", subscription_id="sub-5", cluster_uuid="uuid-h038", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h038", subscription_id="sub-5", cluster_uuid="uuid-h038", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h039", subscription_id="sub-5", cluster_uuid="uuid-h039", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h039", subscription_id="sub-5", cluster_uuid="uuid-h039", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h040", subscription_id="sub-5", cluster_uuid="uuid-h040", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h040", subscription_id="sub-5", cluster_uuid="uuid-h040", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h041", subscription_id="sub-5", cluster_uuid="uuid-h041", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h041", subscription_id="sub-5", cluster_uuid="uuid-h041", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h042", subscription_id="sub-5", cluster_uuid="uuid-h042", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h042", subscription_id="sub-5", cluster_uuid="uuid-h042", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h043", subscription_id="sub-5", cluster_uuid="uuid-h043", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h043", subscription_id="sub-5", cluster_uuid="uuid-h043", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h044", subscription_id="sub-5", cluster_uuid="uuid-h044", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h044", subscription_id="sub-5", cluster_uuid="uuid-h044", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h045", subscription_id="sub-5", cluster_uuid="uuid-h045", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h045", subscription_id="sub-5", cluster_uuid="uuid-h045", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h046", subscription_id="sub-5", cluster_uuid="uuid-h046", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h046", subscription_id="sub-5", cluster_uuid="uuid-h046", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h047", subscription_id="sub-5", cluster_uuid="uuid-h047", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h047", subscription_id="sub-5", cluster_uuid="uuid-h047", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h048", subscription_id="sub-5", cluster_uuid="uuid-h048", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h048", subscription_id="sub-5", cluster_uuid="uuid-h048", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h049", subscription_id="sub-5", cluster_uuid="uuid-h049", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h049", subscription_id="sub-5", cluster_uuid="uuid-h049", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h050", subscription_id="sub-5", cluster_uuid="uuid-h050", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h050", subscription_id="sub-5", cluster_uuid="uuid-h050", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h051", subscription_id="sub-5", cluster_uuid="uuid-h051", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h051", subscription_id="sub-5", cluster_uuid="uuid-h051", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h052", subscription_id="sub-5", cluster_uuid="uuid-h052", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h052", subscription_id="sub-5", cluster_uuid="uuid-h052", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h053", subscription_id="sub-5", cluster_uuid="uuid-h053", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h053", subscription_id="sub-5", cluster_uuid="uuid-h053", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h054", subscription_id="sub-5", cluster_uuid="uuid-h054", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h054", subscription_id="sub-5", cluster_uuid="uuid-h054", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h055", subscription_id="sub-5", cluster_uuid="uuid-h055", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h055", subscription_id="sub-5", cluster_uuid="uuid-h055", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h056", subscription_id="sub-5", cluster_uuid="uuid-h056", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h056", subscription_id="sub-5", cluster_uuid="uuid-h056", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h057", subscription_id="sub-5", cluster_uuid="uuid-h057", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h057", subscription_id="sub-5", cluster_uuid="uuid-h057", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h058", subscription_id="sub-5", cluster_uuid="uuid-h058", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h058", subscription_id="sub-5", cluster_uuid="uuid-h058", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h059", subscription_id="sub-5", cluster_uuid="uuid-h059", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h059", subscription_id="sub-5", cluster_uuid="uuid-h059", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h060", subscription_id="sub-5", cluster_uuid="uuid-h060", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h060", subscription_id="sub-5", cluster_uuid="uuid-h060", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h061", subscription_id="sub-5", cluster_uuid="uuid-h061", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h061", subscription_id="sub-5", cluster_uuid="uuid-h061", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h062", subscription_id="sub-5", cluster_uuid="uuid-h062", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h062", subscription_id="sub-5", cluster_uuid="uuid-h062", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h063", subscription_id="sub-5", cluster_uuid="uuid-h063", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h063", subscription_id="sub-5", cluster_uuid="uuid-h063", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h064", subscription_id="sub-5", cluster_uuid="uuid-h064", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h064", subscription_id="sub-5", cluster_uuid="uuid-h064", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h065", subscription_id="sub-5", cluster_uuid="uuid-h065", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h065", subscription_id="sub-5", cluster_uuid="uuid-h065", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h066", subscription_id="sub-5", cluster_uuid="uuid-h066", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h066", subscription_id="sub-5", cluster_uuid="uuid-h066", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h067", subscription_id="sub-5", cluster_uuid="uuid-h067", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h067", subscription_id="sub-5", cluster_uuid="uuid-h067", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h068", subscription_id="sub-5", cluster_uuid="uuid-h068", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h068", subscription_id="sub-5", cluster_uuid="uuid-h068", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h069", subscription_id="sub-5", cluster_uuid="uuid-h069", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h069", subscription_id="sub-5", cluster_uuid="uuid-h069", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h070", subscription_id="sub-5", cluster_uuid="uuid-h070", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h070", subscription_id="sub-5", cluster_uuid="uuid-h070", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h071", subscription_id="sub-5", cluster_uuid="uuid-h071", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h071", subscription_id="sub-5", cluster_uuid="uuid-h071", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h072", subscription_id="sub-5", cluster_uuid="uuid-h072", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h072", subscription_id="sub-5", cluster_uuid="uuid-h072", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h073", subscription_id="sub-5", cluster_uuid="uuid-h073", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h073", subscription_id="sub-5", cluster_uuid="uuid-h073", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h074", subscription_id="sub-5", cluster_uuid="uuid-h074", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h074", subscription_id="sub-5", cluster_uuid="uuid-h074", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h075", subscription_id="sub-5", cluster_uuid="uuid-h075", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h075", subscription_id="sub-5", cluster_uuid="uuid-h075", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h076", subscription_id="sub-5", cluster_uuid="uuid-h076", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h076", subscription_id="sub-5", cluster_uuid="uuid-h076", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h077", subscription_id="sub-5", cluster_uuid="uuid-h077", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h077", subscription_id="sub-5", cluster_uuid="uuid-h077", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h078", subscription_id="sub-5", cluster_uuid="uuid-h078", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h078", subscription_id="sub-5", cluster_uuid="uuid-h078", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h079", subscription_id="sub-5", cluster_uuid="uuid-h079", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h079", subscription_id="sub-5", cluster_uuid="uuid-h079", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h080", subscription_id="sub-5", cluster_uuid="uuid-h080", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h080", subscription_id="sub-5", cluster_uuid="uuid-h080", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h081", subscription_id="sub-5", cluster_uuid="uuid-h081", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h081", subscription_id="sub-5", cluster_uuid="uuid-h081", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h082", subscription_id="sub-5", cluster_uuid="uuid-h082", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h082", subscription_id="sub-5", cluster_uuid="uuid-h082", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h083", subscription_id="sub-5", cluster_uuid="uuid-h083", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h083", subscription_id="sub-5", cluster_uuid="uuid-h083", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h084", subscription_id="sub-5", cluster_uuid="uuid-h084", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h084", subscription_id="sub-5", cluster_uuid="uuid-h084", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h085", subscription_id="sub-5", cluster_uuid="uuid-h085", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h085", subscription_id="sub-5", cluster_uuid="uuid-h085", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h086", subscription_id="sub-5", cluster_uuid="uuid-h086", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h086", subscription_id="sub-5", cluster_uuid="uuid-h086", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h087", subscription_id="sub-5", cluster_uuid="uuid-h087", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h087", subscription_id="sub-5", cluster_uuid="uuid-h087", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h088", subscription_id="sub-5", cluster_uuid="uuid-h088", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h088", subscription_id="sub-5", cluster_uuid="uuid-h088", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h089", subscription_id="sub-5", cluster_uuid="uuid-h089", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h089", subscription_id="sub-5", cluster_uuid="uuid-h089", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h090", subscription_id="sub-5", cluster_uuid="uuid-h090", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h090", subscription_id="sub-5", cluster_uuid="uuid-h090", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h091", subscription_id="sub-5", cluster_uuid="uuid-h091", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h091", subscription_id="sub-5", cluster_uuid="uuid-h091", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h092", subscription_id="sub-5", cluster_uuid="uuid-h092", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h092", subscription_id="sub-5", cluster_uuid="uuid-h092", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h093", subscription_id="sub-5", cluster_uuid="uuid-h093", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h093", subscription_id="sub-5", cluster_uuid="uuid-h093", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h094", subscription_id="sub-5", cluster_uuid="uuid-h094", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h094", subscription_id="sub-5", cluster_uuid="uuid-h094", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h095", subscription_id="sub-5", cluster_uuid="uuid-h095", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h095", subscription_id="sub-5", cluster_uuid="uuid-h095", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h096", subscription_id="sub-5", cluster_uuid="uuid-h096", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h096", subscription_id="sub-5", cluster_uuid="uuid-h096", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h097", subscription_id="sub-5", cluster_uuid="uuid-h097", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h097", subscription_id="sub-5", cluster_uuid="uuid-h097", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h098", subscription_id="sub-5", cluster_uuid="uuid-h098", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h098", subscription_id="sub-5", cluster_uuid="uuid-h098", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h099", subscription_id="sub-5", cluster_uuid="uuid-h099", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h099", subscription_id="sub-5", cluster_uuid="uuid-h099", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h100", subscription_id="sub-5", cluster_uuid="uuid-h100", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h100", subscription_id="sub-5", cluster_uuid="uuid-h100", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h101", subscription_id="sub-5", cluster_uuid="uuid-h101", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h101", subscription_id="sub-5", cluster_uuid="uuid-h101", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h102", subscription_id="sub-5", cluster_uuid="uuid-h102", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h102", subscription_id="sub-5", cluster_uuid="uuid-h102", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h103", subscription_id="sub-5", cluster_uuid="uuid-h103", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h103", subscription_id="sub-5", cluster_uuid="uuid-h103", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h104", subscription_id="sub-5", cluster_uuid="uuid-h104", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h104", subscription_id="sub-5", cluster_uuid="uuid-h104", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h105", subscription_id="sub-5", cluster_uuid="uuid-h105", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h105", subscription_id="sub-5", cluster_uuid="uuid-h105", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h106", subscription_id="sub-5", cluster_uuid="uuid-h106", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h106", subscription_id="sub-5", cluster_uuid="uuid-h106", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h107", subscription_id="sub-5", cluster_uuid="uuid-h107", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h107", subscription_id="sub-5", cluster_uuid="uuid-h107", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h108", subscription_id="sub-5", cluster_uuid="uuid-h108", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h108", subscription_id="sub-5", cluster_uuid="uuid-h108", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h109", subscription_id="sub-5", cluster_uuid="uuid-h109", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h109", subscription_id="sub-5", cluster_uuid="uuid-h109", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h110", subscription_id="sub-5", cluster_uuid="uuid-h110", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h110", subscription_id="sub-5", cluster_uuid="uuid-h110", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h111", subscription_id="sub-5", cluster_uuid="uuid-h111", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h111", subscription_id="sub-5", cluster_uuid="uuid-h111", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h112", subscription_id="sub-5", cluster_uuid="uuid-h112", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h112", subscription_id="sub-5", cluster_uuid="uuid-h112", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h113", subscription_id="sub-5", cluster_uuid="uuid-h113", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h113", subscription_id="sub-5", cluster_uuid="uuid-h113", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h114", subscription_id="sub-5", cluster_uuid="uuid-h114", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h114", subscription_id="sub-5", cluster_uuid="uuid-h114", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h115", subscription_id="sub-5", cluster_uuid="uuid-h115", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h115", subscription_id="sub-5", cluster_uuid="uuid-h115", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h116", subscription_id="sub-5", cluster_uuid="uuid-h116", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h116", subscription_id="sub-5", cluster_uuid="uuid-h116", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h117", subscription_id="sub-5", cluster_uuid="uuid-h117", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h117", subscription_id="sub-5", cluster_uuid="uuid-h117", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h118", subscription_id="sub-5", cluster_uuid="uuid-h118", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h118", subscription_id="sub-5", cluster_uuid="uuid-h118", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h119", subscription_id="sub-5", cluster_uuid="uuid-h119", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h119", subscription_id="sub-5", cluster_uuid="uuid-h119", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h120", subscription_id="sub-5", cluster_uuid="uuid-h120", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h120", subscription_id="sub-5", cluster_uuid="uuid-h120", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h121", subscription_id="sub-5", cluster_uuid="uuid-h121", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h121", subscription_id="sub-5", cluster_uuid="uuid-h121", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h122", subscription_id="sub-5", cluster_uuid="uuid-h122", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h122", subscription_id="sub-5", cluster_uuid="uuid-h122", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h123", subscription_id="sub-5", cluster_uuid="uuid-h123", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h123", subscription_id="sub-5", cluster_uuid="uuid-h123", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h124", subscription_id="sub-5", cluster_uuid="uuid-h124", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h124", subscription_id="sub-5", cluster_uuid="uuid-h124", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h125", subscription_id="sub-5", cluster_uuid="uuid-h125", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h125", subscription_id="sub-5", cluster_uuid="uuid-h125", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h126", subscription_id="sub-5", cluster_uuid="uuid-h126", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h126", subscription_id="sub-5", cluster_uuid="uuid-h126", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h127", subscription_id="sub-5", cluster_uuid="uuid-h127", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h127", subscription_id="sub-5", cluster_uuid="uuid-h127", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h128", subscription_id="sub-5", cluster_uuid="uuid-h128", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h128", subscription_id="sub-5", cluster_uuid="uuid-h128", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h129", subscription_id="sub-5", cluster_uuid="uuid-h129", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h129", subscription_id="sub-5", cluster_uuid="uuid-h129", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h130", subscription_id="sub-5", cluster_uuid="uuid-h130", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h130", subscription_id="sub-5", cluster_uuid="uuid-h130", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h131", subscription_id="sub-5", cluster_uuid="uuid-h131", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h131", subscription_id="sub-5", cluster_uuid="uuid-h131", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h132", subscription_id="sub-5", cluster_uuid="uuid-h132", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h132", subscription_id="sub-5", cluster_uuid="uuid-h132", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h133", subscription_id="sub-5", cluster_uuid="uuid-h133", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h133", subscription_id="sub-5", cluster_uuid="uuid-h133", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h134", subscription_id="sub-5", cluster_uuid="uuid-h134", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h134", subscription_id="sub-5", cluster_uuid="uuid-h134", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h135", subscription_id="sub-5", cluster_uuid="uuid-h135", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h135", subscription_id="sub-5", cluster_uuid="uuid-h135", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h136", subscription_id="sub-5", cluster_uuid="uuid-h136", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h136", subscription_id="sub-5", cluster_uuid="uuid-h136", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h137", subscription_id="sub-5", cluster_uuid="uuid-h137", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h137", subscription_id="sub-5", cluster_uuid="uuid-h137", version="4.19.22", state="partial"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h138", subscription_id="sub-5", cluster_uuid="uuid-h138", version="4.19.20", state="completed"}' + values: "1+0x100" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h138", subscription_id="sub-5", cluster_uuid="uuid-h138", version="4.19.22", state="partial"}' + values: "1+0x100" + promql_expr_test: + - expr: hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio{cluster="svc-1"} + eval_time: 90m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio{cluster="svc-1"}' + value: 0.007142857142857143 + - expr: hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate30m{cluster="svc-1"} + eval_time: 90m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate30m{cluster="svc-1"}' + value: 1.4285714285715859e+01 + alert_rule_test: + - eval_time: 90m + alertname: userJourneyClusterUpgradeComplete6h30m + exp_alerts: + - exp_labels: + cluster: "svc-1" + severity: info + slo: cluster-upgrade-complete + short_window: 30m + long_window: 6h + exp_annotations: + summary: "[HCPUpgradeCompleteBurn] svc-1" + description: | + More than 6x of the 99.95% cluster upgrade complete error budget is being consumed. Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. + Window: 6h/30m + Management Cluster: svc-1 + runbook_url: "https://aka.ms/arohcp-runbook-cluster-upgrade" + correlationId: "userJourneyClusterUpgradeComplete/svc-1" + - eval_time: 90m + alertname: userJourneyClusterUpgradeComplete1h5m + - eval_time: 90m + alertname: userJourneyClusterUpgradeComplete3d6h + exp_alerts: + - exp_labels: + cluster: "svc-1" + severity: info + slo: cluster-upgrade-complete + short_window: 6h + long_window: 3d + exp_annotations: + summary: "[HCPUpgradeCompleteBurn] svc-1" + description: | + Cluster upgrade complete has been steadily consuming the 99.95% SLO error budget (1x burn). Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. + Window: 3d/6h + Management Cluster: svc-1 + runbook_url: "https://aka.ms/arohcp-runbook-cluster-upgrade" + correlationId: "userJourneyClusterUpgradeComplete/svc-1" diff --git a/observability/alerts/userJourneyClusterUpgradeReachPartialMonitor-prometheusRule.yaml b/observability/alerts/userJourneyClusterUpgradeReachPartialMonitor-prometheusRule.yaml new file mode 100644 index 00000000000..ad44173759d --- /dev/null +++ b/observability/alerts/userJourneyClusterUpgradeReachPartialMonitor-prometheusRule.yaml @@ -0,0 +1,72 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + app.kubernetes.io/name: kube-prometheus + app.kubernetes.io/part-of: kube-prometheus + prometheus: k8s + role: alert-rules + name: user-journey-cluster-upgrade-reach-partial-alerts + namespace: monitoring +spec: + groups: + # Multi-window burn-rate alerts for cluster upgrade reach-partial SLO. Sev 4 (info). + - name: rp-user-journey-cluster-upgrade-reach-partial-monitor-rules + rules: + - alert: userJourneyClusterUpgradeReachPartial1h5m + expr: | + hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate5m > 14.4 + and + hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate1h > 14.4 + for: 5m + labels: + severity: info + slo: cluster-upgrade-reach-partial + short_window: 5m + long_window: 1h + annotations: + summary: "[HCPUpgradeReachPartialBurn] {{ $labels.cluster }}" + description: | + More than 14.4x of the 99.95% cluster upgrade reach-partial error budget is being consumed. Cluster upgrades are stuck for over 20 minutes without the target version reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. + Window: 1h/5m + Management Cluster: {{ $labels.cluster }} + runbook_url: https://aka.ms/arohcp-runbook-cluster-upgrade + correlationId: "userJourneyClusterUpgradeReachPartial/{{ $labels.cluster }}" + - alert: userJourneyClusterUpgradeReachPartial6h30m + expr: | + hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate30m > 6 + and + hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate6h > 6 + for: 30m + labels: + severity: info + slo: cluster-upgrade-reach-partial + short_window: 30m + long_window: 6h + annotations: + summary: "[HCPUpgradeReachPartialBurn] {{ $labels.cluster }}" + description: | + More than 6x of the 99.95% cluster upgrade reach-partial error budget is being consumed. Cluster upgrades are stuck for over 20 minutes without the target version reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. + Window: 6h/30m + Management Cluster: {{ $labels.cluster }} + runbook_url: https://aka.ms/arohcp-runbook-cluster-upgrade + correlationId: "userJourneyClusterUpgradeReachPartial/{{ $labels.cluster }}" + - alert: userJourneyClusterUpgradeReachPartial3d6h + expr: | + hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate6h > 1 + and + hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate3d > 1 + for: 30m + labels: + severity: info + slo: cluster-upgrade-reach-partial + short_window: 6h + long_window: 3d + annotations: + summary: "[HCPUpgradeReachPartialBurn] {{ $labels.cluster }}" + description: | + Cluster upgrade reach-partial has been steadily consuming the 99.95% SLO error budget (1x burn). Cluster upgrades are stuck for over 20 minutes without reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. + Window: 3d/6h + Management Cluster: {{ $labels.cluster }} + runbook_url: https://aka.ms/arohcp-runbook-cluster-upgrade + correlationId: "userJourneyClusterUpgradeReachPartial/{{ $labels.cluster }}" diff --git a/observability/alerts/userJourneyClusterUpgradeReachPartialMonitor-prometheusRule_test.yaml b/observability/alerts/userJourneyClusterUpgradeReachPartialMonitor-prometheusRule_test.yaml new file mode 100644 index 00000000000..c375eaf132a --- /dev/null +++ b/observability/alerts/userJourneyClusterUpgradeReachPartialMonitor-prometheusRule_test.yaml @@ -0,0 +1,778 @@ +rule_files: +- userJourneyClusterUpgradeRecord-prometheusRule.yaml +- userJourneyClusterUpgradeReachPartialMonitor-prometheusRule.yaml +evaluation_interval: 1m +tests: +# ======================================== +# Test 1: Successful reach partial — no alert +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed"}' + values: "1+0x40" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="desired"}' + values: "1x10 0x25" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="partial"}' + values: "0x10 1x25" + alert_rule_test: + - eval_time: 40m + alertname: userJourneyClusterUpgradeReachPartial1h5m + - eval_time: 40m + alertname: userJourneyClusterUpgradeReachPartial6h30m + - eval_time: 40m + alertname: userJourneyClusterUpgradeReachPartial3d6h +# ======================================== +# Test 2: Fast burn — 2 of 10 in-flight upgrades stuck in desired >20m → 5m+1h alert fires +# Eval at 80m: ratio has been 0.2 since minute 20 (all 10 in flight, 8 healthy in partial). +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/a", subscription_id="sub-3", cluster_uuid="uuid-a", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/a", subscription_id="sub-3", cluster_uuid="uuid-a", version="4.19.22", state="desired"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/b", subscription_id="sub-3", cluster_uuid="uuid-b", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/b", subscription_id="sub-3", cluster_uuid="uuid-b", version="4.19.22", state="desired"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/c", subscription_id="sub-3", cluster_uuid="uuid-c", version="4.19.19", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/c", subscription_id="sub-3", cluster_uuid="uuid-c", version="4.19.20", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/d", subscription_id="sub-3", cluster_uuid="uuid-d", version="4.19.19", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/d", subscription_id="sub-3", cluster_uuid="uuid-d", version="4.19.20", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/e", subscription_id="sub-3", cluster_uuid="uuid-e", version="4.19.19", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/e", subscription_id="sub-3", cluster_uuid="uuid-e", version="4.19.20", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/f", subscription_id="sub-3", cluster_uuid="uuid-f", version="4.19.19", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/f", subscription_id="sub-3", cluster_uuid="uuid-f", version="4.19.20", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/g", subscription_id="sub-3", cluster_uuid="uuid-g", version="4.19.19", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/g", subscription_id="sub-3", cluster_uuid="uuid-g", version="4.19.20", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/h", subscription_id="sub-3", cluster_uuid="uuid-h", version="4.19.19", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/h", subscription_id="sub-3", cluster_uuid="uuid-h", version="4.19.20", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/i", subscription_id="sub-3", cluster_uuid="uuid-i", version="4.19.19", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/i", subscription_id="sub-3", cluster_uuid="uuid-i", version="4.19.20", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/j", subscription_id="sub-3", cluster_uuid="uuid-j", version="4.19.19", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/j", subscription_id="sub-3", cluster_uuid="uuid-j", version="4.19.20", state="partial"}' + values: "1+0x90" + promql_expr_test: + - expr: hosted_control_plane_upgrade:desired:stuck_over_20m:ratio{cluster="svc-1"} + eval_time: 80m + exp_samples: + - labels: 'hosted_control_plane_upgrade:desired:stuck_over_20m:ratio{cluster="svc-1"}' + value: 0.2 + - expr: hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate5m{cluster="svc-1"} + eval_time: 80m + exp_samples: + - labels: 'hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate5m{cluster="svc-1"}' + value: 4.0000000000004405e+02 + - expr: hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate1h{cluster="svc-1"} + eval_time: 80m + exp_samples: + - labels: 'hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate1h{cluster="svc-1"}' + value: 4.0000000000004405e+02 + alert_rule_test: + - eval_time: 80m + alertname: userJourneyClusterUpgradeReachPartial1h5m + exp_alerts: + - exp_labels: + cluster: "svc-1" + severity: info + slo: cluster-upgrade-reach-partial + short_window: 5m + long_window: 1h + exp_annotations: + summary: "[HCPUpgradeReachPartialBurn] svc-1" + description: | + More than 14.4x of the 99.95% cluster upgrade reach-partial error budget is being consumed. Cluster upgrades are stuck for over 20 minutes without the target version reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. + Window: 1h/5m + Management Cluster: svc-1 + runbook_url: "https://aka.ms/arohcp-runbook-cluster-upgrade" + correlationId: "userJourneyClusterUpgradeReachPartial/svc-1" + - eval_time: 80m + alertname: userJourneyClusterUpgradeReachPartial6h30m + exp_alerts: + - exp_labels: + cluster: "svc-1" + severity: info + slo: cluster-upgrade-reach-partial + short_window: 30m + long_window: 6h + exp_annotations: + summary: "[HCPUpgradeReachPartialBurn] svc-1" + description: | + More than 6x of the 99.95% cluster upgrade reach-partial error budget is being consumed. Cluster upgrades are stuck for over 20 minutes without the target version reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. + Window: 6h/30m + Management Cluster: svc-1 + runbook_url: "https://aka.ms/arohcp-runbook-cluster-upgrade" + correlationId: "userJourneyClusterUpgradeReachPartial/svc-1" + - eval_time: 80m + alertname: userJourneyClusterUpgradeReachPartial3d6h + exp_alerts: + - exp_labels: + cluster: "svc-1" + severity: info + slo: cluster-upgrade-reach-partial + short_window: 6h + long_window: 3d + exp_annotations: + summary: "[HCPUpgradeReachPartialBurn] svc-1" + description: | + Cluster upgrade reach-partial has been steadily consuming the 99.95% SLO error budget (1x burn). Cluster upgrades are stuck for over 20 minutes without reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. + Window: 3d/6h + Management Cluster: svc-1 + runbook_url: "https://aka.ms/arohcp-runbook-cluster-upgrade" + correlationId: "userJourneyClusterUpgradeReachPartial/svc-1" +# ======================================== +# Test 3: Fleet healthy — no SLO alert +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/4/rg/a", subscription_id="sub-4", cluster_uuid="uuid-a", version="4.19.20", state="completed"}' + values: "1+0x40" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/4/rg/b", subscription_id="sub-4", cluster_uuid="uuid-b", version="4.19.20", state="completed"}' + values: "1+0x40" + promql_expr_test: + - expr: hosted_control_plane_upgrade:desired:stuck_over_20m:ratio{cluster="svc-1"} + eval_time: 10m + exp_samples: + - labels: 'hosted_control_plane_upgrade:desired:stuck_over_20m:ratio{cluster="svc-1"}' + value: 0 + alert_rule_test: + - eval_time: 40m + alertname: userJourneyClusterUpgradeReachPartial1h5m + - eval_time: 40m + alertname: userJourneyClusterUpgradeReachPartial6h30m + - eval_time: 40m + alertname: userJourneyClusterUpgradeReachPartial3d6h +# ======================================== +# Test 4: Medium burn — 1 of 140 stuck >20m → 6h/30m alert fires, 1h/5m does not +# Ratio 1/140 ≈ 0.71% → burn ≈ 14.3 (< 14.4 fast, > 6 medium). +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/stuck", subscription_id="sub-5", cluster_uuid="uuid-stuck", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/stuck", subscription_id="sub-5", cluster_uuid="uuid-stuck", version="4.19.22", state="desired"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h000", subscription_id="sub-5", cluster_uuid="uuid-h000", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h000", subscription_id="sub-5", cluster_uuid="uuid-h000", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h001", subscription_id="sub-5", cluster_uuid="uuid-h001", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h001", subscription_id="sub-5", cluster_uuid="uuid-h001", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h002", subscription_id="sub-5", cluster_uuid="uuid-h002", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h002", subscription_id="sub-5", cluster_uuid="uuid-h002", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h003", subscription_id="sub-5", cluster_uuid="uuid-h003", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h003", subscription_id="sub-5", cluster_uuid="uuid-h003", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h004", subscription_id="sub-5", cluster_uuid="uuid-h004", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h004", subscription_id="sub-5", cluster_uuid="uuid-h004", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h005", subscription_id="sub-5", cluster_uuid="uuid-h005", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h005", subscription_id="sub-5", cluster_uuid="uuid-h005", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h006", subscription_id="sub-5", cluster_uuid="uuid-h006", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h006", subscription_id="sub-5", cluster_uuid="uuid-h006", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h007", subscription_id="sub-5", cluster_uuid="uuid-h007", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h007", subscription_id="sub-5", cluster_uuid="uuid-h007", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h008", subscription_id="sub-5", cluster_uuid="uuid-h008", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h008", subscription_id="sub-5", cluster_uuid="uuid-h008", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h009", subscription_id="sub-5", cluster_uuid="uuid-h009", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h009", subscription_id="sub-5", cluster_uuid="uuid-h009", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h010", subscription_id="sub-5", cluster_uuid="uuid-h010", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h010", subscription_id="sub-5", cluster_uuid="uuid-h010", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h011", subscription_id="sub-5", cluster_uuid="uuid-h011", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h011", subscription_id="sub-5", cluster_uuid="uuid-h011", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h012", subscription_id="sub-5", cluster_uuid="uuid-h012", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h012", subscription_id="sub-5", cluster_uuid="uuid-h012", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h013", subscription_id="sub-5", cluster_uuid="uuid-h013", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h013", subscription_id="sub-5", cluster_uuid="uuid-h013", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h014", subscription_id="sub-5", cluster_uuid="uuid-h014", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h014", subscription_id="sub-5", cluster_uuid="uuid-h014", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h015", subscription_id="sub-5", cluster_uuid="uuid-h015", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h015", subscription_id="sub-5", cluster_uuid="uuid-h015", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h016", subscription_id="sub-5", cluster_uuid="uuid-h016", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h016", subscription_id="sub-5", cluster_uuid="uuid-h016", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h017", subscription_id="sub-5", cluster_uuid="uuid-h017", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h017", subscription_id="sub-5", cluster_uuid="uuid-h017", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h018", subscription_id="sub-5", cluster_uuid="uuid-h018", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h018", subscription_id="sub-5", cluster_uuid="uuid-h018", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h019", subscription_id="sub-5", cluster_uuid="uuid-h019", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h019", subscription_id="sub-5", cluster_uuid="uuid-h019", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h020", subscription_id="sub-5", cluster_uuid="uuid-h020", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h020", subscription_id="sub-5", cluster_uuid="uuid-h020", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h021", subscription_id="sub-5", cluster_uuid="uuid-h021", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h021", subscription_id="sub-5", cluster_uuid="uuid-h021", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h022", subscription_id="sub-5", cluster_uuid="uuid-h022", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h022", subscription_id="sub-5", cluster_uuid="uuid-h022", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h023", subscription_id="sub-5", cluster_uuid="uuid-h023", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h023", subscription_id="sub-5", cluster_uuid="uuid-h023", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h024", subscription_id="sub-5", cluster_uuid="uuid-h024", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h024", subscription_id="sub-5", cluster_uuid="uuid-h024", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h025", subscription_id="sub-5", cluster_uuid="uuid-h025", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h025", subscription_id="sub-5", cluster_uuid="uuid-h025", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h026", subscription_id="sub-5", cluster_uuid="uuid-h026", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h026", subscription_id="sub-5", cluster_uuid="uuid-h026", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h027", subscription_id="sub-5", cluster_uuid="uuid-h027", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h027", subscription_id="sub-5", cluster_uuid="uuid-h027", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h028", subscription_id="sub-5", cluster_uuid="uuid-h028", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h028", subscription_id="sub-5", cluster_uuid="uuid-h028", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h029", subscription_id="sub-5", cluster_uuid="uuid-h029", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h029", subscription_id="sub-5", cluster_uuid="uuid-h029", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h030", subscription_id="sub-5", cluster_uuid="uuid-h030", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h030", subscription_id="sub-5", cluster_uuid="uuid-h030", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h031", subscription_id="sub-5", cluster_uuid="uuid-h031", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h031", subscription_id="sub-5", cluster_uuid="uuid-h031", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h032", subscription_id="sub-5", cluster_uuid="uuid-h032", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h032", subscription_id="sub-5", cluster_uuid="uuid-h032", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h033", subscription_id="sub-5", cluster_uuid="uuid-h033", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h033", subscription_id="sub-5", cluster_uuid="uuid-h033", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h034", subscription_id="sub-5", cluster_uuid="uuid-h034", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h034", subscription_id="sub-5", cluster_uuid="uuid-h034", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h035", subscription_id="sub-5", cluster_uuid="uuid-h035", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h035", subscription_id="sub-5", cluster_uuid="uuid-h035", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h036", subscription_id="sub-5", cluster_uuid="uuid-h036", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h036", subscription_id="sub-5", cluster_uuid="uuid-h036", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h037", subscription_id="sub-5", cluster_uuid="uuid-h037", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h037", subscription_id="sub-5", cluster_uuid="uuid-h037", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h038", subscription_id="sub-5", cluster_uuid="uuid-h038", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h038", subscription_id="sub-5", cluster_uuid="uuid-h038", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h039", subscription_id="sub-5", cluster_uuid="uuid-h039", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h039", subscription_id="sub-5", cluster_uuid="uuid-h039", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h040", subscription_id="sub-5", cluster_uuid="uuid-h040", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h040", subscription_id="sub-5", cluster_uuid="uuid-h040", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h041", subscription_id="sub-5", cluster_uuid="uuid-h041", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h041", subscription_id="sub-5", cluster_uuid="uuid-h041", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h042", subscription_id="sub-5", cluster_uuid="uuid-h042", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h042", subscription_id="sub-5", cluster_uuid="uuid-h042", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h043", subscription_id="sub-5", cluster_uuid="uuid-h043", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h043", subscription_id="sub-5", cluster_uuid="uuid-h043", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h044", subscription_id="sub-5", cluster_uuid="uuid-h044", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h044", subscription_id="sub-5", cluster_uuid="uuid-h044", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h045", subscription_id="sub-5", cluster_uuid="uuid-h045", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h045", subscription_id="sub-5", cluster_uuid="uuid-h045", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h046", subscription_id="sub-5", cluster_uuid="uuid-h046", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h046", subscription_id="sub-5", cluster_uuid="uuid-h046", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h047", subscription_id="sub-5", cluster_uuid="uuid-h047", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h047", subscription_id="sub-5", cluster_uuid="uuid-h047", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h048", subscription_id="sub-5", cluster_uuid="uuid-h048", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h048", subscription_id="sub-5", cluster_uuid="uuid-h048", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h049", subscription_id="sub-5", cluster_uuid="uuid-h049", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h049", subscription_id="sub-5", cluster_uuid="uuid-h049", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h050", subscription_id="sub-5", cluster_uuid="uuid-h050", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h050", subscription_id="sub-5", cluster_uuid="uuid-h050", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h051", subscription_id="sub-5", cluster_uuid="uuid-h051", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h051", subscription_id="sub-5", cluster_uuid="uuid-h051", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h052", subscription_id="sub-5", cluster_uuid="uuid-h052", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h052", subscription_id="sub-5", cluster_uuid="uuid-h052", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h053", subscription_id="sub-5", cluster_uuid="uuid-h053", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h053", subscription_id="sub-5", cluster_uuid="uuid-h053", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h054", subscription_id="sub-5", cluster_uuid="uuid-h054", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h054", subscription_id="sub-5", cluster_uuid="uuid-h054", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h055", subscription_id="sub-5", cluster_uuid="uuid-h055", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h055", subscription_id="sub-5", cluster_uuid="uuid-h055", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h056", subscription_id="sub-5", cluster_uuid="uuid-h056", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h056", subscription_id="sub-5", cluster_uuid="uuid-h056", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h057", subscription_id="sub-5", cluster_uuid="uuid-h057", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h057", subscription_id="sub-5", cluster_uuid="uuid-h057", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h058", subscription_id="sub-5", cluster_uuid="uuid-h058", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h058", subscription_id="sub-5", cluster_uuid="uuid-h058", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h059", subscription_id="sub-5", cluster_uuid="uuid-h059", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h059", subscription_id="sub-5", cluster_uuid="uuid-h059", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h060", subscription_id="sub-5", cluster_uuid="uuid-h060", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h060", subscription_id="sub-5", cluster_uuid="uuid-h060", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h061", subscription_id="sub-5", cluster_uuid="uuid-h061", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h061", subscription_id="sub-5", cluster_uuid="uuid-h061", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h062", subscription_id="sub-5", cluster_uuid="uuid-h062", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h062", subscription_id="sub-5", cluster_uuid="uuid-h062", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h063", subscription_id="sub-5", cluster_uuid="uuid-h063", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h063", subscription_id="sub-5", cluster_uuid="uuid-h063", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h064", subscription_id="sub-5", cluster_uuid="uuid-h064", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h064", subscription_id="sub-5", cluster_uuid="uuid-h064", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h065", subscription_id="sub-5", cluster_uuid="uuid-h065", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h065", subscription_id="sub-5", cluster_uuid="uuid-h065", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h066", subscription_id="sub-5", cluster_uuid="uuid-h066", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h066", subscription_id="sub-5", cluster_uuid="uuid-h066", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h067", subscription_id="sub-5", cluster_uuid="uuid-h067", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h067", subscription_id="sub-5", cluster_uuid="uuid-h067", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h068", subscription_id="sub-5", cluster_uuid="uuid-h068", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h068", subscription_id="sub-5", cluster_uuid="uuid-h068", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h069", subscription_id="sub-5", cluster_uuid="uuid-h069", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h069", subscription_id="sub-5", cluster_uuid="uuid-h069", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h070", subscription_id="sub-5", cluster_uuid="uuid-h070", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h070", subscription_id="sub-5", cluster_uuid="uuid-h070", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h071", subscription_id="sub-5", cluster_uuid="uuid-h071", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h071", subscription_id="sub-5", cluster_uuid="uuid-h071", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h072", subscription_id="sub-5", cluster_uuid="uuid-h072", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h072", subscription_id="sub-5", cluster_uuid="uuid-h072", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h073", subscription_id="sub-5", cluster_uuid="uuid-h073", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h073", subscription_id="sub-5", cluster_uuid="uuid-h073", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h074", subscription_id="sub-5", cluster_uuid="uuid-h074", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h074", subscription_id="sub-5", cluster_uuid="uuid-h074", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h075", subscription_id="sub-5", cluster_uuid="uuid-h075", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h075", subscription_id="sub-5", cluster_uuid="uuid-h075", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h076", subscription_id="sub-5", cluster_uuid="uuid-h076", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h076", subscription_id="sub-5", cluster_uuid="uuid-h076", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h077", subscription_id="sub-5", cluster_uuid="uuid-h077", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h077", subscription_id="sub-5", cluster_uuid="uuid-h077", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h078", subscription_id="sub-5", cluster_uuid="uuid-h078", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h078", subscription_id="sub-5", cluster_uuid="uuid-h078", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h079", subscription_id="sub-5", cluster_uuid="uuid-h079", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h079", subscription_id="sub-5", cluster_uuid="uuid-h079", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h080", subscription_id="sub-5", cluster_uuid="uuid-h080", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h080", subscription_id="sub-5", cluster_uuid="uuid-h080", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h081", subscription_id="sub-5", cluster_uuid="uuid-h081", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h081", subscription_id="sub-5", cluster_uuid="uuid-h081", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h082", subscription_id="sub-5", cluster_uuid="uuid-h082", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h082", subscription_id="sub-5", cluster_uuid="uuid-h082", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h083", subscription_id="sub-5", cluster_uuid="uuid-h083", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h083", subscription_id="sub-5", cluster_uuid="uuid-h083", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h084", subscription_id="sub-5", cluster_uuid="uuid-h084", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h084", subscription_id="sub-5", cluster_uuid="uuid-h084", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h085", subscription_id="sub-5", cluster_uuid="uuid-h085", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h085", subscription_id="sub-5", cluster_uuid="uuid-h085", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h086", subscription_id="sub-5", cluster_uuid="uuid-h086", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h086", subscription_id="sub-5", cluster_uuid="uuid-h086", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h087", subscription_id="sub-5", cluster_uuid="uuid-h087", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h087", subscription_id="sub-5", cluster_uuid="uuid-h087", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h088", subscription_id="sub-5", cluster_uuid="uuid-h088", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h088", subscription_id="sub-5", cluster_uuid="uuid-h088", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h089", subscription_id="sub-5", cluster_uuid="uuid-h089", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h089", subscription_id="sub-5", cluster_uuid="uuid-h089", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h090", subscription_id="sub-5", cluster_uuid="uuid-h090", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h090", subscription_id="sub-5", cluster_uuid="uuid-h090", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h091", subscription_id="sub-5", cluster_uuid="uuid-h091", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h091", subscription_id="sub-5", cluster_uuid="uuid-h091", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h092", subscription_id="sub-5", cluster_uuid="uuid-h092", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h092", subscription_id="sub-5", cluster_uuid="uuid-h092", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h093", subscription_id="sub-5", cluster_uuid="uuid-h093", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h093", subscription_id="sub-5", cluster_uuid="uuid-h093", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h094", subscription_id="sub-5", cluster_uuid="uuid-h094", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h094", subscription_id="sub-5", cluster_uuid="uuid-h094", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h095", subscription_id="sub-5", cluster_uuid="uuid-h095", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h095", subscription_id="sub-5", cluster_uuid="uuid-h095", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h096", subscription_id="sub-5", cluster_uuid="uuid-h096", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h096", subscription_id="sub-5", cluster_uuid="uuid-h096", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h097", subscription_id="sub-5", cluster_uuid="uuid-h097", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h097", subscription_id="sub-5", cluster_uuid="uuid-h097", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h098", subscription_id="sub-5", cluster_uuid="uuid-h098", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h098", subscription_id="sub-5", cluster_uuid="uuid-h098", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h099", subscription_id="sub-5", cluster_uuid="uuid-h099", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h099", subscription_id="sub-5", cluster_uuid="uuid-h099", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h100", subscription_id="sub-5", cluster_uuid="uuid-h100", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h100", subscription_id="sub-5", cluster_uuid="uuid-h100", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h101", subscription_id="sub-5", cluster_uuid="uuid-h101", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h101", subscription_id="sub-5", cluster_uuid="uuid-h101", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h102", subscription_id="sub-5", cluster_uuid="uuid-h102", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h102", subscription_id="sub-5", cluster_uuid="uuid-h102", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h103", subscription_id="sub-5", cluster_uuid="uuid-h103", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h103", subscription_id="sub-5", cluster_uuid="uuid-h103", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h104", subscription_id="sub-5", cluster_uuid="uuid-h104", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h104", subscription_id="sub-5", cluster_uuid="uuid-h104", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h105", subscription_id="sub-5", cluster_uuid="uuid-h105", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h105", subscription_id="sub-5", cluster_uuid="uuid-h105", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h106", subscription_id="sub-5", cluster_uuid="uuid-h106", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h106", subscription_id="sub-5", cluster_uuid="uuid-h106", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h107", subscription_id="sub-5", cluster_uuid="uuid-h107", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h107", subscription_id="sub-5", cluster_uuid="uuid-h107", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h108", subscription_id="sub-5", cluster_uuid="uuid-h108", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h108", subscription_id="sub-5", cluster_uuid="uuid-h108", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h109", subscription_id="sub-5", cluster_uuid="uuid-h109", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h109", subscription_id="sub-5", cluster_uuid="uuid-h109", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h110", subscription_id="sub-5", cluster_uuid="uuid-h110", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h110", subscription_id="sub-5", cluster_uuid="uuid-h110", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h111", subscription_id="sub-5", cluster_uuid="uuid-h111", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h111", subscription_id="sub-5", cluster_uuid="uuid-h111", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h112", subscription_id="sub-5", cluster_uuid="uuid-h112", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h112", subscription_id="sub-5", cluster_uuid="uuid-h112", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h113", subscription_id="sub-5", cluster_uuid="uuid-h113", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h113", subscription_id="sub-5", cluster_uuid="uuid-h113", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h114", subscription_id="sub-5", cluster_uuid="uuid-h114", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h114", subscription_id="sub-5", cluster_uuid="uuid-h114", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h115", subscription_id="sub-5", cluster_uuid="uuid-h115", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h115", subscription_id="sub-5", cluster_uuid="uuid-h115", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h116", subscription_id="sub-5", cluster_uuid="uuid-h116", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h116", subscription_id="sub-5", cluster_uuid="uuid-h116", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h117", subscription_id="sub-5", cluster_uuid="uuid-h117", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h117", subscription_id="sub-5", cluster_uuid="uuid-h117", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h118", subscription_id="sub-5", cluster_uuid="uuid-h118", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h118", subscription_id="sub-5", cluster_uuid="uuid-h118", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h119", subscription_id="sub-5", cluster_uuid="uuid-h119", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h119", subscription_id="sub-5", cluster_uuid="uuid-h119", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h120", subscription_id="sub-5", cluster_uuid="uuid-h120", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h120", subscription_id="sub-5", cluster_uuid="uuid-h120", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h121", subscription_id="sub-5", cluster_uuid="uuid-h121", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h121", subscription_id="sub-5", cluster_uuid="uuid-h121", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h122", subscription_id="sub-5", cluster_uuid="uuid-h122", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h122", subscription_id="sub-5", cluster_uuid="uuid-h122", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h123", subscription_id="sub-5", cluster_uuid="uuid-h123", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h123", subscription_id="sub-5", cluster_uuid="uuid-h123", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h124", subscription_id="sub-5", cluster_uuid="uuid-h124", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h124", subscription_id="sub-5", cluster_uuid="uuid-h124", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h125", subscription_id="sub-5", cluster_uuid="uuid-h125", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h125", subscription_id="sub-5", cluster_uuid="uuid-h125", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h126", subscription_id="sub-5", cluster_uuid="uuid-h126", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h126", subscription_id="sub-5", cluster_uuid="uuid-h126", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h127", subscription_id="sub-5", cluster_uuid="uuid-h127", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h127", subscription_id="sub-5", cluster_uuid="uuid-h127", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h128", subscription_id="sub-5", cluster_uuid="uuid-h128", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h128", subscription_id="sub-5", cluster_uuid="uuid-h128", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h129", subscription_id="sub-5", cluster_uuid="uuid-h129", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h129", subscription_id="sub-5", cluster_uuid="uuid-h129", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h130", subscription_id="sub-5", cluster_uuid="uuid-h130", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h130", subscription_id="sub-5", cluster_uuid="uuid-h130", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h131", subscription_id="sub-5", cluster_uuid="uuid-h131", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h131", subscription_id="sub-5", cluster_uuid="uuid-h131", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h132", subscription_id="sub-5", cluster_uuid="uuid-h132", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h132", subscription_id="sub-5", cluster_uuid="uuid-h132", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h133", subscription_id="sub-5", cluster_uuid="uuid-h133", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h133", subscription_id="sub-5", cluster_uuid="uuid-h133", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h134", subscription_id="sub-5", cluster_uuid="uuid-h134", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h134", subscription_id="sub-5", cluster_uuid="uuid-h134", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h135", subscription_id="sub-5", cluster_uuid="uuid-h135", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h135", subscription_id="sub-5", cluster_uuid="uuid-h135", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h136", subscription_id="sub-5", cluster_uuid="uuid-h136", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h136", subscription_id="sub-5", cluster_uuid="uuid-h136", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h137", subscription_id="sub-5", cluster_uuid="uuid-h137", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h137", subscription_id="sub-5", cluster_uuid="uuid-h137", version="4.19.22", state="partial"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h138", subscription_id="sub-5", cluster_uuid="uuid-h138", version="4.19.20", state="completed"}' + values: "1+0x90" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h138", subscription_id="sub-5", cluster_uuid="uuid-h138", version="4.19.22", state="partial"}' + values: "1+0x90" + promql_expr_test: + - expr: hosted_control_plane_upgrade:desired:stuck_over_20m:ratio{cluster="svc-1"} + eval_time: 80m + exp_samples: + - labels: 'hosted_control_plane_upgrade:desired:stuck_over_20m:ratio{cluster="svc-1"}' + value: 0.007142857142857143 + - expr: hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate30m{cluster="svc-1"} + eval_time: 80m + exp_samples: + - labels: 'hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate30m{cluster="svc-1"}' + value: 1.4285714285715859e+01 + - expr: hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate5m{cluster="svc-1"} + eval_time: 80m + exp_samples: + - labels: 'hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate5m{cluster="svc-1"}' + value: 1.4285714285715859e+01 + alert_rule_test: + - eval_time: 80m + alertname: userJourneyClusterUpgradeReachPartial6h30m + exp_alerts: + - exp_labels: + cluster: "svc-1" + severity: info + slo: cluster-upgrade-reach-partial + short_window: 30m + long_window: 6h + exp_annotations: + summary: "[HCPUpgradeReachPartialBurn] svc-1" + description: | + More than 6x of the 99.95% cluster upgrade reach-partial error budget is being consumed. Cluster upgrades are stuck for over 20 minutes without the target version reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. + Window: 6h/30m + Management Cluster: svc-1 + runbook_url: "https://aka.ms/arohcp-runbook-cluster-upgrade" + correlationId: "userJourneyClusterUpgradeReachPartial/svc-1" + - eval_time: 80m + alertname: userJourneyClusterUpgradeReachPartial1h5m + - eval_time: 80m + alertname: userJourneyClusterUpgradeReachPartial3d6h + exp_alerts: + - exp_labels: + cluster: "svc-1" + severity: info + slo: cluster-upgrade-reach-partial + short_window: 6h + long_window: 3d + exp_annotations: + summary: "[HCPUpgradeReachPartialBurn] svc-1" + description: | + Cluster upgrade reach-partial has been steadily consuming the 99.95% SLO error budget (1x burn). Cluster upgrades are stuck for over 20 minutes without reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. + Window: 3d/6h + Management Cluster: svc-1 + runbook_url: "https://aka.ms/arohcp-runbook-cluster-upgrade" + correlationId: "userJourneyClusterUpgradeReachPartial/svc-1" diff --git a/observability/alerts/userJourneyClusterUpgradeRecord-prometheusRule.yaml b/observability/alerts/userJourneyClusterUpgradeRecord-prometheusRule.yaml new file mode 100644 index 00000000000..75976813306 --- /dev/null +++ b/observability/alerts/userJourneyClusterUpgradeRecord-prometheusRule.yaml @@ -0,0 +1,197 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + app.kubernetes.io/name: kube-prometheus + app.kubernetes.io/part-of: kube-prometheus + prometheus: k8s + role: recording-rules + name: user-journey-cluster-upgrade-recording-rules + namespace: monitoring +spec: + groups: + # ======================================== + # Cluster Upgrade — shared + reach-partial SLO (20m) + # ======================================== + # Tracks cluster upgrade version state transitions from backend_cluster_version_info + # (desired, partial, completed). + # + # SLO: 99.95% of cluster upgrades reach partial within 20 minutes of selecting the target version. + # Partial is the time for the target version to become active (recognized as active). It indicates + # the upgrade is in progress on the HCP cluster but not yet complete. + # Error budget: 0.05% (1 - 0.9995) + # + # Scope: upgrades only — initial version rollout is excluded; this user journey covers upgrades exclusively. + # Cluster create must succeed only after at least one CP version is installed on the hosted cluster + # (operation_cluster_create.go); that bounds when install ends and upgrades begin. + # https://github.com/Azure/ARO-HCP/blob/main/backend/pkg/controllers/operationcontrollers/operation_cluster_create.go#L316-L320 + # + # Upgrade-only guard: a completed CP version must exist that is not the same version as the current + # desired/partial target. During initial install the first version is desired/partial with no other + # completed version yet, so it is excluded; post-create upgrades have a prior completed version. + # + # Naming: level:metric:operations per https://prometheus.io/docs/practices/rules/#naming + - name: user-journey-cluster-upgrade-reach-partial-recording-rules + interval: 1m + rules: + # When each upgrade target version first entered desired, partial, or completed (start time for SLO durations). + # Upgrade-only: excludes initial install — see operation_cluster_create.go and upgrade-only guard above. + - record: hosted_control_plane_upgrade:version_state_first_seen:timestamp + expr: | + min without(prometheus_replica) ( + min by (cluster, resource_id, subscription_id, cluster_uuid, version, state) ( + hosted_control_plane_upgrade:version_state_first_seen:timestamp + or + ( + timestamp(backend_cluster_version_info{state=~"desired|partial"} == 1) + and on (cluster, resource_id) ( + count by (cluster, resource_id) ( + backend_cluster_version_info{state="completed"} == 1 + unless on (cluster, resource_id, version) + backend_cluster_version_info{state=~"desired|partial"} == 1 + ) >= 1 + ) + ) + or + timestamp(backend_cluster_version_info{state="completed"} == 1) + ) + ) + # Count of clusters with an upgrade currently in progress (desired or partial on a new target version). + - record: hosted_control_plane_upgrade:in_progress:count + expr: | + count by (cluster) ( + count by (cluster, resource_id) ( + backend_cluster_version_info{state=~"desired|partial"} == 1 + ) >= 1 + and on (cluster, resource_id) ( + count by (cluster, resource_id) ( + backend_cluster_version_info{state="completed"} == 1 + unless on (cluster, resource_id, version) + backend_cluster_version_info{state=~"desired|partial"} == 1 + ) >= 1 + ) + ) + or + 0 * count by (cluster) (backend_cluster_version_info) + # Seconds the target version has been in desired without reaching partial. Upgrade-only. + - record: hosted_control_plane_upgrade:duration_in_desired:seconds + expr: | + ( + time() - hosted_control_plane_upgrade:version_state_first_seen:timestamp{state="desired"} + ) + and on(cluster, resource_id, subscription_id, cluster_uuid, version) ( + backend_cluster_version_info{state="desired"} == 1 + ) + and on (cluster, resource_id) ( + count by (cluster, resource_id) ( + backend_cluster_version_info{state="completed"} == 1 + unless on (cluster, resource_id, version) + backend_cluster_version_info{state=~"desired|partial"} == 1 + ) >= 1 + ) + # Clusters whose target version has been in desired longer than 20 minutes (reach-partial SLO miss). + - record: hosted_control_plane_upgrade:desired:stuck_over_20m:count + expr: | + count by (cluster) (hosted_control_plane_upgrade:duration_in_desired:seconds > 1200) + or + 0 * hosted_control_plane_upgrade:in_progress:count + # Share of in-progress upgrades stuck in desired over 20 minutes (reach-partial error rate). + - record: hosted_control_plane_upgrade:desired:stuck_over_20m:ratio + expr: | + hosted_control_plane_upgrade:desired:stuck_over_20m:count + / + clamp_min(hosted_control_plane_upgrade:in_progress:count, 1) + # Reach-partial error-budget burn rate over 5 minutes (99.95% SLO). + - record: hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate5m + expr: | + avg_over_time(hosted_control_plane_upgrade:desired:stuck_over_20m:ratio[5m]) + / (1 - 0.9995) + # Reach-partial error-budget burn rate over 30 minutes. + - record: hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate30m + expr: | + avg_over_time(hosted_control_plane_upgrade:desired:stuck_over_20m:ratio[30m]) + / (1 - 0.9995) + # Reach-partial error-budget burn rate over 1 hour. + - record: hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate1h + expr: | + avg_over_time(hosted_control_plane_upgrade:desired:stuck_over_20m:ratio[1h]) + / (1 - 0.9995) + # Reach-partial error-budget burn rate over 6 hours. + - record: hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate6h + expr: | + avg_over_time(hosted_control_plane_upgrade:desired:stuck_over_20m:ratio[6h]) + / (1 - 0.9995) + # Reach-partial error-budget burn rate over 3 days. + - record: hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate3d + expr: | + avg_over_time(hosted_control_plane_upgrade:desired:stuck_over_20m:ratio[3d]) + / (1 - 0.9995) + # ======================================== + # Cluster Upgrade — complete SLO (30m) + # ======================================== + # SLO: 99.95% of cluster upgrades complete within 30 minutes of selecting the target version. + # Error budget: 0.05% (1 - 0.9995) + # + # Success: desired first-seen to completed first-seen <= 1800s. + # In-flight: desired or partial without completed for the target version. + # Upgrade-only scope: same guard as reach-partial (operation_cluster_create.go + different-version completed). + - name: user-journey-cluster-upgrade-complete-recording-rules + interval: 1m + rules: + # Seconds since the target version was selected until it completes; stops when that version reaches completed. + # Upgrade-only: excludes initial install — see operation_cluster_create.go and upgrade-only guard above. + - record: hosted_control_plane_upgrade:duration_in_progress:seconds + expr: | + ( + time() - hosted_control_plane_upgrade:version_state_first_seen:timestamp{state="desired"} + ) + unless on(cluster, resource_id, subscription_id, cluster_uuid, version) ( + backend_cluster_version_info{state="completed"} == 1 + ) + and on(cluster, resource_id, subscription_id, cluster_uuid, version) ( + backend_cluster_version_info{state=~"desired|partial"} == 1 + ) + and on (cluster, resource_id) ( + count by (cluster, resource_id) ( + backend_cluster_version_info{state="completed"} == 1 + unless on (cluster, resource_id, version) + backend_cluster_version_info{state=~"desired|partial"} == 1 + ) >= 1 + ) + # Clusters whose upgrade has been in progress longer than 30 minutes without completing (complete SLO miss). + - record: hosted_control_plane_upgrade:in_progress:stuck_over_30m:count + expr: | + count by (cluster) (hosted_control_plane_upgrade:duration_in_progress:seconds > 1800) + or + 0 * hosted_control_plane_upgrade:in_progress:count + # Share of in-progress upgrades stuck over 30 minutes without completing (complete error rate). + - record: hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio + expr: | + hosted_control_plane_upgrade:in_progress:stuck_over_30m:count + / + clamp_min(hosted_control_plane_upgrade:in_progress:count, 1) + # Complete error-budget burn rate over 5 minutes (99.95% SLO). + - record: hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate5m + expr: | + avg_over_time(hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio[5m]) + / (1 - 0.9995) + # Complete error-budget burn rate over 30 minutes. + - record: hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate30m + expr: | + avg_over_time(hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio[30m]) + / (1 - 0.9995) + # Complete error-budget burn rate over 1 hour. + - record: hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate1h + expr: | + avg_over_time(hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio[1h]) + / (1 - 0.9995) + # Complete error-budget burn rate over 6 hours. + - record: hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate6h + expr: | + avg_over_time(hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio[6h]) + / (1 - 0.9995) + # Complete error-budget burn rate over 3 days. + - record: hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate3d + expr: | + avg_over_time(hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio[3d]) + / (1 - 0.9995) diff --git a/observability/alerts/userJourneyClusterUpgradeRecord-prometheusRule_test.yaml b/observability/alerts/userJourneyClusterUpgradeRecord-prometheusRule_test.yaml new file mode 100644 index 00000000000..650feb7137c --- /dev/null +++ b/observability/alerts/userJourneyClusterUpgradeRecord-prometheusRule_test.yaml @@ -0,0 +1,135 @@ +rule_files: +- userJourneyClusterUpgradeRecord-prometheusRule.yaml +evaluation_interval: 1m +tests: +# ======================================== +# Test 1: Reach partial — desired transitions to partial within 20m (upgrade) +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed"}' + values: "1+0x35" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="desired"}' + values: "1x10 0x25" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="partial"}' + values: "0x10 1x25" + promql_expr_test: + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 9m + exp_samples: + - labels: 'hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", cluster_uuid="uuid-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", version="4.19.20", state="desired"}' + value: 540 + - expr: hosted_control_plane_upgrade:desired:stuck_over_20m:count{cluster="svc-1"} + eval_time: 9m + exp_samples: + - labels: 'hosted_control_plane_upgrade:desired:stuck_over_20m:count{cluster="svc-1"}' + value: 0 +# ======================================== +# Test 2: Stuck in desired — 1 of 1 active upgrades stuck >20m +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.20", state="completed"}' + values: "1+0x50" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.21", state="desired"}' + values: "1+0x50" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c3", subscription_id="sub-2", cluster_uuid="uuid-3", version="4.19.20", state="completed"}' + values: "1+0x50" + promql_expr_test: + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/2/rg/c2", version="4.19.21", state="desired"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", cluster_uuid="uuid-2", resource_id="/sub/2/rg/c2", subscription_id="sub-2", version="4.19.21", state="desired"}' + value: 2100 + - expr: hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"}' + value: 1 + - expr: hosted_control_plane_upgrade:desired:stuck_over_20m:count{cluster="svc-1"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:desired:stuck_over_20m:count{cluster="svc-1"}' + value: 1 + - expr: hosted_control_plane_upgrade:desired:stuck_over_20m:ratio{cluster="svc-1"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:desired:stuck_over_20m:ratio{cluster="svc-1"}' + value: 1 +# ======================================== +# Test 3: Install only — desired without completed is excluded from SLO +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/c4", subscription_id="sub-3", cluster_uuid="uuid-4", version="4.19.20", state="desired"}' + values: "1+0x50" + promql_expr_test: + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/3/rg/c4", version="4.19.20", state="desired"} + eval_time: 35m + exp_samples: [] + - expr: hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"}' + value: 0 + - expr: hosted_control_plane_upgrade:desired:stuck_over_20m:count{cluster="svc-1"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:desired:stuck_over_20m:count{cluster="svc-1"}' + value: 0 +# ======================================== +# Test 4: Complete — desired transitions to completed within 30m (upgrade) +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed"}' + values: "1+0x35" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="desired"}' + values: "1x10 0x25" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="completed"}' + values: "0x10 1x25" + promql_expr_test: + - expr: hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20", state="desired"} + eval_time: 9m + exp_samples: + - labels: 'hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", cluster_uuid="uuid-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", version="4.19.20", state="desired"}' + value: 540 + - expr: hosted_control_plane_upgrade:in_progress:stuck_over_30m:count{cluster="svc-1"} + eval_time: 9m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:count{cluster="svc-1"}' + value: 0 +# ======================================== +# Test 5: Stuck in progress — partial without completed after 35m +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.20", state="completed"}' + values: "1+0x50" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.21", state="desired"}' + values: "1x10 0x50" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.21", state="partial"}' + values: "0x10 1x50" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c3", subscription_id="sub-2", cluster_uuid="uuid-3", version="4.19.20", state="completed"}' + values: "1+0x50" + promql_expr_test: + - expr: hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", resource_id="/sub/2/rg/c2", version="4.19.21", state="desired"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", cluster_uuid="uuid-2", resource_id="/sub/2/rg/c2", subscription_id="sub-2", version="4.19.21", state="desired"}' + value: 2100 + - expr: hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"}' + value: 1 + - expr: hosted_control_plane_upgrade:in_progress:stuck_over_30m:count{cluster="svc-1"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:count{cluster="svc-1"}' + value: 1 + - expr: hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio{cluster="svc-1"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio{cluster="svc-1"}' + value: 1 diff --git a/observability/grafana-dashboards/sre/user-journey/cluster-upgrade-slo-dashboard.json b/observability/grafana-dashboards/sre/user-journey/cluster-upgrade-slo-dashboard.json new file mode 100644 index 00000000000..5acfd1b6ec9 --- /dev/null +++ b/observability/grafana-dashboards/sre/user-journey/cluster-upgrade-slo-dashboard.json @@ -0,0 +1,1490 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "### Cluster Upgrade SLOs\n\nFleet-wide observability for HCP cluster version upgrades from `backend_cluster_version_info`.\n\n| SLO | Target | Window |\n|-----|--------|--------|\n| **Reach partial** | 99.95% reach `partial` | 20 minutes |\n| **Upgrade complete** | 99.95% reach `completed` | 30 minutes |\n\n**Reach partial:** Time for the target version to become active (recognized as active). Indicates the upgrade is in progress on the HCP cluster but not yet complete.\n\n**Error budget:** 0.05% of in-flight upgrades (stuck / in-flight count).\n\n**Metrics:** `hosted_control_plane_upgrade:*` recording rules.\n\n**TSG:** [Backend TSG](https://eng.ms/docs/cloud-ai-platform/azure-core/azure-cloud-native-and-management-platform/control-plane-bburns/azure-red-hat-openshift/azure-redhat-openshift-team-doc/hcp/troubleshooting/backend-tsg.html)", + "mode": "markdown" + }, + "pluginVersion": "11.6.9", + "title": "Dashboard Info", + "type": "text" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 2, + "panels": [], + "title": "Reach Partial (20m SLO)", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "SLI: share of in-flight upgrades not contributing to stuck rate (1 − stuck/in-flight).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 3, + "mappings": [], + "max": 100, + "min": 99.9, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "yellow", + "value": 99.95 + }, + { + "color": "green", + "value": 99.99 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 6 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "(1 - (hosted_control_plane_upgrade:desired:stuck_over_20m:ratio{cluster=~\"$cluster\"} or vector(0))) * 100", + "instant": true, + "legendFormat": "SLI %", + "refId": "A" + } + ], + "title": "Cluster Upgrade Version Reach Partial SLI", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "HCP clusters with any reported cluster version state.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 6 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "none", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(hosted_control_plane_upgrade:in_progress:count{cluster=~\"$cluster\"}) or vector(0)", + "instant": true, + "legendFormat": "In-flight upgrades", + "refId": "A" + } + ], + "title": "In-Flight Upgrades", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Upgrades stuck in desired for over 20 minutes without activating.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 6 + }, + "id": 5, + "options": { + "colorMode": "background", + "graphMode": "none", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(hosted_control_plane_upgrade:desired:stuck_over_20m:count{cluster=~\"$cluster\"}) or vector(0)", + "instant": true, + "legendFormat": "Stuck clusters", + "refId": "A" + } + ], + "title": "Stuck in Desired Over 20m", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Current 5 minute error budget burn rate. Fast burn alert threshold: 14.4x.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 6 + }, + { + "color": "red", + "value": 14.4 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 6 + }, + "id": 6, + "options": { + "colorMode": "background", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "max(hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate5m{cluster=~\"$cluster\"}) or vector(0)", + "instant": true, + "legendFormat": "5m burn", + "refId": "A" + } + ], + "title": "Burn Rate (5m)", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Fraction of in-flight upgrades stuck past 20 minutes. SLO error budget line at 0.05%.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "thresholdsStyle": { + "mode": "line" + } + }, + "decimals": 4, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0.0005 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "hosted_control_plane_upgrade:desired:stuck_over_20m:ratio{cluster=~\"$cluster\"}", + "legendFormat": "{{ cluster }} in-flight stuck rate", + "refId": "A" + } + ], + "title": "In-Flight Stuck Rate Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Multi-window error budget burn rates for cluster upgrade reach-partial SLO.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate5m{cluster=~\"$cluster\"}", + "legendFormat": "5m {{ cluster }}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate30m{cluster=~\"$cluster\"}", + "legendFormat": "30m {{ cluster }}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate1h{cluster=~\"$cluster\"}", + "legendFormat": "1h {{ cluster }}", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate6h{cluster=~\"$cluster\"}", + "legendFormat": "6h {{ cluster }}", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate3d{cluster=~\"$cluster\"}", + "legendFormat": "3d {{ cluster }}", + "refId": "E" + } + ], + "title": "Error Budget Burn Rates", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Time from desired first-seen to first active (partial or completed) observation, in minutes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 20 + } + ] + }, + "unit": "m" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "(\n (\n min without(state) (hosted_control_plane_upgrade:version_state_first_seen:timestamp{cluster=~\"$cluster\", state=\"partial\"})\n - on(cluster, resource_id, subscription_id, cluster_uuid, version) group_left()\n hosted_control_plane_upgrade:version_state_first_seen:timestamp{cluster=~\"$cluster\", state=\"desired\"}\n )\n and on(cluster, resource_id, subscription_id, cluster_uuid, version) (\n max without(state) (backend_cluster_version_info{cluster=~\"$cluster\", state=\"partial\"}) == 1\n )\n) / 60", + "legendFormat": "{{ resource_id }} {{ version }}", + "refId": "A" + } + ], + "title": "Cluster Upgrade Version Reach Partial Duration (Minutes)", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Clusters currently waiting in desired state (minutes since target version was selected).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 20 + } + ] + }, + "unit": "m" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "hosted_control_plane_upgrade:duration_in_desired:seconds{cluster=~\"$cluster\"} / 60", + "legendFormat": "{{ resource_id }} {{ version }}", + "refId": "A" + } + ], + "title": "Time in Desired (Minutes)", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Clusters with cluster upgrades currently stuck past the 20 minute reach-partial window.", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 11, + "options": { + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "version" + } + ] + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "(\n hosted_control_plane_upgrade:duration_in_desired:seconds{cluster=~\"$cluster\"}\n and\n hosted_control_plane_upgrade:duration_in_desired:seconds{cluster=~\"$cluster\"} > 1200\n) / 60", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Stuck in Desired Detail", + "transformations": [ + { + "id": "seriesToRows", + "options": {} + }, + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "state": true + }, + "renameByName": { + "Value": "Time in Desired (Minutes)", + "cluster": "Service Cluster", + "cluster_uuid": "Cluster UUID", + "resource_id": "Resource ID", + "subscription_id": "Subscription", + "version": "Target Version" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 12, + "panels": [], + "title": "Upgrade Complete (30m SLO)", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "SLI: share of in-flight upgrades not contributing to stuck rate (1 − stuck/in-flight).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 3, + "mappings": [], + "max": 100, + "min": 99.9, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "yellow", + "value": 99.95 + }, + { + "color": "green", + "value": 99.99 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 37 + }, + "id": 13, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "(1 - (hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio{cluster=~\"$cluster\"} or vector(0))) * 100", + "instant": true, + "legendFormat": "SLI %", + "refId": "A" + } + ], + "title": "Cluster Upgrade Complete SLI", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "HCP clusters with any reported cluster version state.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 37 + }, + "id": 14, + "options": { + "colorMode": "background", + "graphMode": "none", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(hosted_control_plane_upgrade:in_progress:count{cluster=~\"$cluster\"}) or vector(0)", + "instant": true, + "legendFormat": "In-flight upgrades", + "refId": "A" + } + ], + "title": "In-Flight Upgrades", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Upgrades stuck in desired or partial for over 30 minutes without reaching completed.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 37 + }, + "id": 15, + "options": { + "colorMode": "background", + "graphMode": "none", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(hosted_control_plane_upgrade:in_progress:stuck_over_30m:count{cluster=~\"$cluster\"}) or vector(0)", + "instant": true, + "legendFormat": "Stuck clusters", + "refId": "A" + } + ], + "title": "Stuck in Progress Over 30m", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Current 5 minute error budget burn rate. Fast burn alert threshold: 14.4x.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 6 + }, + { + "color": "red", + "value": 14.4 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 37 + }, + "id": 16, + "options": { + "colorMode": "background", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "max(hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate5m{cluster=~\"$cluster\"}) or vector(0)", + "instant": true, + "legendFormat": "5m burn", + "refId": "A" + } + ], + "title": "Burn Rate (5m)", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Fraction of in-flight upgrades stuck past 30 minutes. SLO error budget line at 0.05%.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "thresholdsStyle": { + "mode": "line" + } + }, + "decimals": 4, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0.0005 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio{cluster=~\"$cluster\"}", + "legendFormat": "{{ cluster }} in-flight stuck rate", + "refId": "A" + } + ], + "title": "In-Flight Stuck Rate Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Multi-window error budget burn rates for cluster upgrade complete SLO.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 18, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate5m{cluster=~\"$cluster\"}", + "legendFormat": "5m {{ cluster }}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate30m{cluster=~\"$cluster\"}", + "legendFormat": "30m {{ cluster }}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate1h{cluster=~\"$cluster\"}", + "legendFormat": "1h {{ cluster }}", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate6h{cluster=~\"$cluster\"}", + "legendFormat": "6h {{ cluster }}", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate3d{cluster=~\"$cluster\"}", + "legendFormat": "3d {{ cluster }}", + "refId": "E" + } + ], + "title": "Error Budget Burn Rates", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Time from desired first-seen to first active (partial or completed) observation, in minutes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 30 + } + ] + }, + "unit": "m" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 19, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "(\n (\n min without(state) (hosted_control_plane_upgrade:version_state_first_seen:timestamp{cluster=~\"$cluster\", state=\"completed\"})\n - on(cluster, resource_id, subscription_id, cluster_uuid, version) group_left()\n hosted_control_plane_upgrade:version_state_first_seen:timestamp{cluster=~\"$cluster\", state=\"desired\"}\n )\n and on(cluster, resource_id, subscription_id, cluster_uuid, version) (\n max without(state) (backend_cluster_version_info{cluster=~\"$cluster\", state=\"completed\"}) == 1\n )\n) / 60", + "legendFormat": "{{ resource_id }} {{ version }}", + "refId": "A" + } + ], + "title": "Cluster Upgrade Complete Duration (Minutes)", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Clusters waiting for upgrade completion (minutes since target version was selected).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 30 + } + ] + }, + "unit": "m" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 20, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "hosted_control_plane_upgrade:duration_in_progress:seconds{cluster=~\"$cluster\"} / 60", + "legendFormat": "{{ resource_id }} {{ version }}", + "refId": "A" + } + ], + "title": "Time in Progress (Minutes)", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Clusters with cluster upgrades currently stuck past the 30 minute upgrade-complete window.", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 21, + "options": { + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "version" + } + ] + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "(\n hosted_control_plane_upgrade:duration_in_progress:seconds{cluster=~\"$cluster\"}\n and\n hosted_control_plane_upgrade:duration_in_progress:seconds{cluster=~\"$cluster\"} > 1800\n) / 60", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Stuck in Progress Detail", + "transformations": [ + { + "id": "seriesToRows", + "options": {} + }, + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "state": true + }, + "renameByName": { + "Value": "Time in Progress (Minutes)", + "cluster": "Service Cluster", + "cluster_uuid": "Cluster UUID", + "resource_id": "Resource ID", + "subscription_id": "Subscription", + "version": "Target Version" + } + } + } + ], + "type": "table" + } + ], + "schemaVersion": 39, + "tags": [ + "cluster-upgrade", + "aro-hcp", + "slo", + "user-journey" + ], + "templating": { + "list": [ + { + "allowCustomValue": false, + "current": { + "text": "All", + "value": "$__all" + }, + "includeAll": true, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "^Managed_Prometheus_services-.*$", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "label_values(backend_cluster_version_info, cluster)", + "includeAll": true, + "multi": true, + "name": "cluster", + "query": "label_values(backend_cluster_version_info, cluster)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Cluster Upgrade SLO", + "uid": "cluster-upgrade-slo", + "version": 1 +} diff --git a/observability/recording-rules-services.yaml b/observability/recording-rules-services.yaml index 183d0e1fb97..16e83b998eb 100644 --- a/observability/recording-rules-services.yaml +++ b/observability/recording-rules-services.yaml @@ -1,5 +1,6 @@ prometheusRules: rulesFolders: - recording-rules/access-cluster-slo-recordingRule.yaml + - alerts/userJourneyClusterUpgradeRecord-prometheusRule.yaml untestedRules: [] outputBicep: ../dev-infrastructure/modules/metrics/rules/generatedRecordingRules.bicep From b11562918df26bc2e8b211434f46fdb0cc5a8f6f Mon Sep 17 00:00:00 2001 From: Manyanda Chitimbo Date: Mon, 6 Jul 2026 11:24:19 +0200 Subject: [PATCH 2/4] feat: add a recording rule about cluster upgrade eligibility Only consider clusters that have 2 distict version labels to effectively ignore clusters that are under install. --- .../rules/generatedRecordingRules.bicep | 30 +++++-- ...eyClusterUpgradeRecord-prometheusRule.yaml | 80 ++++++++++--------- ...sterUpgradeRecord-prometheusRule_test.yaml | 45 +++++++++++ 3 files changed, 110 insertions(+), 45 deletions(-) diff --git a/dev-infrastructure/modules/metrics/rules/generatedRecordingRules.bicep b/dev-infrastructure/modules/metrics/rules/generatedRecordingRules.bicep index 03583f38862..d92176532da 100644 --- a/dev-infrastructure/modules/metrics/rules/generatedRecordingRules.bicep +++ b/dev-infrastructure/modules/metrics/rules/generatedRecordingRules.bicep @@ -28,8 +28,8 @@ resource arohcpAccessClusterSloRecordingRules 'Microsoft.AlertsManagement/promet } } -resource userJourneyClusterUpgradeReachPartialRecordingRules 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { - name: 'user-journey-cluster-upgrade-reach-partial-recording-rules' +resource userJourneyClusterUpgradeSharedRecordingRules 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'user-journey-cluster-upgrade-shared-recording-rules' location: location properties: { scopes: [ @@ -38,17 +38,35 @@ resource userJourneyClusterUpgradeReachPartialRecordingRules 'Microsoft.AlertsMa enabled: true interval: 'PT1M' rules: [ + { + record: 'hosted_control_plane_upgrade:upgrade_eligible:info' + expression: '((count by (cluster, resource_id, subscription_id, cluster_uuid) (count by (cluster, resource_id, subscription_id, cluster_uuid, version) (backend_cluster_version_info == 1)) >= 2) and on (cluster, resource_id) (count by (cluster, resource_id) (backend_cluster_version_info{state="completed"} == 1) >= 1)) * 0 + 1' + } { record: 'hosted_control_plane_upgrade:version_state_first_seen:timestamp' - expression: 'min without (prometheus_replica) (min by (cluster, resource_id, subscription_id, cluster_uuid, version, state) (hosted_control_plane_upgrade:version_state_first_seen:timestamp or (timestamp(backend_cluster_version_info{state=~"desired|partial"} == 1) and on (cluster, resource_id) (count by (cluster, resource_id) (backend_cluster_version_info{state="completed"} == 1 unless on (cluster, resource_id, version) backend_cluster_version_info{state=~"desired|partial"} == 1) >= 1)) or timestamp(backend_cluster_version_info{state="completed"} == 1)))' + expression: 'min without (prometheus_replica) (min by (cluster, resource_id, subscription_id, cluster_uuid, version, state) (hosted_control_plane_upgrade:version_state_first_seen:timestamp or (timestamp(backend_cluster_version_info{state=~"desired|partial"} == 1) and on (cluster, resource_id) (hosted_control_plane_upgrade:upgrade_eligible:info == 1)) or timestamp(backend_cluster_version_info{state="completed"} == 1)))' } { record: 'hosted_control_plane_upgrade:in_progress:count' - expression: 'count by (cluster) (count by (cluster, resource_id) (backend_cluster_version_info{state=~"desired|partial"} == 1) >= 1 and on (cluster, resource_id) (count by (cluster, resource_id) (backend_cluster_version_info{state="completed"} == 1 unless on (cluster, resource_id, version) backend_cluster_version_info{state=~"desired|partial"} == 1) >= 1)) or 0 * count by (cluster) (backend_cluster_version_info)' + expression: 'count by (cluster) (count by (cluster, resource_id) (backend_cluster_version_info{state=~"desired|partial"} == 1) >= 1 and on (cluster, resource_id) (hosted_control_plane_upgrade:upgrade_eligible:info == 1)) or 0 * count by (cluster) (backend_cluster_version_info)' } + ] + } +} + +resource userJourneyClusterUpgradeReachPartialRecordingRules 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'user-journey-cluster-upgrade-reach-partial-recording-rules' + location: location + properties: { + scopes: [ + azureMonitoring + ] + enabled: true + interval: 'PT1M' + rules: [ { record: 'hosted_control_plane_upgrade:duration_in_desired:seconds' - expression: '(time() - hosted_control_plane_upgrade:version_state_first_seen:timestamp{state="desired"}) and on (cluster, resource_id, subscription_id, cluster_uuid, version) (backend_cluster_version_info{state="desired"} == 1) and on (cluster, resource_id) (count by (cluster, resource_id) (backend_cluster_version_info{state="completed"} == 1 unless on (cluster, resource_id, version) backend_cluster_version_info{state=~"desired|partial"} == 1) >= 1)' + expression: '(time() - hosted_control_plane_upgrade:version_state_first_seen:timestamp{state="desired"}) and on (cluster, resource_id, subscription_id, cluster_uuid, version) (backend_cluster_version_info{state="desired"} == 1) and on (cluster, resource_id) (hosted_control_plane_upgrade:upgrade_eligible:info == 1)' } { record: 'hosted_control_plane_upgrade:desired:stuck_over_20m:count' @@ -94,7 +112,7 @@ resource userJourneyClusterUpgradeCompleteRecordingRules 'Microsoft.AlertsManage rules: [ { record: 'hosted_control_plane_upgrade:duration_in_progress:seconds' - expression: '(time() - hosted_control_plane_upgrade:version_state_first_seen:timestamp{state="desired"}) unless on (cluster, resource_id, subscription_id, cluster_uuid, version) (backend_cluster_version_info{state="completed"} == 1) and on (cluster, resource_id, subscription_id, cluster_uuid, version) (backend_cluster_version_info{state=~"desired|partial"} == 1) and on (cluster, resource_id) (count by (cluster, resource_id) (backend_cluster_version_info{state="completed"} == 1 unless on (cluster, resource_id, version) backend_cluster_version_info{state=~"desired|partial"} == 1) >= 1)' + expression: '(time() - hosted_control_plane_upgrade:version_state_first_seen:timestamp{state="desired"}) unless on (cluster, resource_id, subscription_id, cluster_uuid, version) (backend_cluster_version_info{state="completed"} == 1) and on (cluster, resource_id, subscription_id, cluster_uuid, version) (backend_cluster_version_info{state=~"desired|partial"} == 1) and on (cluster, resource_id) (hosted_control_plane_upgrade:upgrade_eligible:info == 1)' } { record: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:count' diff --git a/observability/alerts/userJourneyClusterUpgradeRecord-prometheusRule.yaml b/observability/alerts/userJourneyClusterUpgradeRecord-prometheusRule.yaml index 75976813306..f8259674e7f 100644 --- a/observability/alerts/userJourneyClusterUpgradeRecord-prometheusRule.yaml +++ b/observability/alerts/userJourneyClusterUpgradeRecord-prometheusRule.yaml @@ -11,31 +11,37 @@ metadata: spec: groups: # ======================================== - # Cluster Upgrade — shared + reach-partial SLO (20m) + # Cluster Upgrade — shared recording rules # ======================================== - # Tracks cluster upgrade version state transitions from backend_cluster_version_info - # (desired, partial, completed). + # Shared eligibility, first-seen timestamps, and in-progress fleet counts for both upgrade SLOs. # - # SLO: 99.95% of cluster upgrades reach partial within 20 minutes of selecting the target version. - # Partial is the time for the target version to become active (recognized as active). It indicates - # the upgrade is in progress on the HCP cluster but not yet complete. - # Error budget: 0.05% (1 - 0.9995) - # - # Scope: upgrades only — initial version rollout is excluded; this user journey covers upgrades exclusively. - # Cluster create must succeed only after at least one CP version is installed on the hosted cluster - # (operation_cluster_create.go); that bounds when install ends and upgrades begin. + # upgrade_eligible:info must be evaluated before version_state_first_seen in this group (same cycle). + # Eligibility: per resource_id, at least two distinct version labels and at least one completed. + # Excludes initial install. Cluster create bounds when install ends (operation_cluster_create.go): # https://github.com/Azure/ARO-HCP/blob/main/backend/pkg/controllers/operationcontrollers/operation_cluster_create.go#L316-L320 # - # Upgrade-only guard: a completed CP version must exist that is not the same version as the current - # desired/partial target. During initial install the first version is desired/partial with no other - # completed version yet, so it is excluded; post-create upgrades have a prior completed version. - # # Naming: level:metric:operations per https://prometheus.io/docs/practices/rules/#naming - - name: user-journey-cluster-upgrade-reach-partial-recording-rules + - name: user-journey-cluster-upgrade-shared-recording-rules interval: 1m rules: + - record: hosted_control_plane_upgrade:upgrade_eligible:info + expr: | + ( + ( + count by (cluster, resource_id, subscription_id, cluster_uuid) ( + count by (cluster, resource_id, subscription_id, cluster_uuid, version) ( + backend_cluster_version_info == 1 + ) + ) >= 2 + ) + and on (cluster, resource_id) ( + count by (cluster, resource_id) ( + backend_cluster_version_info{state="completed"} == 1 + ) >= 1 + ) + ) * 0 + 1 # When each upgrade target version first entered desired, partial, or completed (start time for SLO durations). - # Upgrade-only: excludes initial install — see operation_cluster_create.go and upgrade-only guard above. + # Upgrade-only: excludes initial install — see upgrade_eligible:info. - record: hosted_control_plane_upgrade:version_state_first_seen:timestamp expr: | min without(prometheus_replica) ( @@ -45,11 +51,7 @@ spec: ( timestamp(backend_cluster_version_info{state=~"desired|partial"} == 1) and on (cluster, resource_id) ( - count by (cluster, resource_id) ( - backend_cluster_version_info{state="completed"} == 1 - unless on (cluster, resource_id, version) - backend_cluster_version_info{state=~"desired|partial"} == 1 - ) >= 1 + hosted_control_plane_upgrade:upgrade_eligible:info == 1 ) ) or @@ -64,15 +66,23 @@ spec: backend_cluster_version_info{state=~"desired|partial"} == 1 ) >= 1 and on (cluster, resource_id) ( - count by (cluster, resource_id) ( - backend_cluster_version_info{state="completed"} == 1 - unless on (cluster, resource_id, version) - backend_cluster_version_info{state=~"desired|partial"} == 1 - ) >= 1 + hosted_control_plane_upgrade:upgrade_eligible:info == 1 ) ) or 0 * count by (cluster) (backend_cluster_version_info) + # ======================================== + # Cluster Upgrade — reach-partial SLO (20m) + # ======================================== + # SLO: 99.95% of cluster upgrades reach partial within 20 minutes of selecting the target version. + # Partial is the time for the target version to become active (recognized as active). It indicates + # the upgrade is in progress on the HCP cluster but not yet complete. + # Error budget: 0.05% (1 - 0.9995) + # + # Scope: upgrades only — gated by hosted_control_plane_upgrade:upgrade_eligible:info. + - name: user-journey-cluster-upgrade-reach-partial-recording-rules + interval: 1m + rules: # Seconds the target version has been in desired without reaching partial. Upgrade-only. - record: hosted_control_plane_upgrade:duration_in_desired:seconds expr: | @@ -83,11 +93,7 @@ spec: backend_cluster_version_info{state="desired"} == 1 ) and on (cluster, resource_id) ( - count by (cluster, resource_id) ( - backend_cluster_version_info{state="completed"} == 1 - unless on (cluster, resource_id, version) - backend_cluster_version_info{state=~"desired|partial"} == 1 - ) >= 1 + hosted_control_plane_upgrade:upgrade_eligible:info == 1 ) # Clusters whose target version has been in desired longer than 20 minutes (reach-partial SLO miss). - record: hosted_control_plane_upgrade:desired:stuck_over_20m:count @@ -134,12 +140,12 @@ spec: # # Success: desired first-seen to completed first-seen <= 1800s. # In-flight: desired or partial without completed for the target version. - # Upgrade-only scope: same guard as reach-partial (operation_cluster_create.go + different-version completed). + # Upgrade-only scope: gated by hosted_control_plane_upgrade:upgrade_eligible:info. - name: user-journey-cluster-upgrade-complete-recording-rules interval: 1m rules: # Seconds since the target version was selected until it completes; stops when that version reaches completed. - # Upgrade-only: excludes initial install — see operation_cluster_create.go and upgrade-only guard above. + # Upgrade-only: excludes initial install — see upgrade_eligible:info. - record: hosted_control_plane_upgrade:duration_in_progress:seconds expr: | ( @@ -152,11 +158,7 @@ spec: backend_cluster_version_info{state=~"desired|partial"} == 1 ) and on (cluster, resource_id) ( - count by (cluster, resource_id) ( - backend_cluster_version_info{state="completed"} == 1 - unless on (cluster, resource_id, version) - backend_cluster_version_info{state=~"desired|partial"} == 1 - ) >= 1 + hosted_control_plane_upgrade:upgrade_eligible:info == 1 ) # Clusters whose upgrade has been in progress longer than 30 minutes without completing (complete SLO miss). - record: hosted_control_plane_upgrade:in_progress:stuck_over_30m:count diff --git a/observability/alerts/userJourneyClusterUpgradeRecord-prometheusRule_test.yaml b/observability/alerts/userJourneyClusterUpgradeRecord-prometheusRule_test.yaml index 650feb7137c..fcc6c3d70ae 100644 --- a/observability/alerts/userJourneyClusterUpgradeRecord-prometheusRule_test.yaml +++ b/observability/alerts/userJourneyClusterUpgradeRecord-prometheusRule_test.yaml @@ -14,6 +14,11 @@ tests: - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="partial"}' values: "0x10 1x25" promql_expr_test: + - expr: hosted_control_plane_upgrade:upgrade_eligible:info{cluster="svc-1", resource_id="/sub/1/rg/c1"} + eval_time: 9m + exp_samples: + - labels: 'hosted_control_plane_upgrade:upgrade_eligible:info{cluster="svc-1", cluster_uuid="uuid-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1"}' + value: 1 - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} eval_time: 9m exp_samples: @@ -36,6 +41,11 @@ tests: - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c3", subscription_id="sub-2", cluster_uuid="uuid-3", version="4.19.20", state="completed"}' values: "1+0x50" promql_expr_test: + - expr: hosted_control_plane_upgrade:upgrade_eligible:info{cluster="svc-1", resource_id="/sub/2/rg/c2"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:upgrade_eligible:info{cluster="svc-1", cluster_uuid="uuid-2", resource_id="/sub/2/rg/c2", subscription_id="sub-2"}' + value: 1 - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/2/rg/c2", version="4.19.21", state="desired"} eval_time: 35m exp_samples: @@ -64,6 +74,9 @@ tests: - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/c4", subscription_id="sub-3", cluster_uuid="uuid-4", version="4.19.20", state="desired"}' values: "1+0x50" promql_expr_test: + - expr: hosted_control_plane_upgrade:upgrade_eligible:info{cluster="svc-1", resource_id="/sub/3/rg/c4"} + eval_time: 35m + exp_samples: [] - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/3/rg/c4", version="4.19.20", state="desired"} eval_time: 35m exp_samples: [] @@ -133,3 +146,35 @@ tests: exp_samples: - labels: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio{cluster="svc-1"}' value: 1 +# ======================================== +# Test 6: Two versions without completed — not upgrade eligible +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/4/rg/c5", subscription_id="sub-4", cluster_uuid="uuid-5", version="4.19.19", state="partial"}' + values: "1+0x50" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/4/rg/c5", subscription_id="sub-4", cluster_uuid="uuid-5", version="4.19.20", state="desired"}' + values: "1+0x50" + promql_expr_test: + - expr: hosted_control_plane_upgrade:upgrade_eligible:info{cluster="svc-1", resource_id="/sub/4/rg/c5"} + eval_time: 35m + exp_samples: [] + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/4/rg/c5", version="4.19.20", state="desired"} + eval_time: 35m + exp_samples: [] + - expr: hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"}' + value: 0 +# ======================================== +# Test 7: Single completed version only — not upgrade eligible +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/c6", subscription_id="sub-5", cluster_uuid="uuid-6", version="4.19.20", state="completed"}' + values: "1+0x50" + promql_expr_test: + - expr: hosted_control_plane_upgrade:upgrade_eligible:info{cluster="svc-1", resource_id="/sub/5/rg/c6"} + eval_time: 35m + exp_samples: [] From 91f3aed962e0437ef4df1531a80c6e35f3857cb8 Mon Sep 17 00:00:00 2001 From: Manyanda Chitimbo Date: Mon, 6 Jul 2026 12:59:33 +0200 Subject: [PATCH 3/4] refactor: move the recording rules to the dedicated folder --- ...ourneyClusterUpgradeCompleteMonitor-prometheusRule_test.yaml | 2 +- ...eyClusterUpgradeReachPartialMonitor-prometheusRule_test.yaml | 2 +- observability/recording-rules-services.yaml | 2 +- .../userJourneyClusterUpgrade-recordingRule.yaml} | 0 .../userJourneyClusterUpgrade-recordingRule_test.yaml} | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename observability/{alerts/userJourneyClusterUpgradeRecord-prometheusRule.yaml => recording-rules/userJourneyClusterUpgrade-recordingRule.yaml} (100%) rename observability/{alerts/userJourneyClusterUpgradeRecord-prometheusRule_test.yaml => recording-rules/userJourneyClusterUpgrade-recordingRule_test.yaml} (99%) diff --git a/observability/alerts/userJourneyClusterUpgradeCompleteMonitor-prometheusRule_test.yaml b/observability/alerts/userJourneyClusterUpgradeCompleteMonitor-prometheusRule_test.yaml index 6c740d2dda8..0c895dc5c06 100644 --- a/observability/alerts/userJourneyClusterUpgradeCompleteMonitor-prometheusRule_test.yaml +++ b/observability/alerts/userJourneyClusterUpgradeCompleteMonitor-prometheusRule_test.yaml @@ -1,5 +1,5 @@ rule_files: -- userJourneyClusterUpgradeRecord-prometheusRule.yaml +- userJourneyClusterUpgrade-recordingRule.yaml - userJourneyClusterUpgradeCompleteMonitor-prometheusRule.yaml evaluation_interval: 1m tests: diff --git a/observability/alerts/userJourneyClusterUpgradeReachPartialMonitor-prometheusRule_test.yaml b/observability/alerts/userJourneyClusterUpgradeReachPartialMonitor-prometheusRule_test.yaml index c375eaf132a..e47e887e830 100644 --- a/observability/alerts/userJourneyClusterUpgradeReachPartialMonitor-prometheusRule_test.yaml +++ b/observability/alerts/userJourneyClusterUpgradeReachPartialMonitor-prometheusRule_test.yaml @@ -1,5 +1,5 @@ rule_files: -- userJourneyClusterUpgradeRecord-prometheusRule.yaml +- userJourneyClusterUpgrade-recordingRule.yaml - userJourneyClusterUpgradeReachPartialMonitor-prometheusRule.yaml evaluation_interval: 1m tests: diff --git a/observability/recording-rules-services.yaml b/observability/recording-rules-services.yaml index 16e83b998eb..b813cea529b 100644 --- a/observability/recording-rules-services.yaml +++ b/observability/recording-rules-services.yaml @@ -1,6 +1,6 @@ prometheusRules: rulesFolders: - recording-rules/access-cluster-slo-recordingRule.yaml - - alerts/userJourneyClusterUpgradeRecord-prometheusRule.yaml + - recording-rules/userJourneyClusterUpgrade-recordingRule.yaml untestedRules: [] outputBicep: ../dev-infrastructure/modules/metrics/rules/generatedRecordingRules.bicep diff --git a/observability/alerts/userJourneyClusterUpgradeRecord-prometheusRule.yaml b/observability/recording-rules/userJourneyClusterUpgrade-recordingRule.yaml similarity index 100% rename from observability/alerts/userJourneyClusterUpgradeRecord-prometheusRule.yaml rename to observability/recording-rules/userJourneyClusterUpgrade-recordingRule.yaml diff --git a/observability/alerts/userJourneyClusterUpgradeRecord-prometheusRule_test.yaml b/observability/recording-rules/userJourneyClusterUpgrade-recordingRule_test.yaml similarity index 99% rename from observability/alerts/userJourneyClusterUpgradeRecord-prometheusRule_test.yaml rename to observability/recording-rules/userJourneyClusterUpgrade-recordingRule_test.yaml index fcc6c3d70ae..9f96104d326 100644 --- a/observability/alerts/userJourneyClusterUpgradeRecord-prometheusRule_test.yaml +++ b/observability/recording-rules/userJourneyClusterUpgrade-recordingRule_test.yaml @@ -1,5 +1,5 @@ rule_files: -- userJourneyClusterUpgradeRecord-prometheusRule.yaml +- userJourneyClusterUpgrade-recordingRule.yaml evaluation_interval: 1m tests: # ======================================== From e8f39dbc01609038f8ec79191d9ce4ba83c04490 Mon Sep 17 00:00:00 2001 From: Manyanda Chitimbo Date: Wed, 8 Jul 2026 17:26:04 +0200 Subject: [PATCH 4/4] refactor: use per-resource stuck alerts for cluster upgrade SLOs Replace fleet burn-rate alerting with per-upgrade duration alerts at low upgrade volumes, and trim observability to operational essentials. Addresses review feedback that regional burn-rate ratios are equivalent to single-failure detection until upgrade volume supports meaningful denominators: https://github.com/Azure/ARO-HCP/pull/5752#discussion_r3539885700 - Merge reach-partial and complete monitors into one PrometheusRule - Add userJourneyClusterUpgradeStuckInDesired and StuckInProgress alerts - Drop burn-rate recording rules; keep duration and in-progress metrics - Slim dashboard to in-flight table, stuck counts, and detail tables --- .../generatedRPPrometheusAlertingRules.bicep | 205 +-- .../rules/generatedRecordingRules.bicep | 96 +- observability/alerts-rp-services.yaml | 3 +- ...UpgradeCompleteMonitor-prometheusRule.yaml | 82 - ...deCompleteMonitor-prometheusRule_test.yaml | 772 --------- ...yClusterUpgradeMonitor-prometheusRule.yaml | 51 + ...terUpgradeMonitor-prometheusRule_test.yaml | 128 ++ ...adeReachPartialMonitor-prometheusRule.yaml | 72 - ...achPartialMonitor-prometheusRule_test.yaml | 778 --------- .../cluster-upgrade-slo-dashboard.json | 1490 ----------------- .../sre/user-journey/cluster-upgrade-slo.json | 318 ++++ ...erJourneyClusterUpgrade-recordingRule.yaml | 181 +- ...rneyClusterUpgrade-recordingRule_test.yaml | 114 +- 13 files changed, 659 insertions(+), 3631 deletions(-) delete mode 100644 observability/alerts/userJourneyClusterUpgradeCompleteMonitor-prometheusRule.yaml delete mode 100644 observability/alerts/userJourneyClusterUpgradeCompleteMonitor-prometheusRule_test.yaml create mode 100644 observability/alerts/userJourneyClusterUpgradeMonitor-prometheusRule.yaml create mode 100644 observability/alerts/userJourneyClusterUpgradeMonitor-prometheusRule_test.yaml delete mode 100644 observability/alerts/userJourneyClusterUpgradeReachPartialMonitor-prometheusRule.yaml delete mode 100644 observability/alerts/userJourneyClusterUpgradeReachPartialMonitor-prometheusRule_test.yaml delete mode 100644 observability/grafana-dashboards/sre/user-journey/cluster-upgrade-slo-dashboard.json create mode 100644 observability/grafana-dashboards/sre/user-journey/cluster-upgrade-slo.json diff --git a/dev-infrastructure/modules/metrics/rules/generatedRPPrometheusAlertingRules.bicep b/dev-infrastructure/modules/metrics/rules/generatedRPPrometheusAlertingRules.bicep index 4b81bebb931..63032b84ad6 100644 --- a/dev-infrastructure/modules/metrics/rules/generatedRPPrometheusAlertingRules.bicep +++ b/dev-infrastructure/modules/metrics/rules/generatedRPPrometheusAlertingRules.bicep @@ -234,8 +234,8 @@ resource arohcpAccessClusterSaturationAlerts 'Microsoft.AlertsManagement/prometh } } -resource rpUserJourneyClusterUpgradeReachPartialMonitorRules 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { - name: 'rp-user-journey-cluster-upgrade-reach-partial-monitor-rules' +resource rpUserJourneyClusterUpgradeMonitorRules 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'rp-user-journey-cluster-upgrade-monitor-rules' location: location properties: { interval: 'PT1M' @@ -250,29 +250,25 @@ resource rpUserJourneyClusterUpgradeReachPartialMonitorRules 'Microsoft.AlertsMa } } ] - alert: 'userJourneyClusterUpgradeReachPartial1h5m' + alert: 'userJourneyClusterUpgradeStuckInDesired' enabled: true labels: { - long_window: '1h' severity: 'info' - short_window: '5m' slo: 'cluster-upgrade-reach-partial' } annotations: { - correlationId: 'userJourneyClusterUpgradeReachPartial/{{ $labels.cluster }}' - description: '''More than 14.4x of the 99.95% cluster upgrade reach-partial error budget is being consumed. Cluster upgrades are stuck for over 20 minutes without the target version reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. -Window: 1h/5m -Management Cluster: {{ $labels.cluster }} + correlationId: 'userJourneyClusterUpgradeStuckInDesired/{{ $labels.resource_id }}/{{ $labels.version }}' + description: '''Cluster upgrade target version {{ $labels.version }} has been in desired for over 20 minutes without reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on the affected cluster. +Service Cluster: {{ $labels.cluster }} ''' - info: '''More than 14.4x of the 99.95% cluster upgrade reach-partial error budget is being consumed. Cluster upgrades are stuck for over 20 minutes without the target version reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. -Window: 1h/5m -Management Cluster: {{ $labels.cluster }} + info: '''Cluster upgrade target version {{ $labels.version }} has been in desired for over 20 minutes without reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on the affected cluster. +Service Cluster: {{ $labels.cluster }} ''' runbook_url: 'https://aka.ms/arohcp-runbook-cluster-upgrade' - summary: '[HCPUpgradeReachPartialBurn] {{ $labels.cluster }}' - title: '[HCPUpgradeReachPartialBurn] {{ $labels.cluster }}' + summary: '[HCPUpgradeStuckInDesired] {{ $labels.resource_id }}' + title: '[HCPUpgradeStuckInDesired] {{ $labels.resource_id }} version:{{ $labels.version }} cluster:{{ $labels.cluster }}' } - expression: 'hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate5m > 14.4 and hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate1h > 14.4' + expression: 'hosted_control_plane_upgrade:duration_in_desired:seconds > 1200' for: 'PT5M' severity: severityCeiling > 0 ? max(4, severityCeiling) : 4 } @@ -286,189 +282,28 @@ Management Cluster: {{ $labels.cluster }} } } ] - alert: 'userJourneyClusterUpgradeReachPartial6h30m' + alert: 'userJourneyClusterUpgradeStuckInProgress' enabled: true labels: { - long_window: '6h' severity: 'info' - short_window: '30m' - slo: 'cluster-upgrade-reach-partial' - } - annotations: { - correlationId: 'userJourneyClusterUpgradeReachPartial/{{ $labels.cluster }}' - description: '''More than 6x of the 99.95% cluster upgrade reach-partial error budget is being consumed. Cluster upgrades are stuck for over 20 minutes without the target version reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. -Window: 6h/30m -Management Cluster: {{ $labels.cluster }} -''' - info: '''More than 6x of the 99.95% cluster upgrade reach-partial error budget is being consumed. Cluster upgrades are stuck for over 20 minutes without the target version reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. -Window: 6h/30m -Management Cluster: {{ $labels.cluster }} -''' - runbook_url: 'https://aka.ms/arohcp-runbook-cluster-upgrade' - summary: '[HCPUpgradeReachPartialBurn] {{ $labels.cluster }}' - title: '[HCPUpgradeReachPartialBurn] {{ $labels.cluster }}' - } - expression: 'hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate30m > 6 and hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate6h > 6' - for: 'PT30M' - severity: severityCeiling > 0 ? max(4, severityCeiling) : 4 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'userJourneyClusterUpgradeReachPartial3d6h' - enabled: true - labels: { - long_window: '3d' - severity: 'info' - short_window: '6h' - slo: 'cluster-upgrade-reach-partial' - } - annotations: { - correlationId: 'userJourneyClusterUpgradeReachPartial/{{ $labels.cluster }}' - description: '''Cluster upgrade reach-partial has been steadily consuming the 99.95% SLO error budget (1x burn). Cluster upgrades are stuck for over 20 minutes without reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. -Window: 3d/6h -Management Cluster: {{ $labels.cluster }} -''' - info: '''Cluster upgrade reach-partial has been steadily consuming the 99.95% SLO error budget (1x burn). Cluster upgrades are stuck for over 20 minutes without reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. -Window: 3d/6h -Management Cluster: {{ $labels.cluster }} -''' - runbook_url: 'https://aka.ms/arohcp-runbook-cluster-upgrade' - summary: '[HCPUpgradeReachPartialBurn] {{ $labels.cluster }}' - title: '[HCPUpgradeReachPartialBurn] {{ $labels.cluster }}' - } - expression: 'hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate6h > 1 and hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate3d > 1' - for: 'PT30M' - severity: severityCeiling > 0 ? max(4, severityCeiling) : 4 - } - ] - scopes: [ - azureMonitoring - ] - } -} - -resource rpUserJourneyClusterUpgradeCompleteMonitorRules 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { - name: 'rp-user-journey-cluster-upgrade-complete-monitor-rules' - location: location - properties: { - interval: 'PT1M' - rules: [ - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'userJourneyClusterUpgradeComplete1h5m' - enabled: true - labels: { - long_window: '1h' - severity: 'info' - short_window: '5m' slo: 'cluster-upgrade-complete' } annotations: { - correlationId: 'userJourneyClusterUpgradeComplete/{{ $labels.cluster }}' - description: '''More than 14.4x of the 99.95% cluster upgrade complete error budget is being consumed. Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. -Window: 1h/5m -Management Cluster: {{ $labels.cluster }} + correlationId: 'userJourneyClusterUpgradeStuckInProgress/{{ $labels.resource_id }}/{{ $labels.version }}' + description: '''Cluster upgrade target version {{ $labels.version }} has been in progress for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on the affected cluster. +Service Cluster: {{ $labels.cluster }} ''' - info: '''More than 14.4x of the 99.95% cluster upgrade complete error budget is being consumed. Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. -Window: 1h/5m -Management Cluster: {{ $labels.cluster }} + info: '''Cluster upgrade target version {{ $labels.version }} has been in progress for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on the affected cluster. +Service Cluster: {{ $labels.cluster }} ''' runbook_url: 'https://aka.ms/arohcp-runbook-cluster-upgrade' - summary: '[HCPUpgradeCompleteBurn] {{ $labels.cluster }}' - title: '[HCPUpgradeCompleteBurn] {{ $labels.cluster }}' + summary: '[HCPUpgradeStuckInProgress] {{ $labels.resource_id }}' + title: '[HCPUpgradeStuckInProgress] {{ $labels.resource_id }} version:{{ $labels.version }} cluster:{{ $labels.cluster }}' } - expression: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate5m > 14.4 and hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate1h > 14.4' + expression: 'hosted_control_plane_upgrade:duration_in_progress:seconds > 1800' for: 'PT5M' severity: severityCeiling > 0 ? max(4, severityCeiling) : 4 } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'userJourneyClusterUpgradeComplete6h30m' - enabled: true - labels: { - long_window: '6h' - severity: 'info' - short_window: '30m' - slo: 'cluster-upgrade-complete' - } - annotations: { - correlationId: 'userJourneyClusterUpgradeComplete/{{ $labels.cluster }}' - description: '''More than 6x of the 99.95% cluster upgrade complete error budget is being consumed. Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. -Window: 6h/30m -Management Cluster: {{ $labels.cluster }} -''' - info: '''More than 6x of the 99.95% cluster upgrade complete error budget is being consumed. Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. -Window: 6h/30m -Management Cluster: {{ $labels.cluster }} -''' - runbook_url: 'https://aka.ms/arohcp-runbook-cluster-upgrade' - summary: '[HCPUpgradeCompleteBurn] {{ $labels.cluster }}' - title: '[HCPUpgradeCompleteBurn] {{ $labels.cluster }}' - } - expression: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate30m > 6 and hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate6h > 6' - for: 'PT30M' - severity: severityCeiling > 0 ? max(4, severityCeiling) : 4 - } - { - actions: [ - for g in actionGroups: { - actionGroupId: g - actionProperties: { - 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' - 'IcM.CorrelationId': '#$.annotations.correlationId#' - } - } - ] - alert: 'userJourneyClusterUpgradeComplete3d6h' - enabled: true - labels: { - long_window: '3d' - severity: 'info' - short_window: '6h' - slo: 'cluster-upgrade-complete' - } - annotations: { - correlationId: 'userJourneyClusterUpgradeComplete/{{ $labels.cluster }}' - description: '''Cluster upgrade complete has been steadily consuming the 99.95% SLO error budget (1x burn). Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. -Window: 3d/6h -Management Cluster: {{ $labels.cluster }} -''' - info: '''Cluster upgrade complete has been steadily consuming the 99.95% SLO error budget (1x burn). Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. -Window: 3d/6h -Management Cluster: {{ $labels.cluster }} -''' - runbook_url: 'https://aka.ms/arohcp-runbook-cluster-upgrade' - summary: '[HCPUpgradeCompleteBurn] {{ $labels.cluster }}' - title: '[HCPUpgradeCompleteBurn] {{ $labels.cluster }}' - } - expression: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate6h > 1 and hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate3d > 1' - for: 'PT30M' - severity: severityCeiling > 0 ? max(4, severityCeiling) : 4 - } ] scopes: [ azureMonitoring diff --git a/dev-infrastructure/modules/metrics/rules/generatedRecordingRules.bicep b/dev-infrastructure/modules/metrics/rules/generatedRecordingRules.bicep index d92176532da..13b69e6a962 100644 --- a/dev-infrastructure/modules/metrics/rules/generatedRecordingRules.bicep +++ b/dev-infrastructure/modules/metrics/rules/generatedRecordingRules.bicep @@ -28,8 +28,8 @@ resource arohcpAccessClusterSloRecordingRules 'Microsoft.AlertsManagement/promet } } -resource userJourneyClusterUpgradeSharedRecordingRules 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { - name: 'user-journey-cluster-upgrade-shared-recording-rules' +resource arohcpUserJourneyClusterUpgradeRecordingRules 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'arohcp_user_journey_cluster_upgrade_recording_rules' location: location properties: { scopes: [ @@ -44,103 +44,19 @@ resource userJourneyClusterUpgradeSharedRecordingRules 'Microsoft.AlertsManageme } { record: 'hosted_control_plane_upgrade:version_state_first_seen:timestamp' - expression: 'min without (prometheus_replica) (min by (cluster, resource_id, subscription_id, cluster_uuid, version, state) (hosted_control_plane_upgrade:version_state_first_seen:timestamp or (timestamp(backend_cluster_version_info{state=~"desired|partial"} == 1) and on (cluster, resource_id) (hosted_control_plane_upgrade:upgrade_eligible:info == 1)) or timestamp(backend_cluster_version_info{state="completed"} == 1)))' + expression: 'min without (prometheus_replica) (min by (cluster, resource_id, subscription_id, cluster_uuid, version, state) ((hosted_control_plane_upgrade:version_state_first_seen:timestamp or (timestamp(backend_cluster_version_info{state=~"desired|partial"} == 1) and on (cluster, resource_id) (hosted_control_plane_upgrade:upgrade_eligible:info == 1))) unless on (cluster, resource_id, subscription_id, cluster_uuid, version) (backend_cluster_version_info{state="completed"} == 1)))' } { record: 'hosted_control_plane_upgrade:in_progress:count' - expression: 'count by (cluster) (count by (cluster, resource_id) (backend_cluster_version_info{state=~"desired|partial"} == 1) >= 1 and on (cluster, resource_id) (hosted_control_plane_upgrade:upgrade_eligible:info == 1)) or 0 * count by (cluster) (backend_cluster_version_info)' + expression: 'count by (cluster) (count by (cluster, resource_id) ((backend_cluster_version_info{state=~"desired|partial"} == 1 unless on (cluster, resource_id, subscription_id, cluster_uuid, version) backend_cluster_version_info{state="completed"} == 1)) >= 1 and on (cluster, resource_id) (hosted_control_plane_upgrade:upgrade_eligible:info == 1)) or 0 * count by (cluster) (backend_cluster_version_info)' } - ] - } -} - -resource userJourneyClusterUpgradeReachPartialRecordingRules 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { - name: 'user-journey-cluster-upgrade-reach-partial-recording-rules' - location: location - properties: { - scopes: [ - azureMonitoring - ] - enabled: true - interval: 'PT1M' - rules: [ { record: 'hosted_control_plane_upgrade:duration_in_desired:seconds' - expression: '(time() - hosted_control_plane_upgrade:version_state_first_seen:timestamp{state="desired"}) and on (cluster, resource_id, subscription_id, cluster_uuid, version) (backend_cluster_version_info{state="desired"} == 1) and on (cluster, resource_id) (hosted_control_plane_upgrade:upgrade_eligible:info == 1)' - } - { - record: 'hosted_control_plane_upgrade:desired:stuck_over_20m:count' - expression: 'count by (cluster) (hosted_control_plane_upgrade:duration_in_desired:seconds > 1200) or 0 * hosted_control_plane_upgrade:in_progress:count' - } - { - record: 'hosted_control_plane_upgrade:desired:stuck_over_20m:ratio' - expression: 'hosted_control_plane_upgrade:desired:stuck_over_20m:count / clamp_min(hosted_control_plane_upgrade:in_progress:count, 1)' - } - { - record: 'hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate5m' - expression: 'avg_over_time(hosted_control_plane_upgrade:desired:stuck_over_20m:ratio[5m]) / (1 - 0.9995)' - } - { - record: 'hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate30m' - expression: 'avg_over_time(hosted_control_plane_upgrade:desired:stuck_over_20m:ratio[30m]) / (1 - 0.9995)' - } - { - record: 'hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate1h' - expression: 'avg_over_time(hosted_control_plane_upgrade:desired:stuck_over_20m:ratio[1h]) / (1 - 0.9995)' - } - { - record: 'hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate6h' - expression: 'avg_over_time(hosted_control_plane_upgrade:desired:stuck_over_20m:ratio[6h]) / (1 - 0.9995)' - } - { - record: 'hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate3d' - expression: 'avg_over_time(hosted_control_plane_upgrade:desired:stuck_over_20m:ratio[3d]) / (1 - 0.9995)' + expression: '(time() - hosted_control_plane_upgrade:version_state_first_seen:timestamp{state="desired"}) and on (cluster, resource_id, subscription_id, cluster_uuid, version) ((backend_cluster_version_info{state="desired"} == 1 unless on (cluster, resource_id, subscription_id, cluster_uuid, version) (backend_cluster_version_info{state="partial"} == 1 or backend_cluster_version_info{state="completed"} == 1))) and on (cluster, resource_id) (hosted_control_plane_upgrade:upgrade_eligible:info == 1)' } - ] - } -} - -resource userJourneyClusterUpgradeCompleteRecordingRules 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { - name: 'user-journey-cluster-upgrade-complete-recording-rules' - location: location - properties: { - scopes: [ - azureMonitoring - ] - enabled: true - interval: 'PT1M' - rules: [ { record: 'hosted_control_plane_upgrade:duration_in_progress:seconds' - expression: '(time() - hosted_control_plane_upgrade:version_state_first_seen:timestamp{state="desired"}) unless on (cluster, resource_id, subscription_id, cluster_uuid, version) (backend_cluster_version_info{state="completed"} == 1) and on (cluster, resource_id, subscription_id, cluster_uuid, version) (backend_cluster_version_info{state=~"desired|partial"} == 1) and on (cluster, resource_id) (hosted_control_plane_upgrade:upgrade_eligible:info == 1)' - } - { - record: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:count' - expression: 'count by (cluster) (hosted_control_plane_upgrade:duration_in_progress:seconds > 1800) or 0 * hosted_control_plane_upgrade:in_progress:count' - } - { - record: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio' - expression: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:count / clamp_min(hosted_control_plane_upgrade:in_progress:count, 1)' - } - { - record: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate5m' - expression: 'avg_over_time(hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio[5m]) / (1 - 0.9995)' - } - { - record: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate30m' - expression: 'avg_over_time(hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio[30m]) / (1 - 0.9995)' - } - { - record: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate1h' - expression: 'avg_over_time(hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio[1h]) / (1 - 0.9995)' - } - { - record: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate6h' - expression: 'avg_over_time(hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio[6h]) / (1 - 0.9995)' - } - { - record: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate3d' - expression: 'avg_over_time(hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio[3d]) / (1 - 0.9995)' + expression: '(sum without (state) ((time() - hosted_control_plane_upgrade:version_state_first_seen:timestamp{state="desired"}) and on (cluster, resource_id) (hosted_control_plane_upgrade:upgrade_eligible:info == 1)) unless on (cluster, resource_id, subscription_id, cluster_uuid, version) (backend_cluster_version_info{state="completed"} == 1)) * on (cluster, resource_id, subscription_id, cluster_uuid, version) group_left (state) (backend_cluster_version_info{state="partial"} == 1 or (backend_cluster_version_info{state="desired"} == 1 unless on (cluster, resource_id, subscription_id, cluster_uuid, version) backend_cluster_version_info{state="partial"} == 1))' } ] } diff --git a/observability/alerts-rp-services.yaml b/observability/alerts-rp-services.yaml index ea83912dfa5..2af317ca77d 100644 --- a/observability/alerts-rp-services.yaml +++ b/observability/alerts-rp-services.yaml @@ -1,8 +1,7 @@ prometheusRules: rulesFolders: - alerts/access-cluster-slo-prometheusRule.yaml - - alerts/userJourneyClusterUpgradeReachPartialMonitor-prometheusRule.yaml - - alerts/userJourneyClusterUpgradeCompleteMonitor-prometheusRule.yaml + - alerts/userJourneyClusterUpgradeMonitor-prometheusRule.yaml - alerts/nodepool-slo-prometheusRule.yaml untestedRules: [] testDependencies: diff --git a/observability/alerts/userJourneyClusterUpgradeCompleteMonitor-prometheusRule.yaml b/observability/alerts/userJourneyClusterUpgradeCompleteMonitor-prometheusRule.yaml deleted file mode 100644 index eaf69b8e1a0..00000000000 --- a/observability/alerts/userJourneyClusterUpgradeCompleteMonitor-prometheusRule.yaml +++ /dev/null @@ -1,82 +0,0 @@ -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - labels: - app.kubernetes.io/name: kube-prometheus - app.kubernetes.io/part-of: kube-prometheus - prometheus: k8s - role: alert-rules - name: user-journey-cluster-upgrade-complete-alerts - namespace: monitoring -spec: - groups: - # ======================================== - # Cluster Upgrade Complete SLO Monitor Alerts - # ======================================== - # Multi-window burn-rate alerts. Sev 4 (info). - # - # In absence of https://issues.redhat.com/browse/ARO-22686 (reporting of control plane - # version history for versions lower than 4.22), cluster upgrade completeness is directly - # impacted by data plane state — e.g. no worker nodes means the upgrade will not reach - # completed; missing console component in external auth prevents CVO from marking the - # upgrade complete. Alerts may therefore fire due to missing data plane configuration - # rather than control plane failure. The runbook will be written to accommodate this. - - name: rp-user-journey-cluster-upgrade-complete-monitor-rules - rules: - - alert: userJourneyClusterUpgradeComplete1h5m - expr: | - hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate5m > 14.4 - and - hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate1h > 14.4 - for: 5m - labels: - severity: info - slo: cluster-upgrade-complete - short_window: 5m - long_window: 1h - annotations: - summary: "[HCPUpgradeCompleteBurn] {{ $labels.cluster }}" - description: | - More than 14.4x of the 99.95% cluster upgrade complete error budget is being consumed. Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. - Window: 1h/5m - Management Cluster: {{ $labels.cluster }} - runbook_url: https://aka.ms/arohcp-runbook-cluster-upgrade - correlationId: "userJourneyClusterUpgradeComplete/{{ $labels.cluster }}" - - alert: userJourneyClusterUpgradeComplete6h30m - expr: | - hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate30m > 6 - and - hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate6h > 6 - for: 30m - labels: - severity: info - slo: cluster-upgrade-complete - short_window: 30m - long_window: 6h - annotations: - summary: "[HCPUpgradeCompleteBurn] {{ $labels.cluster }}" - description: | - More than 6x of the 99.95% cluster upgrade complete error budget is being consumed. Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. - Window: 6h/30m - Management Cluster: {{ $labels.cluster }} - runbook_url: https://aka.ms/arohcp-runbook-cluster-upgrade - correlationId: "userJourneyClusterUpgradeComplete/{{ $labels.cluster }}" - - alert: userJourneyClusterUpgradeComplete3d6h - expr: | - hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate6h > 1 - and - hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate3d > 1 - for: 30m - labels: - severity: info - slo: cluster-upgrade-complete - short_window: 6h - long_window: 3d - annotations: - summary: "[HCPUpgradeCompleteBurn] {{ $labels.cluster }}" - description: | - Cluster upgrade complete has been steadily consuming the 99.95% SLO error budget (1x burn). Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. - Window: 3d/6h - Management Cluster: {{ $labels.cluster }} - runbook_url: https://aka.ms/arohcp-runbook-cluster-upgrade - correlationId: "userJourneyClusterUpgradeComplete/{{ $labels.cluster }}" diff --git a/observability/alerts/userJourneyClusterUpgradeCompleteMonitor-prometheusRule_test.yaml b/observability/alerts/userJourneyClusterUpgradeCompleteMonitor-prometheusRule_test.yaml deleted file mode 100644 index 0c895dc5c06..00000000000 --- a/observability/alerts/userJourneyClusterUpgradeCompleteMonitor-prometheusRule_test.yaml +++ /dev/null @@ -1,772 +0,0 @@ -rule_files: -- userJourneyClusterUpgrade-recordingRule.yaml -- userJourneyClusterUpgradeCompleteMonitor-prometheusRule.yaml -evaluation_interval: 1m -tests: -# ======================================== -# Test 1: Successful upgrade complete — no alert -# ======================================== -- interval: 1m - input_series: - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed"}' - values: "1+0x40" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="desired"}' - values: "1x10 0x25" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="completed"}' - values: "0x10 1x25" - alert_rule_test: - - eval_time: 40m - alertname: userJourneyClusterUpgradeComplete1h5m - - eval_time: 40m - alertname: userJourneyClusterUpgradeComplete6h30m - - eval_time: 40m - alertname: userJourneyClusterUpgradeComplete3d6h -# ======================================== -# Test 2: Fast burn — 2 of 10 in-flight upgrades stuck in progress >30m → 5m+1h alert fires -# a/b stuck from t=0; c–j join at minute 50 with <30m in progress at eval (healthy in-flight). -# Eval at 80m: ratio 0.2 stable since minute 50. burnrate1h omitted (1h window spans mixed fleet phases). -# ======================================== -- interval: 1m - input_series: - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/a", subscription_id="sub-3", cluster_uuid="uuid-a", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/a", subscription_id="sub-3", cluster_uuid="uuid-a", version="4.19.22", state="desired"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/b", subscription_id="sub-3", cluster_uuid="uuid-b", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/b", subscription_id="sub-3", cluster_uuid="uuid-b", version="4.19.22", state="desired"}' - values: "1x1 0x89" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/b", subscription_id="sub-3", cluster_uuid="uuid-b", version="4.19.22", state="partial"}' - values: "0x1 1x89" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/c", subscription_id="sub-3", cluster_uuid="uuid-c", version="4.19.19", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/c", subscription_id="sub-3", cluster_uuid="uuid-c", version="4.19.20", state="desired"}' - values: "0x50 1x40" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/d", subscription_id="sub-3", cluster_uuid="uuid-d", version="4.19.19", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/d", subscription_id="sub-3", cluster_uuid="uuid-d", version="4.19.20", state="desired"}' - values: "0x50 1x40" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/e", subscription_id="sub-3", cluster_uuid="uuid-e", version="4.19.19", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/e", subscription_id="sub-3", cluster_uuid="uuid-e", version="4.19.20", state="desired"}' - values: "0x50 1x40" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/f", subscription_id="sub-3", cluster_uuid="uuid-f", version="4.19.19", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/f", subscription_id="sub-3", cluster_uuid="uuid-f", version="4.19.20", state="desired"}' - values: "0x50 1x40" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/g", subscription_id="sub-3", cluster_uuid="uuid-g", version="4.19.19", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/g", subscription_id="sub-3", cluster_uuid="uuid-g", version="4.19.20", state="desired"}' - values: "0x50 1x40" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/h", subscription_id="sub-3", cluster_uuid="uuid-h", version="4.19.19", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/h", subscription_id="sub-3", cluster_uuid="uuid-h", version="4.19.20", state="desired"}' - values: "0x50 1x40" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/i", subscription_id="sub-3", cluster_uuid="uuid-i", version="4.19.19", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/i", subscription_id="sub-3", cluster_uuid="uuid-i", version="4.19.20", state="desired"}' - values: "0x50 1x40" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/j", subscription_id="sub-3", cluster_uuid="uuid-j", version="4.19.19", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/j", subscription_id="sub-3", cluster_uuid="uuid-j", version="4.19.20", state="desired"}' - values: "0x50 1x40" - promql_expr_test: - - expr: hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio{cluster="svc-1"} - eval_time: 80m - exp_samples: - - labels: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio{cluster="svc-1"}' - value: 0.2 - - expr: hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate5m{cluster="svc-1"} - eval_time: 80m - exp_samples: - - labels: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate5m{cluster="svc-1"}' - value: 4.0000000000004405e+02 - alert_rule_test: - - eval_time: 80m - alertname: userJourneyClusterUpgradeComplete1h5m - exp_alerts: - - exp_labels: - cluster: "svc-1" - severity: info - slo: cluster-upgrade-complete - short_window: 5m - long_window: 1h - exp_annotations: - summary: "[HCPUpgradeCompleteBurn] svc-1" - description: | - More than 14.4x of the 99.95% cluster upgrade complete error budget is being consumed. Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. - Window: 1h/5m - Management Cluster: svc-1 - runbook_url: "https://aka.ms/arohcp-runbook-cluster-upgrade" - correlationId: "userJourneyClusterUpgradeComplete/svc-1" - - eval_time: 80m - alertname: userJourneyClusterUpgradeComplete6h30m - exp_alerts: - - exp_labels: - cluster: "svc-1" - severity: info - slo: cluster-upgrade-complete - short_window: 30m - long_window: 6h - exp_annotations: - summary: "[HCPUpgradeCompleteBurn] svc-1" - description: | - More than 6x of the 99.95% cluster upgrade complete error budget is being consumed. Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. - Window: 6h/30m - Management Cluster: svc-1 - runbook_url: "https://aka.ms/arohcp-runbook-cluster-upgrade" - correlationId: "userJourneyClusterUpgradeComplete/svc-1" - - eval_time: 80m - alertname: userJourneyClusterUpgradeComplete3d6h - exp_alerts: - - exp_labels: - cluster: "svc-1" - severity: info - slo: cluster-upgrade-complete - short_window: 6h - long_window: 3d - exp_annotations: - summary: "[HCPUpgradeCompleteBurn] svc-1" - description: | - Cluster upgrade complete has been steadily consuming the 99.95% SLO error budget (1x burn). Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. - Window: 3d/6h - Management Cluster: svc-1 - runbook_url: "https://aka.ms/arohcp-runbook-cluster-upgrade" - correlationId: "userJourneyClusterUpgradeComplete/svc-1" -# ======================================== -# Test 3: Fleet healthy — no SLO alert -# ======================================== -- interval: 1m - input_series: - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/4/rg/a", subscription_id="sub-4", cluster_uuid="uuid-a", version="4.19.20", state="completed"}' - values: "1+0x40" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/4/rg/b", subscription_id="sub-4", cluster_uuid="uuid-b", version="4.19.20", state="completed"}' - values: "1+0x40" - promql_expr_test: - - expr: hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio{cluster="svc-1"} - eval_time: 10m - exp_samples: - - labels: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio{cluster="svc-1"}' - value: 0 - alert_rule_test: - - eval_time: 40m - alertname: userJourneyClusterUpgradeComplete1h5m - - eval_time: 40m - alertname: userJourneyClusterUpgradeComplete6h30m - - eval_time: 40m - alertname: userJourneyClusterUpgradeComplete3d6h -# ======================================== -# Test 4: Medium burn — 1 of 140 stuck >30m → 6h/30m alert fires, 1h/5m does not -# Fleet stable from t=0: 1 stuck in desired, 139 healthy in partial (not complete-SLO-stuck). -# Ratio 1/140 ≈ 0.71% from minute 31; eval at 90m satisfies 6h30m for:30m once burnrate6h > 6. -# ======================================== -- interval: 1m - input_series: - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/stuck", subscription_id="sub-5", cluster_uuid="uuid-stuck", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/stuck", subscription_id="sub-5", cluster_uuid="uuid-stuck", version="4.19.22", state="desired"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h000", subscription_id="sub-5", cluster_uuid="uuid-h000", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h000", subscription_id="sub-5", cluster_uuid="uuid-h000", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h001", subscription_id="sub-5", cluster_uuid="uuid-h001", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h001", subscription_id="sub-5", cluster_uuid="uuid-h001", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h002", subscription_id="sub-5", cluster_uuid="uuid-h002", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h002", subscription_id="sub-5", cluster_uuid="uuid-h002", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h003", subscription_id="sub-5", cluster_uuid="uuid-h003", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h003", subscription_id="sub-5", cluster_uuid="uuid-h003", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h004", subscription_id="sub-5", cluster_uuid="uuid-h004", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h004", subscription_id="sub-5", cluster_uuid="uuid-h004", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h005", subscription_id="sub-5", cluster_uuid="uuid-h005", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h005", subscription_id="sub-5", cluster_uuid="uuid-h005", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h006", subscription_id="sub-5", cluster_uuid="uuid-h006", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h006", subscription_id="sub-5", cluster_uuid="uuid-h006", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h007", subscription_id="sub-5", cluster_uuid="uuid-h007", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h007", subscription_id="sub-5", cluster_uuid="uuid-h007", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h008", subscription_id="sub-5", cluster_uuid="uuid-h008", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h008", subscription_id="sub-5", cluster_uuid="uuid-h008", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h009", subscription_id="sub-5", cluster_uuid="uuid-h009", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h009", subscription_id="sub-5", cluster_uuid="uuid-h009", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h010", subscription_id="sub-5", cluster_uuid="uuid-h010", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h010", subscription_id="sub-5", cluster_uuid="uuid-h010", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h011", subscription_id="sub-5", cluster_uuid="uuid-h011", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h011", subscription_id="sub-5", cluster_uuid="uuid-h011", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h012", subscription_id="sub-5", cluster_uuid="uuid-h012", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h012", subscription_id="sub-5", cluster_uuid="uuid-h012", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h013", subscription_id="sub-5", cluster_uuid="uuid-h013", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h013", subscription_id="sub-5", cluster_uuid="uuid-h013", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h014", subscription_id="sub-5", cluster_uuid="uuid-h014", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h014", subscription_id="sub-5", cluster_uuid="uuid-h014", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h015", subscription_id="sub-5", cluster_uuid="uuid-h015", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h015", subscription_id="sub-5", cluster_uuid="uuid-h015", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h016", subscription_id="sub-5", cluster_uuid="uuid-h016", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h016", subscription_id="sub-5", cluster_uuid="uuid-h016", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h017", subscription_id="sub-5", cluster_uuid="uuid-h017", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h017", subscription_id="sub-5", cluster_uuid="uuid-h017", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h018", subscription_id="sub-5", cluster_uuid="uuid-h018", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h018", subscription_id="sub-5", cluster_uuid="uuid-h018", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h019", subscription_id="sub-5", cluster_uuid="uuid-h019", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h019", subscription_id="sub-5", cluster_uuid="uuid-h019", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h020", subscription_id="sub-5", cluster_uuid="uuid-h020", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h020", subscription_id="sub-5", cluster_uuid="uuid-h020", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h021", subscription_id="sub-5", cluster_uuid="uuid-h021", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h021", subscription_id="sub-5", cluster_uuid="uuid-h021", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h022", subscription_id="sub-5", cluster_uuid="uuid-h022", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h022", subscription_id="sub-5", cluster_uuid="uuid-h022", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h023", subscription_id="sub-5", cluster_uuid="uuid-h023", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h023", subscription_id="sub-5", cluster_uuid="uuid-h023", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h024", subscription_id="sub-5", cluster_uuid="uuid-h024", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h024", subscription_id="sub-5", cluster_uuid="uuid-h024", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h025", subscription_id="sub-5", cluster_uuid="uuid-h025", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h025", subscription_id="sub-5", cluster_uuid="uuid-h025", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h026", subscription_id="sub-5", cluster_uuid="uuid-h026", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h026", subscription_id="sub-5", cluster_uuid="uuid-h026", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h027", subscription_id="sub-5", cluster_uuid="uuid-h027", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h027", subscription_id="sub-5", cluster_uuid="uuid-h027", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h028", subscription_id="sub-5", cluster_uuid="uuid-h028", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h028", subscription_id="sub-5", cluster_uuid="uuid-h028", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h029", subscription_id="sub-5", cluster_uuid="uuid-h029", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h029", subscription_id="sub-5", cluster_uuid="uuid-h029", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h030", subscription_id="sub-5", cluster_uuid="uuid-h030", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h030", subscription_id="sub-5", cluster_uuid="uuid-h030", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h031", subscription_id="sub-5", cluster_uuid="uuid-h031", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h031", subscription_id="sub-5", cluster_uuid="uuid-h031", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h032", subscription_id="sub-5", cluster_uuid="uuid-h032", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h032", subscription_id="sub-5", cluster_uuid="uuid-h032", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h033", subscription_id="sub-5", cluster_uuid="uuid-h033", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h033", subscription_id="sub-5", cluster_uuid="uuid-h033", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h034", subscription_id="sub-5", cluster_uuid="uuid-h034", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h034", subscription_id="sub-5", cluster_uuid="uuid-h034", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h035", subscription_id="sub-5", cluster_uuid="uuid-h035", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h035", subscription_id="sub-5", cluster_uuid="uuid-h035", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h036", subscription_id="sub-5", cluster_uuid="uuid-h036", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h036", subscription_id="sub-5", cluster_uuid="uuid-h036", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h037", subscription_id="sub-5", cluster_uuid="uuid-h037", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h037", subscription_id="sub-5", cluster_uuid="uuid-h037", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h038", subscription_id="sub-5", cluster_uuid="uuid-h038", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h038", subscription_id="sub-5", cluster_uuid="uuid-h038", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h039", subscription_id="sub-5", cluster_uuid="uuid-h039", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h039", subscription_id="sub-5", cluster_uuid="uuid-h039", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h040", subscription_id="sub-5", cluster_uuid="uuid-h040", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h040", subscription_id="sub-5", cluster_uuid="uuid-h040", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h041", subscription_id="sub-5", cluster_uuid="uuid-h041", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h041", subscription_id="sub-5", cluster_uuid="uuid-h041", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h042", subscription_id="sub-5", cluster_uuid="uuid-h042", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h042", subscription_id="sub-5", cluster_uuid="uuid-h042", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h043", subscription_id="sub-5", cluster_uuid="uuid-h043", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h043", subscription_id="sub-5", cluster_uuid="uuid-h043", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h044", subscription_id="sub-5", cluster_uuid="uuid-h044", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h044", subscription_id="sub-5", cluster_uuid="uuid-h044", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h045", subscription_id="sub-5", cluster_uuid="uuid-h045", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h045", subscription_id="sub-5", cluster_uuid="uuid-h045", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h046", subscription_id="sub-5", cluster_uuid="uuid-h046", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h046", subscription_id="sub-5", cluster_uuid="uuid-h046", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h047", subscription_id="sub-5", cluster_uuid="uuid-h047", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h047", subscription_id="sub-5", cluster_uuid="uuid-h047", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h048", subscription_id="sub-5", cluster_uuid="uuid-h048", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h048", subscription_id="sub-5", cluster_uuid="uuid-h048", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h049", subscription_id="sub-5", cluster_uuid="uuid-h049", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h049", subscription_id="sub-5", cluster_uuid="uuid-h049", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h050", subscription_id="sub-5", cluster_uuid="uuid-h050", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h050", subscription_id="sub-5", cluster_uuid="uuid-h050", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h051", subscription_id="sub-5", cluster_uuid="uuid-h051", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h051", subscription_id="sub-5", cluster_uuid="uuid-h051", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h052", subscription_id="sub-5", cluster_uuid="uuid-h052", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h052", subscription_id="sub-5", cluster_uuid="uuid-h052", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h053", subscription_id="sub-5", cluster_uuid="uuid-h053", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h053", subscription_id="sub-5", cluster_uuid="uuid-h053", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h054", subscription_id="sub-5", cluster_uuid="uuid-h054", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h054", subscription_id="sub-5", cluster_uuid="uuid-h054", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h055", subscription_id="sub-5", cluster_uuid="uuid-h055", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h055", subscription_id="sub-5", cluster_uuid="uuid-h055", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h056", subscription_id="sub-5", cluster_uuid="uuid-h056", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h056", subscription_id="sub-5", cluster_uuid="uuid-h056", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h057", subscription_id="sub-5", cluster_uuid="uuid-h057", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h057", subscription_id="sub-5", cluster_uuid="uuid-h057", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h058", subscription_id="sub-5", cluster_uuid="uuid-h058", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h058", subscription_id="sub-5", cluster_uuid="uuid-h058", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h059", subscription_id="sub-5", cluster_uuid="uuid-h059", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h059", subscription_id="sub-5", cluster_uuid="uuid-h059", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h060", subscription_id="sub-5", cluster_uuid="uuid-h060", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h060", subscription_id="sub-5", cluster_uuid="uuid-h060", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h061", subscription_id="sub-5", cluster_uuid="uuid-h061", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h061", subscription_id="sub-5", cluster_uuid="uuid-h061", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h062", subscription_id="sub-5", cluster_uuid="uuid-h062", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h062", subscription_id="sub-5", cluster_uuid="uuid-h062", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h063", subscription_id="sub-5", cluster_uuid="uuid-h063", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h063", subscription_id="sub-5", cluster_uuid="uuid-h063", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h064", subscription_id="sub-5", cluster_uuid="uuid-h064", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h064", subscription_id="sub-5", cluster_uuid="uuid-h064", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h065", subscription_id="sub-5", cluster_uuid="uuid-h065", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h065", subscription_id="sub-5", cluster_uuid="uuid-h065", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h066", subscription_id="sub-5", cluster_uuid="uuid-h066", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h066", subscription_id="sub-5", cluster_uuid="uuid-h066", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h067", subscription_id="sub-5", cluster_uuid="uuid-h067", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h067", subscription_id="sub-5", cluster_uuid="uuid-h067", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h068", subscription_id="sub-5", cluster_uuid="uuid-h068", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h068", subscription_id="sub-5", cluster_uuid="uuid-h068", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h069", subscription_id="sub-5", cluster_uuid="uuid-h069", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h069", subscription_id="sub-5", cluster_uuid="uuid-h069", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h070", subscription_id="sub-5", cluster_uuid="uuid-h070", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h070", subscription_id="sub-5", cluster_uuid="uuid-h070", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h071", subscription_id="sub-5", cluster_uuid="uuid-h071", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h071", subscription_id="sub-5", cluster_uuid="uuid-h071", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h072", subscription_id="sub-5", cluster_uuid="uuid-h072", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h072", subscription_id="sub-5", cluster_uuid="uuid-h072", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h073", subscription_id="sub-5", cluster_uuid="uuid-h073", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h073", subscription_id="sub-5", cluster_uuid="uuid-h073", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h074", subscription_id="sub-5", cluster_uuid="uuid-h074", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h074", subscription_id="sub-5", cluster_uuid="uuid-h074", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h075", subscription_id="sub-5", cluster_uuid="uuid-h075", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h075", subscription_id="sub-5", cluster_uuid="uuid-h075", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h076", subscription_id="sub-5", cluster_uuid="uuid-h076", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h076", subscription_id="sub-5", cluster_uuid="uuid-h076", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h077", subscription_id="sub-5", cluster_uuid="uuid-h077", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h077", subscription_id="sub-5", cluster_uuid="uuid-h077", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h078", subscription_id="sub-5", cluster_uuid="uuid-h078", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h078", subscription_id="sub-5", cluster_uuid="uuid-h078", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h079", subscription_id="sub-5", cluster_uuid="uuid-h079", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h079", subscription_id="sub-5", cluster_uuid="uuid-h079", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h080", subscription_id="sub-5", cluster_uuid="uuid-h080", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h080", subscription_id="sub-5", cluster_uuid="uuid-h080", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h081", subscription_id="sub-5", cluster_uuid="uuid-h081", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h081", subscription_id="sub-5", cluster_uuid="uuid-h081", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h082", subscription_id="sub-5", cluster_uuid="uuid-h082", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h082", subscription_id="sub-5", cluster_uuid="uuid-h082", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h083", subscription_id="sub-5", cluster_uuid="uuid-h083", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h083", subscription_id="sub-5", cluster_uuid="uuid-h083", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h084", subscription_id="sub-5", cluster_uuid="uuid-h084", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h084", subscription_id="sub-5", cluster_uuid="uuid-h084", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h085", subscription_id="sub-5", cluster_uuid="uuid-h085", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h085", subscription_id="sub-5", cluster_uuid="uuid-h085", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h086", subscription_id="sub-5", cluster_uuid="uuid-h086", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h086", subscription_id="sub-5", cluster_uuid="uuid-h086", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h087", subscription_id="sub-5", cluster_uuid="uuid-h087", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h087", subscription_id="sub-5", cluster_uuid="uuid-h087", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h088", subscription_id="sub-5", cluster_uuid="uuid-h088", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h088", subscription_id="sub-5", cluster_uuid="uuid-h088", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h089", subscription_id="sub-5", cluster_uuid="uuid-h089", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h089", subscription_id="sub-5", cluster_uuid="uuid-h089", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h090", subscription_id="sub-5", cluster_uuid="uuid-h090", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h090", subscription_id="sub-5", cluster_uuid="uuid-h090", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h091", subscription_id="sub-5", cluster_uuid="uuid-h091", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h091", subscription_id="sub-5", cluster_uuid="uuid-h091", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h092", subscription_id="sub-5", cluster_uuid="uuid-h092", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h092", subscription_id="sub-5", cluster_uuid="uuid-h092", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h093", subscription_id="sub-5", cluster_uuid="uuid-h093", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h093", subscription_id="sub-5", cluster_uuid="uuid-h093", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h094", subscription_id="sub-5", cluster_uuid="uuid-h094", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h094", subscription_id="sub-5", cluster_uuid="uuid-h094", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h095", subscription_id="sub-5", cluster_uuid="uuid-h095", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h095", subscription_id="sub-5", cluster_uuid="uuid-h095", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h096", subscription_id="sub-5", cluster_uuid="uuid-h096", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h096", subscription_id="sub-5", cluster_uuid="uuid-h096", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h097", subscription_id="sub-5", cluster_uuid="uuid-h097", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h097", subscription_id="sub-5", cluster_uuid="uuid-h097", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h098", subscription_id="sub-5", cluster_uuid="uuid-h098", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h098", subscription_id="sub-5", cluster_uuid="uuid-h098", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h099", subscription_id="sub-5", cluster_uuid="uuid-h099", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h099", subscription_id="sub-5", cluster_uuid="uuid-h099", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h100", subscription_id="sub-5", cluster_uuid="uuid-h100", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h100", subscription_id="sub-5", cluster_uuid="uuid-h100", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h101", subscription_id="sub-5", cluster_uuid="uuid-h101", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h101", subscription_id="sub-5", cluster_uuid="uuid-h101", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h102", subscription_id="sub-5", cluster_uuid="uuid-h102", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h102", subscription_id="sub-5", cluster_uuid="uuid-h102", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h103", subscription_id="sub-5", cluster_uuid="uuid-h103", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h103", subscription_id="sub-5", cluster_uuid="uuid-h103", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h104", subscription_id="sub-5", cluster_uuid="uuid-h104", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h104", subscription_id="sub-5", cluster_uuid="uuid-h104", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h105", subscription_id="sub-5", cluster_uuid="uuid-h105", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h105", subscription_id="sub-5", cluster_uuid="uuid-h105", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h106", subscription_id="sub-5", cluster_uuid="uuid-h106", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h106", subscription_id="sub-5", cluster_uuid="uuid-h106", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h107", subscription_id="sub-5", cluster_uuid="uuid-h107", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h107", subscription_id="sub-5", cluster_uuid="uuid-h107", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h108", subscription_id="sub-5", cluster_uuid="uuid-h108", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h108", subscription_id="sub-5", cluster_uuid="uuid-h108", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h109", subscription_id="sub-5", cluster_uuid="uuid-h109", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h109", subscription_id="sub-5", cluster_uuid="uuid-h109", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h110", subscription_id="sub-5", cluster_uuid="uuid-h110", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h110", subscription_id="sub-5", cluster_uuid="uuid-h110", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h111", subscription_id="sub-5", cluster_uuid="uuid-h111", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h111", subscription_id="sub-5", cluster_uuid="uuid-h111", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h112", subscription_id="sub-5", cluster_uuid="uuid-h112", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h112", subscription_id="sub-5", cluster_uuid="uuid-h112", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h113", subscription_id="sub-5", cluster_uuid="uuid-h113", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h113", subscription_id="sub-5", cluster_uuid="uuid-h113", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h114", subscription_id="sub-5", cluster_uuid="uuid-h114", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h114", subscription_id="sub-5", cluster_uuid="uuid-h114", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h115", subscription_id="sub-5", cluster_uuid="uuid-h115", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h115", subscription_id="sub-5", cluster_uuid="uuid-h115", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h116", subscription_id="sub-5", cluster_uuid="uuid-h116", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h116", subscription_id="sub-5", cluster_uuid="uuid-h116", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h117", subscription_id="sub-5", cluster_uuid="uuid-h117", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h117", subscription_id="sub-5", cluster_uuid="uuid-h117", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h118", subscription_id="sub-5", cluster_uuid="uuid-h118", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h118", subscription_id="sub-5", cluster_uuid="uuid-h118", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h119", subscription_id="sub-5", cluster_uuid="uuid-h119", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h119", subscription_id="sub-5", cluster_uuid="uuid-h119", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h120", subscription_id="sub-5", cluster_uuid="uuid-h120", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h120", subscription_id="sub-5", cluster_uuid="uuid-h120", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h121", subscription_id="sub-5", cluster_uuid="uuid-h121", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h121", subscription_id="sub-5", cluster_uuid="uuid-h121", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h122", subscription_id="sub-5", cluster_uuid="uuid-h122", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h122", subscription_id="sub-5", cluster_uuid="uuid-h122", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h123", subscription_id="sub-5", cluster_uuid="uuid-h123", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h123", subscription_id="sub-5", cluster_uuid="uuid-h123", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h124", subscription_id="sub-5", cluster_uuid="uuid-h124", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h124", subscription_id="sub-5", cluster_uuid="uuid-h124", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h125", subscription_id="sub-5", cluster_uuid="uuid-h125", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h125", subscription_id="sub-5", cluster_uuid="uuid-h125", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h126", subscription_id="sub-5", cluster_uuid="uuid-h126", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h126", subscription_id="sub-5", cluster_uuid="uuid-h126", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h127", subscription_id="sub-5", cluster_uuid="uuid-h127", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h127", subscription_id="sub-5", cluster_uuid="uuid-h127", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h128", subscription_id="sub-5", cluster_uuid="uuid-h128", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h128", subscription_id="sub-5", cluster_uuid="uuid-h128", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h129", subscription_id="sub-5", cluster_uuid="uuid-h129", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h129", subscription_id="sub-5", cluster_uuid="uuid-h129", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h130", subscription_id="sub-5", cluster_uuid="uuid-h130", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h130", subscription_id="sub-5", cluster_uuid="uuid-h130", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h131", subscription_id="sub-5", cluster_uuid="uuid-h131", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h131", subscription_id="sub-5", cluster_uuid="uuid-h131", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h132", subscription_id="sub-5", cluster_uuid="uuid-h132", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h132", subscription_id="sub-5", cluster_uuid="uuid-h132", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h133", subscription_id="sub-5", cluster_uuid="uuid-h133", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h133", subscription_id="sub-5", cluster_uuid="uuid-h133", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h134", subscription_id="sub-5", cluster_uuid="uuid-h134", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h134", subscription_id="sub-5", cluster_uuid="uuid-h134", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h135", subscription_id="sub-5", cluster_uuid="uuid-h135", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h135", subscription_id="sub-5", cluster_uuid="uuid-h135", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h136", subscription_id="sub-5", cluster_uuid="uuid-h136", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h136", subscription_id="sub-5", cluster_uuid="uuid-h136", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h137", subscription_id="sub-5", cluster_uuid="uuid-h137", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h137", subscription_id="sub-5", cluster_uuid="uuid-h137", version="4.19.22", state="partial"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h138", subscription_id="sub-5", cluster_uuid="uuid-h138", version="4.19.20", state="completed"}' - values: "1+0x100" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h138", subscription_id="sub-5", cluster_uuid="uuid-h138", version="4.19.22", state="partial"}' - values: "1+0x100" - promql_expr_test: - - expr: hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio{cluster="svc-1"} - eval_time: 90m - exp_samples: - - labels: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio{cluster="svc-1"}' - value: 0.007142857142857143 - - expr: hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate30m{cluster="svc-1"} - eval_time: 90m - exp_samples: - - labels: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate30m{cluster="svc-1"}' - value: 1.4285714285715859e+01 - alert_rule_test: - - eval_time: 90m - alertname: userJourneyClusterUpgradeComplete6h30m - exp_alerts: - - exp_labels: - cluster: "svc-1" - severity: info - slo: cluster-upgrade-complete - short_window: 30m - long_window: 6h - exp_annotations: - summary: "[HCPUpgradeCompleteBurn] svc-1" - description: | - More than 6x of the 99.95% cluster upgrade complete error budget is being consumed. Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. - Window: 6h/30m - Management Cluster: svc-1 - runbook_url: "https://aka.ms/arohcp-runbook-cluster-upgrade" - correlationId: "userJourneyClusterUpgradeComplete/svc-1" - - eval_time: 90m - alertname: userJourneyClusterUpgradeComplete1h5m - - eval_time: 90m - alertname: userJourneyClusterUpgradeComplete3d6h - exp_alerts: - - exp_labels: - cluster: "svc-1" - severity: info - slo: cluster-upgrade-complete - short_window: 6h - long_window: 3d - exp_annotations: - summary: "[HCPUpgradeCompleteBurn] svc-1" - description: | - Cluster upgrade complete has been steadily consuming the 99.95% SLO error budget (1x burn). Cluster upgrades are stuck for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on affected clusters. - Window: 3d/6h - Management Cluster: svc-1 - runbook_url: "https://aka.ms/arohcp-runbook-cluster-upgrade" - correlationId: "userJourneyClusterUpgradeComplete/svc-1" diff --git a/observability/alerts/userJourneyClusterUpgradeMonitor-prometheusRule.yaml b/observability/alerts/userJourneyClusterUpgradeMonitor-prometheusRule.yaml new file mode 100644 index 00000000000..d3b5c9f14ff --- /dev/null +++ b/observability/alerts/userJourneyClusterUpgradeMonitor-prometheusRule.yaml @@ -0,0 +1,51 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + app.kubernetes.io/name: kube-prometheus + app.kubernetes.io/part-of: kube-prometheus + prometheus: k8s + role: alert-rules + name: user-journey-cluster-upgrade-alerts + namespace: monitoring +spec: + groups: + # Per-upgrade stuck alerts for cluster upgrade SLOs. Sev 4 (info). + # At current upgrade volumes, alert directly on duration instead of fleet burn-rate. + # + # In absence of https://issues.redhat.com/browse/ARO-22686 (reporting of control plane + # version history for versions lower than 4.22), cluster upgrade completeness is directly + # impacted by data plane state — e.g. no worker nodes means the upgrade will not reach + # completed; missing console component in external auth prevents CVO from marking the + # upgrade complete. The complete alert may therefore fire due to missing data plane + # configuration rather than control plane failure. The runbook accommodates this. + - name: rp-user-journey-cluster-upgrade-monitor-rules + rules: + - alert: userJourneyClusterUpgradeStuckInDesired + expr: | + hosted_control_plane_upgrade:duration_in_desired:seconds > 1200 + for: 5m + labels: + severity: info + slo: cluster-upgrade-reach-partial + annotations: + summary: "[HCPUpgradeStuckInDesired] {{ $labels.resource_id }}" + description: | + Cluster upgrade target version {{ $labels.version }} has been in desired for over 20 minutes without reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on the affected cluster. + Service Cluster: {{ $labels.cluster }} + runbook_url: https://aka.ms/arohcp-runbook-cluster-upgrade + correlationId: "userJourneyClusterUpgradeStuckInDesired/{{ $labels.resource_id }}/{{ $labels.version }}" + - alert: userJourneyClusterUpgradeStuckInProgress + expr: | + hosted_control_plane_upgrade:duration_in_progress:seconds > 1800 + for: 5m + labels: + severity: info + slo: cluster-upgrade-complete + annotations: + summary: "[HCPUpgradeStuckInProgress] {{ $labels.resource_id }}" + description: | + Cluster upgrade target version {{ $labels.version }} has been in progress for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on the affected cluster. + Service Cluster: {{ $labels.cluster }} + runbook_url: https://aka.ms/arohcp-runbook-cluster-upgrade + correlationId: "userJourneyClusterUpgradeStuckInProgress/{{ $labels.resource_id }}/{{ $labels.version }}" diff --git a/observability/alerts/userJourneyClusterUpgradeMonitor-prometheusRule_test.yaml b/observability/alerts/userJourneyClusterUpgradeMonitor-prometheusRule_test.yaml new file mode 100644 index 00000000000..829f338231c --- /dev/null +++ b/observability/alerts/userJourneyClusterUpgradeMonitor-prometheusRule_test.yaml @@ -0,0 +1,128 @@ +rule_files: +- userJourneyClusterUpgrade-recordingRule.yaml +- userJourneyClusterUpgradeMonitor-prometheusRule.yaml +evaluation_interval: 1m +tests: +# ======================================== +# Reach partial: successful — no alert +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed"}' + values: "1+0x40" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="desired"}' + values: "1x10 0x25" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="partial"}' + values: "0x10 1x25" + alert_rule_test: + - eval_time: 25m + alertname: userJourneyClusterUpgradeStuckInDesired + exp_alerts: [] + - eval_time: 40m + alertname: userJourneyClusterUpgradeStuckInDesired + exp_alerts: [] +# ======================================== +# Reach partial: stuck in desired >20m — alert fires at SLO breach +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.20", state="completed"}' + values: "1+0x50" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.21", state="desired"}' + values: "1+0x50" + alert_rule_test: + - eval_time: 20m + alertname: userJourneyClusterUpgradeStuckInDesired + exp_alerts: [] + - eval_time: 30m + alertname: userJourneyClusterUpgradeStuckInDesired + exp_alerts: + - exp_labels: + alertname: userJourneyClusterUpgradeStuckInDesired + cluster: "svc-1" + cluster_uuid: "uuid-2" + resource_id: "/sub/2/rg/c2" + subscription_id: "sub-2" + version: "4.19.21" + state: "desired" + severity: info + slo: cluster-upgrade-reach-partial + exp_annotations: + summary: "[HCPUpgradeStuckInDesired] /sub/2/rg/c2" + description: | + Cluster upgrade target version 4.19.21 has been in desired for over 20 minutes without reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on the affected cluster. + Service Cluster: svc-1 + runbook_url: "https://aka.ms/arohcp-runbook-cluster-upgrade" + correlationId: "userJourneyClusterUpgradeStuckInDesired//sub/2/rg/c2/4.19.21" +# ======================================== +# Reach partial: install only — not upgrade eligible, no alert +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/c4", subscription_id="sub-3", cluster_uuid="uuid-4", version="4.19.20", state="desired"}' + values: "1+0x50" + alert_rule_test: + - eval_time: 50m + alertname: userJourneyClusterUpgradeStuckInDesired + exp_alerts: [] +# ======================================== +# Upgrade complete: successful — no alert +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed"}' + values: "1+0x40" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="desired"}' + values: "1x10 0x25" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="completed"}' + values: "0x10 1x25" + alert_rule_test: + - eval_time: 40m + alertname: userJourneyClusterUpgradeStuckInProgress + exp_alerts: [] +# ======================================== +# Upgrade complete: stuck in progress >30m — alert fires at SLO breach +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.20", state="completed"}' + values: "1+0x50" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.21", state="desired"}' + values: "1x10 0x50" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.21", state="partial"}' + values: "0x10 1x50" + alert_rule_test: + - eval_time: 30m + alertname: userJourneyClusterUpgradeStuckInProgress + exp_alerts: [] + - eval_time: 40m + alertname: userJourneyClusterUpgradeStuckInProgress + exp_alerts: + - exp_labels: + alertname: userJourneyClusterUpgradeStuckInProgress + cluster: "svc-1" + cluster_uuid: "uuid-2" + resource_id: "/sub/2/rg/c2" + subscription_id: "sub-2" + version: "4.19.21" + state: "partial" + severity: info + slo: cluster-upgrade-complete + exp_annotations: + summary: "[HCPUpgradeStuckInProgress] /sub/2/rg/c2" + description: | + Cluster upgrade target version 4.19.21 has been in progress for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on the affected cluster. + Service Cluster: svc-1 + runbook_url: "https://aka.ms/arohcp-runbook-cluster-upgrade" + correlationId: "userJourneyClusterUpgradeStuckInProgress//sub/2/rg/c2/4.19.21" +# ======================================== +# Upgrade complete: install only — not upgrade eligible, no alert +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/c4", subscription_id="sub-3", cluster_uuid="uuid-4", version="4.19.20", state="desired"}' + values: "1+0x50" + alert_rule_test: + - eval_time: 50m + alertname: userJourneyClusterUpgradeStuckInProgress + exp_alerts: [] diff --git a/observability/alerts/userJourneyClusterUpgradeReachPartialMonitor-prometheusRule.yaml b/observability/alerts/userJourneyClusterUpgradeReachPartialMonitor-prometheusRule.yaml deleted file mode 100644 index ad44173759d..00000000000 --- a/observability/alerts/userJourneyClusterUpgradeReachPartialMonitor-prometheusRule.yaml +++ /dev/null @@ -1,72 +0,0 @@ -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - labels: - app.kubernetes.io/name: kube-prometheus - app.kubernetes.io/part-of: kube-prometheus - prometheus: k8s - role: alert-rules - name: user-journey-cluster-upgrade-reach-partial-alerts - namespace: monitoring -spec: - groups: - # Multi-window burn-rate alerts for cluster upgrade reach-partial SLO. Sev 4 (info). - - name: rp-user-journey-cluster-upgrade-reach-partial-monitor-rules - rules: - - alert: userJourneyClusterUpgradeReachPartial1h5m - expr: | - hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate5m > 14.4 - and - hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate1h > 14.4 - for: 5m - labels: - severity: info - slo: cluster-upgrade-reach-partial - short_window: 5m - long_window: 1h - annotations: - summary: "[HCPUpgradeReachPartialBurn] {{ $labels.cluster }}" - description: | - More than 14.4x of the 99.95% cluster upgrade reach-partial error budget is being consumed. Cluster upgrades are stuck for over 20 minutes without the target version reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. - Window: 1h/5m - Management Cluster: {{ $labels.cluster }} - runbook_url: https://aka.ms/arohcp-runbook-cluster-upgrade - correlationId: "userJourneyClusterUpgradeReachPartial/{{ $labels.cluster }}" - - alert: userJourneyClusterUpgradeReachPartial6h30m - expr: | - hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate30m > 6 - and - hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate6h > 6 - for: 30m - labels: - severity: info - slo: cluster-upgrade-reach-partial - short_window: 30m - long_window: 6h - annotations: - summary: "[HCPUpgradeReachPartialBurn] {{ $labels.cluster }}" - description: | - More than 6x of the 99.95% cluster upgrade reach-partial error budget is being consumed. Cluster upgrades are stuck for over 20 minutes without the target version reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. - Window: 6h/30m - Management Cluster: {{ $labels.cluster }} - runbook_url: https://aka.ms/arohcp-runbook-cluster-upgrade - correlationId: "userJourneyClusterUpgradeReachPartial/{{ $labels.cluster }}" - - alert: userJourneyClusterUpgradeReachPartial3d6h - expr: | - hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate6h > 1 - and - hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate3d > 1 - for: 30m - labels: - severity: info - slo: cluster-upgrade-reach-partial - short_window: 6h - long_window: 3d - annotations: - summary: "[HCPUpgradeReachPartialBurn] {{ $labels.cluster }}" - description: | - Cluster upgrade reach-partial has been steadily consuming the 99.95% SLO error budget (1x burn). Cluster upgrades are stuck for over 20 minutes without reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. - Window: 3d/6h - Management Cluster: {{ $labels.cluster }} - runbook_url: https://aka.ms/arohcp-runbook-cluster-upgrade - correlationId: "userJourneyClusterUpgradeReachPartial/{{ $labels.cluster }}" diff --git a/observability/alerts/userJourneyClusterUpgradeReachPartialMonitor-prometheusRule_test.yaml b/observability/alerts/userJourneyClusterUpgradeReachPartialMonitor-prometheusRule_test.yaml deleted file mode 100644 index e47e887e830..00000000000 --- a/observability/alerts/userJourneyClusterUpgradeReachPartialMonitor-prometheusRule_test.yaml +++ /dev/null @@ -1,778 +0,0 @@ -rule_files: -- userJourneyClusterUpgrade-recordingRule.yaml -- userJourneyClusterUpgradeReachPartialMonitor-prometheusRule.yaml -evaluation_interval: 1m -tests: -# ======================================== -# Test 1: Successful reach partial — no alert -# ======================================== -- interval: 1m - input_series: - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed"}' - values: "1+0x40" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="desired"}' - values: "1x10 0x25" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="partial"}' - values: "0x10 1x25" - alert_rule_test: - - eval_time: 40m - alertname: userJourneyClusterUpgradeReachPartial1h5m - - eval_time: 40m - alertname: userJourneyClusterUpgradeReachPartial6h30m - - eval_time: 40m - alertname: userJourneyClusterUpgradeReachPartial3d6h -# ======================================== -# Test 2: Fast burn — 2 of 10 in-flight upgrades stuck in desired >20m → 5m+1h alert fires -# Eval at 80m: ratio has been 0.2 since minute 20 (all 10 in flight, 8 healthy in partial). -# ======================================== -- interval: 1m - input_series: - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/a", subscription_id="sub-3", cluster_uuid="uuid-a", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/a", subscription_id="sub-3", cluster_uuid="uuid-a", version="4.19.22", state="desired"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/b", subscription_id="sub-3", cluster_uuid="uuid-b", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/b", subscription_id="sub-3", cluster_uuid="uuid-b", version="4.19.22", state="desired"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/c", subscription_id="sub-3", cluster_uuid="uuid-c", version="4.19.19", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/c", subscription_id="sub-3", cluster_uuid="uuid-c", version="4.19.20", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/d", subscription_id="sub-3", cluster_uuid="uuid-d", version="4.19.19", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/d", subscription_id="sub-3", cluster_uuid="uuid-d", version="4.19.20", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/e", subscription_id="sub-3", cluster_uuid="uuid-e", version="4.19.19", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/e", subscription_id="sub-3", cluster_uuid="uuid-e", version="4.19.20", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/f", subscription_id="sub-3", cluster_uuid="uuid-f", version="4.19.19", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/f", subscription_id="sub-3", cluster_uuid="uuid-f", version="4.19.20", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/g", subscription_id="sub-3", cluster_uuid="uuid-g", version="4.19.19", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/g", subscription_id="sub-3", cluster_uuid="uuid-g", version="4.19.20", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/h", subscription_id="sub-3", cluster_uuid="uuid-h", version="4.19.19", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/h", subscription_id="sub-3", cluster_uuid="uuid-h", version="4.19.20", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/i", subscription_id="sub-3", cluster_uuid="uuid-i", version="4.19.19", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/i", subscription_id="sub-3", cluster_uuid="uuid-i", version="4.19.20", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/j", subscription_id="sub-3", cluster_uuid="uuid-j", version="4.19.19", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/j", subscription_id="sub-3", cluster_uuid="uuid-j", version="4.19.20", state="partial"}' - values: "1+0x90" - promql_expr_test: - - expr: hosted_control_plane_upgrade:desired:stuck_over_20m:ratio{cluster="svc-1"} - eval_time: 80m - exp_samples: - - labels: 'hosted_control_plane_upgrade:desired:stuck_over_20m:ratio{cluster="svc-1"}' - value: 0.2 - - expr: hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate5m{cluster="svc-1"} - eval_time: 80m - exp_samples: - - labels: 'hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate5m{cluster="svc-1"}' - value: 4.0000000000004405e+02 - - expr: hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate1h{cluster="svc-1"} - eval_time: 80m - exp_samples: - - labels: 'hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate1h{cluster="svc-1"}' - value: 4.0000000000004405e+02 - alert_rule_test: - - eval_time: 80m - alertname: userJourneyClusterUpgradeReachPartial1h5m - exp_alerts: - - exp_labels: - cluster: "svc-1" - severity: info - slo: cluster-upgrade-reach-partial - short_window: 5m - long_window: 1h - exp_annotations: - summary: "[HCPUpgradeReachPartialBurn] svc-1" - description: | - More than 14.4x of the 99.95% cluster upgrade reach-partial error budget is being consumed. Cluster upgrades are stuck for over 20 minutes without the target version reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. - Window: 1h/5m - Management Cluster: svc-1 - runbook_url: "https://aka.ms/arohcp-runbook-cluster-upgrade" - correlationId: "userJourneyClusterUpgradeReachPartial/svc-1" - - eval_time: 80m - alertname: userJourneyClusterUpgradeReachPartial6h30m - exp_alerts: - - exp_labels: - cluster: "svc-1" - severity: info - slo: cluster-upgrade-reach-partial - short_window: 30m - long_window: 6h - exp_annotations: - summary: "[HCPUpgradeReachPartialBurn] svc-1" - description: | - More than 6x of the 99.95% cluster upgrade reach-partial error budget is being consumed. Cluster upgrades are stuck for over 20 minutes without the target version reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. - Window: 6h/30m - Management Cluster: svc-1 - runbook_url: "https://aka.ms/arohcp-runbook-cluster-upgrade" - correlationId: "userJourneyClusterUpgradeReachPartial/svc-1" - - eval_time: 80m - alertname: userJourneyClusterUpgradeReachPartial3d6h - exp_alerts: - - exp_labels: - cluster: "svc-1" - severity: info - slo: cluster-upgrade-reach-partial - short_window: 6h - long_window: 3d - exp_annotations: - summary: "[HCPUpgradeReachPartialBurn] svc-1" - description: | - Cluster upgrade reach-partial has been steadily consuming the 99.95% SLO error budget (1x burn). Cluster upgrades are stuck for over 20 minutes without reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. - Window: 3d/6h - Management Cluster: svc-1 - runbook_url: "https://aka.ms/arohcp-runbook-cluster-upgrade" - correlationId: "userJourneyClusterUpgradeReachPartial/svc-1" -# ======================================== -# Test 3: Fleet healthy — no SLO alert -# ======================================== -- interval: 1m - input_series: - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/4/rg/a", subscription_id="sub-4", cluster_uuid="uuid-a", version="4.19.20", state="completed"}' - values: "1+0x40" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/4/rg/b", subscription_id="sub-4", cluster_uuid="uuid-b", version="4.19.20", state="completed"}' - values: "1+0x40" - promql_expr_test: - - expr: hosted_control_plane_upgrade:desired:stuck_over_20m:ratio{cluster="svc-1"} - eval_time: 10m - exp_samples: - - labels: 'hosted_control_plane_upgrade:desired:stuck_over_20m:ratio{cluster="svc-1"}' - value: 0 - alert_rule_test: - - eval_time: 40m - alertname: userJourneyClusterUpgradeReachPartial1h5m - - eval_time: 40m - alertname: userJourneyClusterUpgradeReachPartial6h30m - - eval_time: 40m - alertname: userJourneyClusterUpgradeReachPartial3d6h -# ======================================== -# Test 4: Medium burn — 1 of 140 stuck >20m → 6h/30m alert fires, 1h/5m does not -# Ratio 1/140 ≈ 0.71% → burn ≈ 14.3 (< 14.4 fast, > 6 medium). -# ======================================== -- interval: 1m - input_series: - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/stuck", subscription_id="sub-5", cluster_uuid="uuid-stuck", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/stuck", subscription_id="sub-5", cluster_uuid="uuid-stuck", version="4.19.22", state="desired"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h000", subscription_id="sub-5", cluster_uuid="uuid-h000", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h000", subscription_id="sub-5", cluster_uuid="uuid-h000", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h001", subscription_id="sub-5", cluster_uuid="uuid-h001", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h001", subscription_id="sub-5", cluster_uuid="uuid-h001", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h002", subscription_id="sub-5", cluster_uuid="uuid-h002", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h002", subscription_id="sub-5", cluster_uuid="uuid-h002", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h003", subscription_id="sub-5", cluster_uuid="uuid-h003", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h003", subscription_id="sub-5", cluster_uuid="uuid-h003", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h004", subscription_id="sub-5", cluster_uuid="uuid-h004", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h004", subscription_id="sub-5", cluster_uuid="uuid-h004", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h005", subscription_id="sub-5", cluster_uuid="uuid-h005", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h005", subscription_id="sub-5", cluster_uuid="uuid-h005", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h006", subscription_id="sub-5", cluster_uuid="uuid-h006", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h006", subscription_id="sub-5", cluster_uuid="uuid-h006", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h007", subscription_id="sub-5", cluster_uuid="uuid-h007", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h007", subscription_id="sub-5", cluster_uuid="uuid-h007", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h008", subscription_id="sub-5", cluster_uuid="uuid-h008", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h008", subscription_id="sub-5", cluster_uuid="uuid-h008", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h009", subscription_id="sub-5", cluster_uuid="uuid-h009", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h009", subscription_id="sub-5", cluster_uuid="uuid-h009", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h010", subscription_id="sub-5", cluster_uuid="uuid-h010", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h010", subscription_id="sub-5", cluster_uuid="uuid-h010", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h011", subscription_id="sub-5", cluster_uuid="uuid-h011", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h011", subscription_id="sub-5", cluster_uuid="uuid-h011", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h012", subscription_id="sub-5", cluster_uuid="uuid-h012", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h012", subscription_id="sub-5", cluster_uuid="uuid-h012", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h013", subscription_id="sub-5", cluster_uuid="uuid-h013", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h013", subscription_id="sub-5", cluster_uuid="uuid-h013", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h014", subscription_id="sub-5", cluster_uuid="uuid-h014", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h014", subscription_id="sub-5", cluster_uuid="uuid-h014", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h015", subscription_id="sub-5", cluster_uuid="uuid-h015", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h015", subscription_id="sub-5", cluster_uuid="uuid-h015", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h016", subscription_id="sub-5", cluster_uuid="uuid-h016", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h016", subscription_id="sub-5", cluster_uuid="uuid-h016", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h017", subscription_id="sub-5", cluster_uuid="uuid-h017", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h017", subscription_id="sub-5", cluster_uuid="uuid-h017", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h018", subscription_id="sub-5", cluster_uuid="uuid-h018", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h018", subscription_id="sub-5", cluster_uuid="uuid-h018", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h019", subscription_id="sub-5", cluster_uuid="uuid-h019", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h019", subscription_id="sub-5", cluster_uuid="uuid-h019", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h020", subscription_id="sub-5", cluster_uuid="uuid-h020", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h020", subscription_id="sub-5", cluster_uuid="uuid-h020", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h021", subscription_id="sub-5", cluster_uuid="uuid-h021", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h021", subscription_id="sub-5", cluster_uuid="uuid-h021", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h022", subscription_id="sub-5", cluster_uuid="uuid-h022", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h022", subscription_id="sub-5", cluster_uuid="uuid-h022", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h023", subscription_id="sub-5", cluster_uuid="uuid-h023", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h023", subscription_id="sub-5", cluster_uuid="uuid-h023", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h024", subscription_id="sub-5", cluster_uuid="uuid-h024", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h024", subscription_id="sub-5", cluster_uuid="uuid-h024", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h025", subscription_id="sub-5", cluster_uuid="uuid-h025", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h025", subscription_id="sub-5", cluster_uuid="uuid-h025", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h026", subscription_id="sub-5", cluster_uuid="uuid-h026", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h026", subscription_id="sub-5", cluster_uuid="uuid-h026", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h027", subscription_id="sub-5", cluster_uuid="uuid-h027", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h027", subscription_id="sub-5", cluster_uuid="uuid-h027", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h028", subscription_id="sub-5", cluster_uuid="uuid-h028", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h028", subscription_id="sub-5", cluster_uuid="uuid-h028", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h029", subscription_id="sub-5", cluster_uuid="uuid-h029", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h029", subscription_id="sub-5", cluster_uuid="uuid-h029", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h030", subscription_id="sub-5", cluster_uuid="uuid-h030", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h030", subscription_id="sub-5", cluster_uuid="uuid-h030", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h031", subscription_id="sub-5", cluster_uuid="uuid-h031", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h031", subscription_id="sub-5", cluster_uuid="uuid-h031", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h032", subscription_id="sub-5", cluster_uuid="uuid-h032", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h032", subscription_id="sub-5", cluster_uuid="uuid-h032", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h033", subscription_id="sub-5", cluster_uuid="uuid-h033", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h033", subscription_id="sub-5", cluster_uuid="uuid-h033", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h034", subscription_id="sub-5", cluster_uuid="uuid-h034", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h034", subscription_id="sub-5", cluster_uuid="uuid-h034", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h035", subscription_id="sub-5", cluster_uuid="uuid-h035", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h035", subscription_id="sub-5", cluster_uuid="uuid-h035", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h036", subscription_id="sub-5", cluster_uuid="uuid-h036", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h036", subscription_id="sub-5", cluster_uuid="uuid-h036", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h037", subscription_id="sub-5", cluster_uuid="uuid-h037", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h037", subscription_id="sub-5", cluster_uuid="uuid-h037", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h038", subscription_id="sub-5", cluster_uuid="uuid-h038", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h038", subscription_id="sub-5", cluster_uuid="uuid-h038", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h039", subscription_id="sub-5", cluster_uuid="uuid-h039", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h039", subscription_id="sub-5", cluster_uuid="uuid-h039", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h040", subscription_id="sub-5", cluster_uuid="uuid-h040", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h040", subscription_id="sub-5", cluster_uuid="uuid-h040", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h041", subscription_id="sub-5", cluster_uuid="uuid-h041", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h041", subscription_id="sub-5", cluster_uuid="uuid-h041", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h042", subscription_id="sub-5", cluster_uuid="uuid-h042", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h042", subscription_id="sub-5", cluster_uuid="uuid-h042", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h043", subscription_id="sub-5", cluster_uuid="uuid-h043", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h043", subscription_id="sub-5", cluster_uuid="uuid-h043", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h044", subscription_id="sub-5", cluster_uuid="uuid-h044", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h044", subscription_id="sub-5", cluster_uuid="uuid-h044", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h045", subscription_id="sub-5", cluster_uuid="uuid-h045", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h045", subscription_id="sub-5", cluster_uuid="uuid-h045", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h046", subscription_id="sub-5", cluster_uuid="uuid-h046", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h046", subscription_id="sub-5", cluster_uuid="uuid-h046", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h047", subscription_id="sub-5", cluster_uuid="uuid-h047", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h047", subscription_id="sub-5", cluster_uuid="uuid-h047", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h048", subscription_id="sub-5", cluster_uuid="uuid-h048", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h048", subscription_id="sub-5", cluster_uuid="uuid-h048", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h049", subscription_id="sub-5", cluster_uuid="uuid-h049", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h049", subscription_id="sub-5", cluster_uuid="uuid-h049", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h050", subscription_id="sub-5", cluster_uuid="uuid-h050", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h050", subscription_id="sub-5", cluster_uuid="uuid-h050", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h051", subscription_id="sub-5", cluster_uuid="uuid-h051", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h051", subscription_id="sub-5", cluster_uuid="uuid-h051", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h052", subscription_id="sub-5", cluster_uuid="uuid-h052", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h052", subscription_id="sub-5", cluster_uuid="uuid-h052", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h053", subscription_id="sub-5", cluster_uuid="uuid-h053", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h053", subscription_id="sub-5", cluster_uuid="uuid-h053", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h054", subscription_id="sub-5", cluster_uuid="uuid-h054", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h054", subscription_id="sub-5", cluster_uuid="uuid-h054", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h055", subscription_id="sub-5", cluster_uuid="uuid-h055", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h055", subscription_id="sub-5", cluster_uuid="uuid-h055", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h056", subscription_id="sub-5", cluster_uuid="uuid-h056", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h056", subscription_id="sub-5", cluster_uuid="uuid-h056", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h057", subscription_id="sub-5", cluster_uuid="uuid-h057", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h057", subscription_id="sub-5", cluster_uuid="uuid-h057", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h058", subscription_id="sub-5", cluster_uuid="uuid-h058", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h058", subscription_id="sub-5", cluster_uuid="uuid-h058", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h059", subscription_id="sub-5", cluster_uuid="uuid-h059", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h059", subscription_id="sub-5", cluster_uuid="uuid-h059", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h060", subscription_id="sub-5", cluster_uuid="uuid-h060", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h060", subscription_id="sub-5", cluster_uuid="uuid-h060", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h061", subscription_id="sub-5", cluster_uuid="uuid-h061", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h061", subscription_id="sub-5", cluster_uuid="uuid-h061", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h062", subscription_id="sub-5", cluster_uuid="uuid-h062", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h062", subscription_id="sub-5", cluster_uuid="uuid-h062", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h063", subscription_id="sub-5", cluster_uuid="uuid-h063", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h063", subscription_id="sub-5", cluster_uuid="uuid-h063", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h064", subscription_id="sub-5", cluster_uuid="uuid-h064", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h064", subscription_id="sub-5", cluster_uuid="uuid-h064", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h065", subscription_id="sub-5", cluster_uuid="uuid-h065", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h065", subscription_id="sub-5", cluster_uuid="uuid-h065", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h066", subscription_id="sub-5", cluster_uuid="uuid-h066", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h066", subscription_id="sub-5", cluster_uuid="uuid-h066", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h067", subscription_id="sub-5", cluster_uuid="uuid-h067", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h067", subscription_id="sub-5", cluster_uuid="uuid-h067", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h068", subscription_id="sub-5", cluster_uuid="uuid-h068", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h068", subscription_id="sub-5", cluster_uuid="uuid-h068", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h069", subscription_id="sub-5", cluster_uuid="uuid-h069", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h069", subscription_id="sub-5", cluster_uuid="uuid-h069", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h070", subscription_id="sub-5", cluster_uuid="uuid-h070", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h070", subscription_id="sub-5", cluster_uuid="uuid-h070", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h071", subscription_id="sub-5", cluster_uuid="uuid-h071", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h071", subscription_id="sub-5", cluster_uuid="uuid-h071", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h072", subscription_id="sub-5", cluster_uuid="uuid-h072", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h072", subscription_id="sub-5", cluster_uuid="uuid-h072", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h073", subscription_id="sub-5", cluster_uuid="uuid-h073", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h073", subscription_id="sub-5", cluster_uuid="uuid-h073", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h074", subscription_id="sub-5", cluster_uuid="uuid-h074", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h074", subscription_id="sub-5", cluster_uuid="uuid-h074", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h075", subscription_id="sub-5", cluster_uuid="uuid-h075", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h075", subscription_id="sub-5", cluster_uuid="uuid-h075", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h076", subscription_id="sub-5", cluster_uuid="uuid-h076", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h076", subscription_id="sub-5", cluster_uuid="uuid-h076", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h077", subscription_id="sub-5", cluster_uuid="uuid-h077", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h077", subscription_id="sub-5", cluster_uuid="uuid-h077", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h078", subscription_id="sub-5", cluster_uuid="uuid-h078", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h078", subscription_id="sub-5", cluster_uuid="uuid-h078", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h079", subscription_id="sub-5", cluster_uuid="uuid-h079", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h079", subscription_id="sub-5", cluster_uuid="uuid-h079", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h080", subscription_id="sub-5", cluster_uuid="uuid-h080", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h080", subscription_id="sub-5", cluster_uuid="uuid-h080", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h081", subscription_id="sub-5", cluster_uuid="uuid-h081", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h081", subscription_id="sub-5", cluster_uuid="uuid-h081", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h082", subscription_id="sub-5", cluster_uuid="uuid-h082", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h082", subscription_id="sub-5", cluster_uuid="uuid-h082", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h083", subscription_id="sub-5", cluster_uuid="uuid-h083", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h083", subscription_id="sub-5", cluster_uuid="uuid-h083", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h084", subscription_id="sub-5", cluster_uuid="uuid-h084", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h084", subscription_id="sub-5", cluster_uuid="uuid-h084", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h085", subscription_id="sub-5", cluster_uuid="uuid-h085", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h085", subscription_id="sub-5", cluster_uuid="uuid-h085", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h086", subscription_id="sub-5", cluster_uuid="uuid-h086", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h086", subscription_id="sub-5", cluster_uuid="uuid-h086", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h087", subscription_id="sub-5", cluster_uuid="uuid-h087", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h087", subscription_id="sub-5", cluster_uuid="uuid-h087", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h088", subscription_id="sub-5", cluster_uuid="uuid-h088", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h088", subscription_id="sub-5", cluster_uuid="uuid-h088", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h089", subscription_id="sub-5", cluster_uuid="uuid-h089", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h089", subscription_id="sub-5", cluster_uuid="uuid-h089", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h090", subscription_id="sub-5", cluster_uuid="uuid-h090", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h090", subscription_id="sub-5", cluster_uuid="uuid-h090", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h091", subscription_id="sub-5", cluster_uuid="uuid-h091", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h091", subscription_id="sub-5", cluster_uuid="uuid-h091", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h092", subscription_id="sub-5", cluster_uuid="uuid-h092", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h092", subscription_id="sub-5", cluster_uuid="uuid-h092", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h093", subscription_id="sub-5", cluster_uuid="uuid-h093", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h093", subscription_id="sub-5", cluster_uuid="uuid-h093", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h094", subscription_id="sub-5", cluster_uuid="uuid-h094", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h094", subscription_id="sub-5", cluster_uuid="uuid-h094", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h095", subscription_id="sub-5", cluster_uuid="uuid-h095", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h095", subscription_id="sub-5", cluster_uuid="uuid-h095", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h096", subscription_id="sub-5", cluster_uuid="uuid-h096", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h096", subscription_id="sub-5", cluster_uuid="uuid-h096", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h097", subscription_id="sub-5", cluster_uuid="uuid-h097", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h097", subscription_id="sub-5", cluster_uuid="uuid-h097", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h098", subscription_id="sub-5", cluster_uuid="uuid-h098", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h098", subscription_id="sub-5", cluster_uuid="uuid-h098", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h099", subscription_id="sub-5", cluster_uuid="uuid-h099", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h099", subscription_id="sub-5", cluster_uuid="uuid-h099", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h100", subscription_id="sub-5", cluster_uuid="uuid-h100", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h100", subscription_id="sub-5", cluster_uuid="uuid-h100", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h101", subscription_id="sub-5", cluster_uuid="uuid-h101", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h101", subscription_id="sub-5", cluster_uuid="uuid-h101", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h102", subscription_id="sub-5", cluster_uuid="uuid-h102", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h102", subscription_id="sub-5", cluster_uuid="uuid-h102", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h103", subscription_id="sub-5", cluster_uuid="uuid-h103", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h103", subscription_id="sub-5", cluster_uuid="uuid-h103", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h104", subscription_id="sub-5", cluster_uuid="uuid-h104", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h104", subscription_id="sub-5", cluster_uuid="uuid-h104", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h105", subscription_id="sub-5", cluster_uuid="uuid-h105", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h105", subscription_id="sub-5", cluster_uuid="uuid-h105", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h106", subscription_id="sub-5", cluster_uuid="uuid-h106", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h106", subscription_id="sub-5", cluster_uuid="uuid-h106", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h107", subscription_id="sub-5", cluster_uuid="uuid-h107", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h107", subscription_id="sub-5", cluster_uuid="uuid-h107", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h108", subscription_id="sub-5", cluster_uuid="uuid-h108", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h108", subscription_id="sub-5", cluster_uuid="uuid-h108", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h109", subscription_id="sub-5", cluster_uuid="uuid-h109", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h109", subscription_id="sub-5", cluster_uuid="uuid-h109", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h110", subscription_id="sub-5", cluster_uuid="uuid-h110", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h110", subscription_id="sub-5", cluster_uuid="uuid-h110", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h111", subscription_id="sub-5", cluster_uuid="uuid-h111", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h111", subscription_id="sub-5", cluster_uuid="uuid-h111", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h112", subscription_id="sub-5", cluster_uuid="uuid-h112", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h112", subscription_id="sub-5", cluster_uuid="uuid-h112", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h113", subscription_id="sub-5", cluster_uuid="uuid-h113", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h113", subscription_id="sub-5", cluster_uuid="uuid-h113", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h114", subscription_id="sub-5", cluster_uuid="uuid-h114", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h114", subscription_id="sub-5", cluster_uuid="uuid-h114", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h115", subscription_id="sub-5", cluster_uuid="uuid-h115", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h115", subscription_id="sub-5", cluster_uuid="uuid-h115", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h116", subscription_id="sub-5", cluster_uuid="uuid-h116", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h116", subscription_id="sub-5", cluster_uuid="uuid-h116", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h117", subscription_id="sub-5", cluster_uuid="uuid-h117", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h117", subscription_id="sub-5", cluster_uuid="uuid-h117", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h118", subscription_id="sub-5", cluster_uuid="uuid-h118", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h118", subscription_id="sub-5", cluster_uuid="uuid-h118", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h119", subscription_id="sub-5", cluster_uuid="uuid-h119", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h119", subscription_id="sub-5", cluster_uuid="uuid-h119", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h120", subscription_id="sub-5", cluster_uuid="uuid-h120", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h120", subscription_id="sub-5", cluster_uuid="uuid-h120", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h121", subscription_id="sub-5", cluster_uuid="uuid-h121", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h121", subscription_id="sub-5", cluster_uuid="uuid-h121", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h122", subscription_id="sub-5", cluster_uuid="uuid-h122", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h122", subscription_id="sub-5", cluster_uuid="uuid-h122", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h123", subscription_id="sub-5", cluster_uuid="uuid-h123", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h123", subscription_id="sub-5", cluster_uuid="uuid-h123", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h124", subscription_id="sub-5", cluster_uuid="uuid-h124", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h124", subscription_id="sub-5", cluster_uuid="uuid-h124", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h125", subscription_id="sub-5", cluster_uuid="uuid-h125", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h125", subscription_id="sub-5", cluster_uuid="uuid-h125", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h126", subscription_id="sub-5", cluster_uuid="uuid-h126", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h126", subscription_id="sub-5", cluster_uuid="uuid-h126", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h127", subscription_id="sub-5", cluster_uuid="uuid-h127", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h127", subscription_id="sub-5", cluster_uuid="uuid-h127", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h128", subscription_id="sub-5", cluster_uuid="uuid-h128", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h128", subscription_id="sub-5", cluster_uuid="uuid-h128", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h129", subscription_id="sub-5", cluster_uuid="uuid-h129", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h129", subscription_id="sub-5", cluster_uuid="uuid-h129", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h130", subscription_id="sub-5", cluster_uuid="uuid-h130", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h130", subscription_id="sub-5", cluster_uuid="uuid-h130", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h131", subscription_id="sub-5", cluster_uuid="uuid-h131", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h131", subscription_id="sub-5", cluster_uuid="uuid-h131", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h132", subscription_id="sub-5", cluster_uuid="uuid-h132", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h132", subscription_id="sub-5", cluster_uuid="uuid-h132", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h133", subscription_id="sub-5", cluster_uuid="uuid-h133", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h133", subscription_id="sub-5", cluster_uuid="uuid-h133", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h134", subscription_id="sub-5", cluster_uuid="uuid-h134", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h134", subscription_id="sub-5", cluster_uuid="uuid-h134", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h135", subscription_id="sub-5", cluster_uuid="uuid-h135", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h135", subscription_id="sub-5", cluster_uuid="uuid-h135", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h136", subscription_id="sub-5", cluster_uuid="uuid-h136", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h136", subscription_id="sub-5", cluster_uuid="uuid-h136", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h137", subscription_id="sub-5", cluster_uuid="uuid-h137", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h137", subscription_id="sub-5", cluster_uuid="uuid-h137", version="4.19.22", state="partial"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h138", subscription_id="sub-5", cluster_uuid="uuid-h138", version="4.19.20", state="completed"}' - values: "1+0x90" - - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/h138", subscription_id="sub-5", cluster_uuid="uuid-h138", version="4.19.22", state="partial"}' - values: "1+0x90" - promql_expr_test: - - expr: hosted_control_plane_upgrade:desired:stuck_over_20m:ratio{cluster="svc-1"} - eval_time: 80m - exp_samples: - - labels: 'hosted_control_plane_upgrade:desired:stuck_over_20m:ratio{cluster="svc-1"}' - value: 0.007142857142857143 - - expr: hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate30m{cluster="svc-1"} - eval_time: 80m - exp_samples: - - labels: 'hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate30m{cluster="svc-1"}' - value: 1.4285714285715859e+01 - - expr: hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate5m{cluster="svc-1"} - eval_time: 80m - exp_samples: - - labels: 'hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate5m{cluster="svc-1"}' - value: 1.4285714285715859e+01 - alert_rule_test: - - eval_time: 80m - alertname: userJourneyClusterUpgradeReachPartial6h30m - exp_alerts: - - exp_labels: - cluster: "svc-1" - severity: info - slo: cluster-upgrade-reach-partial - short_window: 30m - long_window: 6h - exp_annotations: - summary: "[HCPUpgradeReachPartialBurn] svc-1" - description: | - More than 6x of the 99.95% cluster upgrade reach-partial error budget is being consumed. Cluster upgrades are stuck for over 20 minutes without the target version reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. - Window: 6h/30m - Management Cluster: svc-1 - runbook_url: "https://aka.ms/arohcp-runbook-cluster-upgrade" - correlationId: "userJourneyClusterUpgradeReachPartial/svc-1" - - eval_time: 80m - alertname: userJourneyClusterUpgradeReachPartial1h5m - - eval_time: 80m - alertname: userJourneyClusterUpgradeReachPartial3d6h - exp_alerts: - - exp_labels: - cluster: "svc-1" - severity: info - slo: cluster-upgrade-reach-partial - short_window: 6h - long_window: 3d - exp_annotations: - summary: "[HCPUpgradeReachPartialBurn] svc-1" - description: | - Cluster upgrade reach-partial has been steadily consuming the 99.95% SLO error budget (1x burn). Cluster upgrades are stuck for over 20 minutes without reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on affected clusters. - Window: 3d/6h - Management Cluster: svc-1 - runbook_url: "https://aka.ms/arohcp-runbook-cluster-upgrade" - correlationId: "userJourneyClusterUpgradeReachPartial/svc-1" diff --git a/observability/grafana-dashboards/sre/user-journey/cluster-upgrade-slo-dashboard.json b/observability/grafana-dashboards/sre/user-journey/cluster-upgrade-slo-dashboard.json deleted file mode 100644 index 5acfd1b6ec9..00000000000 --- a/observability/grafana-dashboards/sre/user-journey/cluster-upgrade-slo-dashboard.json +++ /dev/null @@ -1,1490 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": null, - "links": [], - "panels": [ - { - "gridPos": { - "h": 5, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "### Cluster Upgrade SLOs\n\nFleet-wide observability for HCP cluster version upgrades from `backend_cluster_version_info`.\n\n| SLO | Target | Window |\n|-----|--------|--------|\n| **Reach partial** | 99.95% reach `partial` | 20 minutes |\n| **Upgrade complete** | 99.95% reach `completed` | 30 minutes |\n\n**Reach partial:** Time for the target version to become active (recognized as active). Indicates the upgrade is in progress on the HCP cluster but not yet complete.\n\n**Error budget:** 0.05% of in-flight upgrades (stuck / in-flight count).\n\n**Metrics:** `hosted_control_plane_upgrade:*` recording rules.\n\n**TSG:** [Backend TSG](https://eng.ms/docs/cloud-ai-platform/azure-core/azure-cloud-native-and-management-platform/control-plane-bburns/azure-red-hat-openshift/azure-redhat-openshift-team-doc/hcp/troubleshooting/backend-tsg.html)", - "mode": "markdown" - }, - "pluginVersion": "11.6.9", - "title": "Dashboard Info", - "type": "text" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 2, - "panels": [], - "title": "Reach Partial (20m SLO)", - "type": "row" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "description": "SLI: share of in-flight upgrades not contributing to stuck rate (1 − stuck/in-flight).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 3, - "mappings": [], - "max": 100, - "min": 99.9, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red" - }, - { - "color": "yellow", - "value": 99.95 - }, - { - "color": "green", - "value": 99.99 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 6 - }, - "id": 3, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "(1 - (hosted_control_plane_upgrade:desired:stuck_over_20m:ratio{cluster=~\"$cluster\"} or vector(0))) * 100", - "instant": true, - "legendFormat": "SLI %", - "refId": "A" - } - ], - "title": "Cluster Upgrade Version Reach Partial SLI", - "type": "stat" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "description": "HCP clusters with any reported cluster version state.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 6 - }, - "id": 4, - "options": { - "colorMode": "background", - "graphMode": "none", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "sum(hosted_control_plane_upgrade:in_progress:count{cluster=~\"$cluster\"}) or vector(0)", - "instant": true, - "legendFormat": "In-flight upgrades", - "refId": "A" - } - ], - "title": "In-Flight Upgrades", - "type": "stat" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "description": "Upgrades stuck in desired for over 20 minutes without activating.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 5 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 6 - }, - "id": 5, - "options": { - "colorMode": "background", - "graphMode": "none", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "sum(hosted_control_plane_upgrade:desired:stuck_over_20m:count{cluster=~\"$cluster\"}) or vector(0)", - "instant": true, - "legendFormat": "Stuck clusters", - "refId": "A" - } - ], - "title": "Stuck in Desired Over 20m", - "type": "stat" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "description": "Current 5 minute error budget burn rate. Fast burn alert threshold: 14.4x.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 2, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "yellow", - "value": 6 - }, - { - "color": "red", - "value": 14.4 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 6 - }, - "id": 6, - "options": { - "colorMode": "background", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "max(hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate5m{cluster=~\"$cluster\"}) or vector(0)", - "instant": true, - "legendFormat": "5m burn", - "refId": "A" - } - ], - "title": "Burn Rate (5m)", - "type": "stat" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "description": "Fraction of in-flight upgrades stuck past 20 minutes. SLO error budget line at 0.05%.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 10, - "lineInterpolation": "smooth", - "lineWidth": 2, - "showPoints": "never", - "thresholdsStyle": { - "mode": "line" - } - }, - "decimals": 4, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 0.0005 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 12 - }, - "id": 7, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "hosted_control_plane_upgrade:desired:stuck_over_20m:ratio{cluster=~\"$cluster\"}", - "legendFormat": "{{ cluster }} in-flight stuck rate", - "refId": "A" - } - ], - "title": "In-Flight Stuck Rate Over Time", - "type": "timeseries" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "description": "Multi-window error budget burn rates for cluster upgrade reach-partial SLO.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 0, - "lineInterpolation": "smooth", - "lineWidth": 2, - "showPoints": "never" - }, - "decimals": 2, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 12 - }, - "id": 8, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate5m{cluster=~\"$cluster\"}", - "legendFormat": "5m {{ cluster }}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate30m{cluster=~\"$cluster\"}", - "legendFormat": "30m {{ cluster }}", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate1h{cluster=~\"$cluster\"}", - "legendFormat": "1h {{ cluster }}", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate6h{cluster=~\"$cluster\"}", - "legendFormat": "6h {{ cluster }}", - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate3d{cluster=~\"$cluster\"}", - "legendFormat": "3d {{ cluster }}", - "refId": "E" - } - ], - "title": "Error Budget Burn Rates", - "type": "timeseries" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "description": "Time from desired first-seen to first active (partial or completed) observation, in minutes.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 0, - "lineInterpolation": "smooth", - "lineWidth": 2, - "showPoints": "never", - "thresholdsStyle": { - "mode": "line" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 20 - } - ] - }, - "unit": "m" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 20 - }, - "id": 9, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "(\n (\n min without(state) (hosted_control_plane_upgrade:version_state_first_seen:timestamp{cluster=~\"$cluster\", state=\"partial\"})\n - on(cluster, resource_id, subscription_id, cluster_uuid, version) group_left()\n hosted_control_plane_upgrade:version_state_first_seen:timestamp{cluster=~\"$cluster\", state=\"desired\"}\n )\n and on(cluster, resource_id, subscription_id, cluster_uuid, version) (\n max without(state) (backend_cluster_version_info{cluster=~\"$cluster\", state=\"partial\"}) == 1\n )\n) / 60", - "legendFormat": "{{ resource_id }} {{ version }}", - "refId": "A" - } - ], - "title": "Cluster Upgrade Version Reach Partial Duration (Minutes)", - "type": "timeseries" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "description": "Clusters currently waiting in desired state (minutes since target version was selected).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 10, - "lineInterpolation": "smooth", - "lineWidth": 2, - "showPoints": "never", - "thresholdsStyle": { - "mode": "line" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 20 - } - ] - }, - "unit": "m" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 20 - }, - "id": 10, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "hosted_control_plane_upgrade:duration_in_desired:seconds{cluster=~\"$cluster\"} / 60", - "legendFormat": "{{ resource_id }} {{ version }}", - "refId": "A" - } - ], - "title": "Time in Desired (Minutes)", - "type": "timeseries" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "description": "Clusters with cluster upgrades currently stuck past the 20 minute reach-partial window.", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 11, - "options": { - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "version" - } - ] - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "(\n hosted_control_plane_upgrade:duration_in_desired:seconds{cluster=~\"$cluster\"}\n and\n hosted_control_plane_upgrade:duration_in_desired:seconds{cluster=~\"$cluster\"} > 1200\n) / 60", - "format": "table", - "instant": true, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Stuck in Desired Detail", - "transformations": [ - { - "id": "seriesToRows", - "options": {} - }, - { - "id": "merge", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "__name__": true, - "state": true - }, - "renameByName": { - "Value": "Time in Desired (Minutes)", - "cluster": "Service Cluster", - "cluster_uuid": "Cluster UUID", - "resource_id": "Resource ID", - "subscription_id": "Subscription", - "version": "Target Version" - } - } - } - ], - "type": "table" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 36 - }, - "id": 12, - "panels": [], - "title": "Upgrade Complete (30m SLO)", - "type": "row" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "description": "SLI: share of in-flight upgrades not contributing to stuck rate (1 − stuck/in-flight).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 3, - "mappings": [], - "max": 100, - "min": 99.9, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red" - }, - { - "color": "yellow", - "value": 99.95 - }, - { - "color": "green", - "value": 99.99 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 37 - }, - "id": 13, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "(1 - (hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio{cluster=~\"$cluster\"} or vector(0))) * 100", - "instant": true, - "legendFormat": "SLI %", - "refId": "A" - } - ], - "title": "Cluster Upgrade Complete SLI", - "type": "stat" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "description": "HCP clusters with any reported cluster version state.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 37 - }, - "id": 14, - "options": { - "colorMode": "background", - "graphMode": "none", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "sum(hosted_control_plane_upgrade:in_progress:count{cluster=~\"$cluster\"}) or vector(0)", - "instant": true, - "legendFormat": "In-flight upgrades", - "refId": "A" - } - ], - "title": "In-Flight Upgrades", - "type": "stat" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "description": "Upgrades stuck in desired or partial for over 30 minutes without reaching completed.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 5 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 37 - }, - "id": 15, - "options": { - "colorMode": "background", - "graphMode": "none", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "sum(hosted_control_plane_upgrade:in_progress:stuck_over_30m:count{cluster=~\"$cluster\"}) or vector(0)", - "instant": true, - "legendFormat": "Stuck clusters", - "refId": "A" - } - ], - "title": "Stuck in Progress Over 30m", - "type": "stat" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "description": "Current 5 minute error budget burn rate. Fast burn alert threshold: 14.4x.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 2, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "yellow", - "value": 6 - }, - { - "color": "red", - "value": 14.4 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 37 - }, - "id": 16, - "options": { - "colorMode": "background", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "max(hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate5m{cluster=~\"$cluster\"}) or vector(0)", - "instant": true, - "legendFormat": "5m burn", - "refId": "A" - } - ], - "title": "Burn Rate (5m)", - "type": "stat" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "description": "Fraction of in-flight upgrades stuck past 30 minutes. SLO error budget line at 0.05%.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 10, - "lineInterpolation": "smooth", - "lineWidth": 2, - "showPoints": "never", - "thresholdsStyle": { - "mode": "line" - } - }, - "decimals": 4, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 0.0005 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 43 - }, - "id": 17, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio{cluster=~\"$cluster\"}", - "legendFormat": "{{ cluster }} in-flight stuck rate", - "refId": "A" - } - ], - "title": "In-Flight Stuck Rate Over Time", - "type": "timeseries" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "description": "Multi-window error budget burn rates for cluster upgrade complete SLO.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 0, - "lineInterpolation": "smooth", - "lineWidth": 2, - "showPoints": "never" - }, - "decimals": 2, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 43 - }, - "id": 18, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate5m{cluster=~\"$cluster\"}", - "legendFormat": "5m {{ cluster }}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate30m{cluster=~\"$cluster\"}", - "legendFormat": "30m {{ cluster }}", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate1h{cluster=~\"$cluster\"}", - "legendFormat": "1h {{ cluster }}", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate6h{cluster=~\"$cluster\"}", - "legendFormat": "6h {{ cluster }}", - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate3d{cluster=~\"$cluster\"}", - "legendFormat": "3d {{ cluster }}", - "refId": "E" - } - ], - "title": "Error Budget Burn Rates", - "type": "timeseries" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "description": "Time from desired first-seen to first active (partial or completed) observation, in minutes.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 0, - "lineInterpolation": "smooth", - "lineWidth": 2, - "showPoints": "never", - "thresholdsStyle": { - "mode": "line" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 30 - } - ] - }, - "unit": "m" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 51 - }, - "id": 19, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "(\n (\n min without(state) (hosted_control_plane_upgrade:version_state_first_seen:timestamp{cluster=~\"$cluster\", state=\"completed\"})\n - on(cluster, resource_id, subscription_id, cluster_uuid, version) group_left()\n hosted_control_plane_upgrade:version_state_first_seen:timestamp{cluster=~\"$cluster\", state=\"desired\"}\n )\n and on(cluster, resource_id, subscription_id, cluster_uuid, version) (\n max without(state) (backend_cluster_version_info{cluster=~\"$cluster\", state=\"completed\"}) == 1\n )\n) / 60", - "legendFormat": "{{ resource_id }} {{ version }}", - "refId": "A" - } - ], - "title": "Cluster Upgrade Complete Duration (Minutes)", - "type": "timeseries" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "description": "Clusters waiting for upgrade completion (minutes since target version was selected).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 10, - "lineInterpolation": "smooth", - "lineWidth": 2, - "showPoints": "never", - "thresholdsStyle": { - "mode": "line" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 30 - } - ] - }, - "unit": "m" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 51 - }, - "id": 20, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "hosted_control_plane_upgrade:duration_in_progress:seconds{cluster=~\"$cluster\"} / 60", - "legendFormat": "{{ resource_id }} {{ version }}", - "refId": "A" - } - ], - "title": "Time in Progress (Minutes)", - "type": "timeseries" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "description": "Clusters with cluster upgrades currently stuck past the 30 minute upgrade-complete window.", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 59 - }, - "id": 21, - "options": { - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "version" - } - ] - }, - "pluginVersion": "11.6.9", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "(\n hosted_control_plane_upgrade:duration_in_progress:seconds{cluster=~\"$cluster\"}\n and\n hosted_control_plane_upgrade:duration_in_progress:seconds{cluster=~\"$cluster\"} > 1800\n) / 60", - "format": "table", - "instant": true, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Stuck in Progress Detail", - "transformations": [ - { - "id": "seriesToRows", - "options": {} - }, - { - "id": "merge", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "__name__": true, - "state": true - }, - "renameByName": { - "Value": "Time in Progress (Minutes)", - "cluster": "Service Cluster", - "cluster_uuid": "Cluster UUID", - "resource_id": "Resource ID", - "subscription_id": "Subscription", - "version": "Target Version" - } - } - } - ], - "type": "table" - } - ], - "schemaVersion": 39, - "tags": [ - "cluster-upgrade", - "aro-hcp", - "slo", - "user-journey" - ], - "templating": { - "list": [ - { - "allowCustomValue": false, - "current": { - "text": "All", - "value": "$__all" - }, - "includeAll": true, - "name": "datasource", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "^Managed_Prometheus_services-.*$", - "type": "datasource" - }, - { - "allValue": ".*", - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "definition": "label_values(backend_cluster_version_info, cluster)", - "includeAll": true, - "multi": true, - "name": "cluster", - "query": "label_values(backend_cluster_version_info, cluster)", - "refresh": 2, - "type": "query" - } - ] - }, - "time": { - "from": "now-7d", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Cluster Upgrade SLO", - "uid": "cluster-upgrade-slo", - "version": 1 -} diff --git a/observability/grafana-dashboards/sre/user-journey/cluster-upgrade-slo.json b/observability/grafana-dashboards/sre/user-journey/cluster-upgrade-slo.json new file mode 100644 index 00000000000..b4b63eada83 --- /dev/null +++ b/observability/grafana-dashboards/sre/user-journey/cluster-upgrade-slo.json @@ -0,0 +1,318 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "gridPos": { "h": 4, "w": 24, "x": 0, "y": 0 }, + "id": 1, + "options": { + "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, + "content": "### Cluster Upgrade SLOs\n\nPer-upgrade observability from `backend_cluster_version_info` via `hosted_control_plane_upgrade:*` recording rules.\n\n| SLO | Target | Alert |\n|-----|--------|-------|\n| **Reach partial** | `partial` within 20m | `userJourneyClusterUpgradeStuckInDesired` (>20m in `desired`) |\n| **Upgrade complete** | `completed` within 30m | `userJourneyClusterUpgradeStuckInProgress` (>30m in progress) |\n\n**Runbook:** [https://aka.ms/arohcp-runbook-cluster-upgrade](https://aka.ms/arohcp-runbook-cluster-upgrade)", + "mode": "markdown" + }, + "pluginVersion": "11.6.9", + "title": "Dashboard Info", + "type": "text" + }, + { + "datasource": { "type": "datasource", "uid": "-- Mixed --" }, + "description": "Upgrade-eligible clusters with a version upgrade currently in progress (desired or partial, not yet completed).", + "fieldConfig": { "defaults": {}, "overrides": [] }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 4 }, + "id": 2, + "options": { + "showHeader": true, + "sortBy": [{ "desc": true, "displayName": "Time in Progress (Minutes)" }] + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "$datasource" }, + "editorMode": "code", + "expr": "hosted_control_plane_upgrade:duration_in_progress:seconds{cluster=~\"$cluster\"} / 60", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "In-Flight Upgrades", + "transformations": [ + { "id": "seriesToRows", "options": {} }, + { "id": "merge", "options": {} }, + { + "id": "organize", + "options": { + "excludeByName": { "Time": true, "__name__": true }, + "renameByName": { + "Value": "Time in Progress (Minutes)", + "cluster": "Service Cluster", + "cluster_uuid": "Cluster UUID", + "resource_id": "Resource ID", + "state": "Current State", + "subscription_id": "Subscription", + "version": "Target Version" + } + } + } + ], + "type": "table" + }, + { + "datasource": { "type": "datasource", "uid": "-- Mixed --" }, + "description": "Upgrade-eligible clusters with a version upgrade currently in progress (desired or partial, not yet completed).", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "blue" }] + } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 8, "x": 0, "y": 12 }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "none", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "$datasource" }, + "editorMode": "code", + "expr": "sum(hosted_control_plane_upgrade:in_progress:count{cluster=~\"$cluster\"}) or vector(0)", + "instant": true, + "legendFormat": "In-flight", + "refId": "A" + } + ], + "title": "In-Flight", + "type": "stat" + }, + { + "datasource": { "type": "datasource", "uid": "-- Mixed --" }, + "description": "Upgrades stuck in desired for over 20 minutes without reaching partial. Matches userJourneyClusterUpgradeStuckInDesired.", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green" }, + { "color": "red", "value": 1 } + ] + } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 8, "x": 8, "y": 12 }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "none", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "$datasource" }, + "editorMode": "code", + "expr": "sum(count by (cluster) (hosted_control_plane_upgrade:duration_in_desired:seconds{cluster=~\"$cluster\"} > 1200) or (0 * hosted_control_plane_upgrade:in_progress:count{cluster=~\"$cluster\"})) or vector(0)", + "instant": true, + "legendFormat": "Stuck in desired", + "refId": "A" + } + ], + "title": "Stuck in Desired (>20m)", + "type": "stat" + }, + { + "datasource": { "type": "datasource", "uid": "-- Mixed --" }, + "description": "Upgrades stuck in desired or partial for over 30 minutes without reaching completed. Matches userJourneyClusterUpgradeStuckInProgress.", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green" }, + { "color": "red", "value": 1 } + ] + } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 8, "x": 16, "y": 12 }, + "id": 5, + "options": { + "colorMode": "background", + "graphMode": "none", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "$datasource" }, + "editorMode": "code", + "expr": "sum(count by (cluster) (hosted_control_plane_upgrade:duration_in_progress:seconds{cluster=~\"$cluster\"} > 1800) or (0 * hosted_control_plane_upgrade:in_progress:count{cluster=~\"$cluster\"})) or vector(0)", + "instant": true, + "legendFormat": "Stuck in progress", + "refId": "A" + } + ], + "title": "Stuck in Progress (>30m)", + "type": "stat" + }, + { + "datasource": { "type": "datasource", "uid": "-- Mixed --" }, + "description": "Clusters with upgrades stuck past the 20 minute reach-partial window.", + "fieldConfig": { "defaults": {}, "overrides": [] }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 }, + "id": 6, + "options": { + "showHeader": true, + "sortBy": [{ "desc": true, "displayName": "Time in Desired (Minutes)" }] + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "$datasource" }, + "editorMode": "code", + "expr": "(\n hosted_control_plane_upgrade:duration_in_desired:seconds{cluster=~\"$cluster\"}\n and\n hosted_control_plane_upgrade:duration_in_desired:seconds{cluster=~\"$cluster\"} > 1200\n) / 60", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Stuck in Desired Detail", + "transformations": [ + { "id": "seriesToRows", "options": {} }, + { "id": "merge", "options": {} }, + { + "id": "organize", + "options": { + "excludeByName": { "Time": true, "__name__": true, "state": true }, + "renameByName": { + "Value": "Time in Desired (Minutes)", + "cluster": "Service Cluster", + "cluster_uuid": "Cluster UUID", + "resource_id": "Resource ID", + "subscription_id": "Subscription", + "version": "Target Version" + } + } + } + ], + "type": "table" + }, + { + "datasource": { "type": "datasource", "uid": "-- Mixed --" }, + "description": "Clusters with upgrades stuck past the 30 minute upgrade-complete window.", + "fieldConfig": { "defaults": {}, "overrides": [] }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 24 }, + "id": 7, + "options": { + "showHeader": true, + "sortBy": [{ "desc": true, "displayName": "Time in Progress (Minutes)" }] + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "$datasource" }, + "editorMode": "code", + "expr": "(\n hosted_control_plane_upgrade:duration_in_progress:seconds{cluster=~\"$cluster\"}\n and\n hosted_control_plane_upgrade:duration_in_progress:seconds{cluster=~\"$cluster\"} > 1800\n) / 60", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Stuck in Progress Detail", + "transformations": [ + { "id": "seriesToRows", "options": {} }, + { "id": "merge", "options": {} }, + { + "id": "organize", + "options": { + "excludeByName": { "Time": true, "__name__": true, "state": true }, + "renameByName": { + "Value": "Time in Progress (Minutes)", + "cluster": "Service Cluster", + "cluster_uuid": "Cluster UUID", + "resource_id": "Resource ID", + "subscription_id": "Subscription", + "version": "Target Version" + } + } + } + ], + "type": "table" + } + ], + "schemaVersion": 39, + "tags": [ + "cluster-upgrade", + "aro-hcp", + "slo", + "user-journey" + ], + "templating": { + "list": [ + { + "allowCustomValue": false, + "current": { "text": "All", "value": "$__all" }, + "includeAll": true, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "^Managed_Prometheus_services-.*$", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { "text": "All", "value": "$__all" }, + "datasource": { "type": "prometheus", "uid": "$datasource" }, + "definition": "label_values(backend_cluster_version_info, cluster)", + "includeAll": true, + "multi": true, + "name": "cluster", + "query": "label_values(backend_cluster_version_info, cluster)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { "from": "now-7d", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "Cluster Upgrade SLO", + "uid": "cluster-upgrade-slo", + "version": 1 +} diff --git a/observability/recording-rules/userJourneyClusterUpgrade-recordingRule.yaml b/observability/recording-rules/userJourneyClusterUpgrade-recordingRule.yaml index f8259674e7f..0d59a31a32c 100644 --- a/observability/recording-rules/userJourneyClusterUpgrade-recordingRule.yaml +++ b/observability/recording-rules/userJourneyClusterUpgrade-recordingRule.yaml @@ -11,17 +11,18 @@ metadata: spec: groups: # ======================================== - # Cluster Upgrade — shared recording rules + # Cluster Upgrade recording rules # ======================================== - # Shared eligibility, first-seen timestamps, and in-progress fleet counts for both upgrade SLOs. + # Eligibility, first-seen timestamps, in-progress fleet counts, and per-upgrade + # duration metrics used by stuck alerts and the cluster-upgrade SLO dashboard. # - # upgrade_eligible:info must be evaluated before version_state_first_seen in this group (same cycle). + # upgrade_eligible:info must be evaluated before version_state_first_seen (same cycle). # Eligibility: per resource_id, at least two distinct version labels and at least one completed. # Excludes initial install. Cluster create bounds when install ends (operation_cluster_create.go): # https://github.com/Azure/ARO-HCP/blob/main/backend/pkg/controllers/operationcontrollers/operation_cluster_create.go#L316-L320 # # Naming: level:metric:operations per https://prometheus.io/docs/practices/rules/#naming - - name: user-journey-cluster-upgrade-shared-recording-rules + - name: arohcp_user_journey_cluster_upgrade_recording_rules interval: 1m rules: - record: hosted_control_plane_upgrade:upgrade_eligible:info @@ -40,30 +41,37 @@ spec: ) >= 1 ) ) * 0 + 1 - # When each upgrade target version first entered desired, partial, or completed (start time for SLO durations). - # Upgrade-only: excludes initial install — see upgrade_eligible:info. + # When each upgrade target version first entered desired or partial (start time for SLO durations). + # Stop recording once the version reaches completed so latched timestamps do not persist forever. - record: hosted_control_plane_upgrade:version_state_first_seen:timestamp expr: | min without(prometheus_replica) ( min by (cluster, resource_id, subscription_id, cluster_uuid, version, state) ( - hosted_control_plane_upgrade:version_state_first_seen:timestamp - or ( - timestamp(backend_cluster_version_info{state=~"desired|partial"} == 1) - and on (cluster, resource_id) ( - hosted_control_plane_upgrade:upgrade_eligible:info == 1 + hosted_control_plane_upgrade:version_state_first_seen:timestamp + or + ( + timestamp(backend_cluster_version_info{state=~"desired|partial"} == 1) + and on (cluster, resource_id) ( + hosted_control_plane_upgrade:upgrade_eligible:info == 1 + ) ) ) - or - timestamp(backend_cluster_version_info{state="completed"} == 1) + unless on(cluster, resource_id, subscription_id, cluster_uuid, version) ( + backend_cluster_version_info{state="completed"} == 1 + ) ) ) - # Count of clusters with an upgrade currently in progress (desired or partial on a new target version). + # Count of clusters with an upgrade currently in progress (desired or partial on a target version, without being completed). - record: hosted_control_plane_upgrade:in_progress:count expr: | count by (cluster) ( count by (cluster, resource_id) ( - backend_cluster_version_info{state=~"desired|partial"} == 1 + ( + backend_cluster_version_info{state=~"desired|partial"} == 1 + unless on(cluster, resource_id, subscription_id, cluster_uuid, version) + backend_cluster_version_info{state="completed"} == 1 + ) ) >= 1 and on (cluster, resource_id) ( hosted_control_plane_upgrade:upgrade_eligible:info == 1 @@ -71,129 +79,48 @@ spec: ) or 0 * count by (cluster) (backend_cluster_version_info) - # ======================================== - # Cluster Upgrade — reach-partial SLO (20m) - # ======================================== - # SLO: 99.95% of cluster upgrades reach partial within 20 minutes of selecting the target version. - # Partial is the time for the target version to become active (recognized as active). It indicates - # the upgrade is in progress on the HCP cluster but not yet complete. - # Error budget: 0.05% (1 - 0.9995) - # - # Scope: upgrades only — gated by hosted_control_plane_upgrade:upgrade_eligible:info. - - name: user-journey-cluster-upgrade-reach-partial-recording-rules - interval: 1m - rules: - # Seconds the target version has been in desired without reaching partial. Upgrade-only. + # Seconds the target version has been in desired without reaching partial or completed. - record: hosted_control_plane_upgrade:duration_in_desired:seconds expr: | ( time() - hosted_control_plane_upgrade:version_state_first_seen:timestamp{state="desired"} ) and on(cluster, resource_id, subscription_id, cluster_uuid, version) ( - backend_cluster_version_info{state="desired"} == 1 + ( + backend_cluster_version_info{state="desired"} == 1 + unless on(cluster, resource_id, subscription_id, cluster_uuid, version) ( + backend_cluster_version_info{state="partial"} == 1 + or + backend_cluster_version_info{state="completed"} == 1 + ) + ) ) and on (cluster, resource_id) ( hosted_control_plane_upgrade:upgrade_eligible:info == 1 ) - # Clusters whose target version has been in desired longer than 20 minutes (reach-partial SLO miss). - - record: hosted_control_plane_upgrade:desired:stuck_over_20m:count - expr: | - count by (cluster) (hosted_control_plane_upgrade:duration_in_desired:seconds > 1200) - or - 0 * hosted_control_plane_upgrade:in_progress:count - # Share of in-progress upgrades stuck in desired over 20 minutes (reach-partial error rate). - - record: hosted_control_plane_upgrade:desired:stuck_over_20m:ratio - expr: | - hosted_control_plane_upgrade:desired:stuck_over_20m:count - / - clamp_min(hosted_control_plane_upgrade:in_progress:count, 1) - # Reach-partial error-budget burn rate over 5 minutes (99.95% SLO). - - record: hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate5m - expr: | - avg_over_time(hosted_control_plane_upgrade:desired:stuck_over_20m:ratio[5m]) - / (1 - 0.9995) - # Reach-partial error-budget burn rate over 30 minutes. - - record: hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate30m - expr: | - avg_over_time(hosted_control_plane_upgrade:desired:stuck_over_20m:ratio[30m]) - / (1 - 0.9995) - # Reach-partial error-budget burn rate over 1 hour. - - record: hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate1h - expr: | - avg_over_time(hosted_control_plane_upgrade:desired:stuck_over_20m:ratio[1h]) - / (1 - 0.9995) - # Reach-partial error-budget burn rate over 6 hours. - - record: hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate6h - expr: | - avg_over_time(hosted_control_plane_upgrade:desired:stuck_over_20m:ratio[6h]) - / (1 - 0.9995) - # Reach-partial error-budget burn rate over 3 days. - - record: hosted_control_plane_upgrade:desired:stuck_over_20m:burnrate3d - expr: | - avg_over_time(hosted_control_plane_upgrade:desired:stuck_over_20m:ratio[3d]) - / (1 - 0.9995) - # ======================================== - # Cluster Upgrade — complete SLO (30m) - # ======================================== - # SLO: 99.95% of cluster upgrades complete within 30 minutes of selecting the target version. - # Error budget: 0.05% (1 - 0.9995) - # - # Success: desired first-seen to completed first-seen <= 1800s. - # In-flight: desired or partial without completed for the target version. - # Upgrade-only scope: gated by hosted_control_plane_upgrade:upgrade_eligible:info. - - name: user-journey-cluster-upgrade-complete-recording-rules - interval: 1m - rules: - # Seconds since the target version was selected until it completes; stops when that version reaches completed. - # Upgrade-only: excludes initial install — see upgrade_eligible:info. + # Seconds since the target version was selected until it completes (desired or partial on a target version, without being completed). - record: hosted_control_plane_upgrade:duration_in_progress:seconds expr: | ( - time() - hosted_control_plane_upgrade:version_state_first_seen:timestamp{state="desired"} - ) - unless on(cluster, resource_id, subscription_id, cluster_uuid, version) ( - backend_cluster_version_info{state="completed"} == 1 - ) - and on(cluster, resource_id, subscription_id, cluster_uuid, version) ( - backend_cluster_version_info{state=~"desired|partial"} == 1 - ) - and on (cluster, resource_id) ( - hosted_control_plane_upgrade:upgrade_eligible:info == 1 + sum without(state) ( + ( + time() - hosted_control_plane_upgrade:version_state_first_seen:timestamp{state="desired"} + ) + and on (cluster, resource_id) ( + hosted_control_plane_upgrade:upgrade_eligible:info == 1 + ) + ) + unless on(cluster, resource_id, subscription_id, cluster_uuid, version) ( + backend_cluster_version_info{state="completed"} == 1 + ) ) - # Clusters whose upgrade has been in progress longer than 30 minutes without completing (complete SLO miss). - - record: hosted_control_plane_upgrade:in_progress:stuck_over_30m:count - expr: | - count by (cluster) (hosted_control_plane_upgrade:duration_in_progress:seconds > 1800) - or - 0 * hosted_control_plane_upgrade:in_progress:count - # Share of in-progress upgrades stuck over 30 minutes without completing (complete error rate). - - record: hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio - expr: | - hosted_control_plane_upgrade:in_progress:stuck_over_30m:count - / - clamp_min(hosted_control_plane_upgrade:in_progress:count, 1) - # Complete error-budget burn rate over 5 minutes (99.95% SLO). - - record: hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate5m - expr: | - avg_over_time(hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio[5m]) - / (1 - 0.9995) - # Complete error-budget burn rate over 30 minutes. - - record: hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate30m - expr: | - avg_over_time(hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio[30m]) - / (1 - 0.9995) - # Complete error-budget burn rate over 1 hour. - - record: hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate1h - expr: | - avg_over_time(hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio[1h]) - / (1 - 0.9995) - # Complete error-budget burn rate over 6 hours. - - record: hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate6h - expr: | - avg_over_time(hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio[6h]) - / (1 - 0.9995) - # Complete error-budget burn rate over 3 days. - - record: hosted_control_plane_upgrade:in_progress:stuck_over_30m:burnrate3d - expr: | - avg_over_time(hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio[3d]) - / (1 - 0.9995) + * on(cluster, resource_id, subscription_id, cluster_uuid, version) group_left(state) + ( + backend_cluster_version_info{state="partial"} == 1 + or + ( + backend_cluster_version_info{state="desired"} == 1 + unless on(cluster, resource_id, subscription_id, cluster_uuid, version) + backend_cluster_version_info{state="partial"} == 1 + ) + ) diff --git a/observability/recording-rules/userJourneyClusterUpgrade-recordingRule_test.yaml b/observability/recording-rules/userJourneyClusterUpgrade-recordingRule_test.yaml index 9f96104d326..8a71dfff5d2 100644 --- a/observability/recording-rules/userJourneyClusterUpgrade-recordingRule_test.yaml +++ b/observability/recording-rules/userJourneyClusterUpgrade-recordingRule_test.yaml @@ -24,10 +24,54 @@ tests: exp_samples: - labels: 'hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", cluster_uuid="uuid-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", version="4.19.20", state="desired"}' value: 540 - - expr: hosted_control_plane_upgrade:desired:stuck_over_20m:count{cluster="svc-1"} - eval_time: 9m + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 15m + exp_samples: [] + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 25m + exp_samples: [] + - expr: hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20", state="partial"} + eval_time: 15m + exp_samples: + - labels: 'hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", cluster_uuid="uuid-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", version="4.19.20", state="partial"}' + value: 900 +# ======================================== +# Test 1b: Partial with stale desired — must not count as stuck in desired +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed"}' + values: "1+0x35" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="desired"}' + values: "1+0x35" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="partial"}' + values: "0x10 1x25" + promql_expr_test: + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 25m + exp_samples: [] +# ======================================== +# Test 1c: Completed with stale desired — must not count as stuck in desired or in progress +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed"}' + values: "1+0x35" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="desired"}' + values: "1+0x35" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="completed"}' + values: "0x10 1x25" + promql_expr_test: + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 25m + exp_samples: [] + - expr: hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 25m + exp_samples: [] + - expr: hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"} + eval_time: 25m exp_samples: - - labels: 'hosted_control_plane_upgrade:desired:stuck_over_20m:count{cluster="svc-1"}' + - labels: 'hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"}' value: 0 # ======================================== # Test 2: Stuck in desired — 1 of 1 active upgrades stuck >20m @@ -56,16 +100,6 @@ tests: exp_samples: - labels: 'hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"}' value: 1 - - expr: hosted_control_plane_upgrade:desired:stuck_over_20m:count{cluster="svc-1"} - eval_time: 35m - exp_samples: - - labels: 'hosted_control_plane_upgrade:desired:stuck_over_20m:count{cluster="svc-1"}' - value: 1 - - expr: hosted_control_plane_upgrade:desired:stuck_over_20m:ratio{cluster="svc-1"} - eval_time: 35m - exp_samples: - - labels: 'hosted_control_plane_upgrade:desired:stuck_over_20m:ratio{cluster="svc-1"}' - value: 1 # ======================================== # Test 3: Install only — desired without completed is excluded from SLO # ======================================== @@ -85,11 +119,6 @@ tests: exp_samples: - labels: 'hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"}' value: 0 - - expr: hosted_control_plane_upgrade:desired:stuck_over_20m:count{cluster="svc-1"} - eval_time: 35m - exp_samples: - - labels: 'hosted_control_plane_upgrade:desired:stuck_over_20m:count{cluster="svc-1"}' - value: 0 # ======================================== # Test 4: Complete — desired transitions to completed within 30m (upgrade) # ======================================== @@ -107,12 +136,41 @@ tests: exp_samples: - labels: 'hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", cluster_uuid="uuid-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", version="4.19.20", state="desired"}' value: 540 - - expr: hosted_control_plane_upgrade:in_progress:stuck_over_30m:count{cluster="svc-1"} - eval_time: 9m + - expr: hosted_control_plane_upgrade:version_state_first_seen:timestamp{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 15m + exp_samples: [] + - expr: hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 15m + exp_samples: [] + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 15m + exp_samples: [] + - expr: hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"} + eval_time: 15m exp_samples: - - labels: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:count{cluster="svc-1"}' + - labels: 'hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"}' value: 0 # ======================================== +# Test 4b: Completed with stale partial — must not count as in progress or emit duration +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed"}' + values: "1+0x35" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="partial"}' + values: "1+0x35" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="completed"}' + values: "1+0x35" + promql_expr_test: + - expr: hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"}' + value: 0 + - expr: hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 35m + exp_samples: [] +# ======================================== # Test 5: Stuck in progress — partial without completed after 35m # ======================================== - interval: 1m @@ -126,26 +184,16 @@ tests: - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c3", subscription_id="sub-2", cluster_uuid="uuid-3", version="4.19.20", state="completed"}' values: "1+0x50" promql_expr_test: - - expr: hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", resource_id="/sub/2/rg/c2", version="4.19.21", state="desired"} + - expr: hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", resource_id="/sub/2/rg/c2", version="4.19.21", state="partial"} eval_time: 35m exp_samples: - - labels: 'hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", cluster_uuid="uuid-2", resource_id="/sub/2/rg/c2", subscription_id="sub-2", version="4.19.21", state="desired"}' + - labels: 'hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", cluster_uuid="uuid-2", resource_id="/sub/2/rg/c2", subscription_id="sub-2", version="4.19.21", state="partial"}' value: 2100 - expr: hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"} eval_time: 35m exp_samples: - labels: 'hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"}' value: 1 - - expr: hosted_control_plane_upgrade:in_progress:stuck_over_30m:count{cluster="svc-1"} - eval_time: 35m - exp_samples: - - labels: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:count{cluster="svc-1"}' - value: 1 - - expr: hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio{cluster="svc-1"} - eval_time: 35m - exp_samples: - - labels: 'hosted_control_plane_upgrade:in_progress:stuck_over_30m:ratio{cluster="svc-1"}' - value: 1 # ======================================== # Test 6: Two versions without completed — not upgrade eligible # ========================================