diff --git a/dev-infrastructure/modules/metrics/rules/generatedRPPrometheusAlertingRules.bicep b/dev-infrastructure/modules/metrics/rules/generatedRPPrometheusAlertingRules.bicep index 57dd23e379d..63032b84ad6 100644 --- a/dev-infrastructure/modules/metrics/rules/generatedRPPrometheusAlertingRules.bicep +++ b/dev-infrastructure/modules/metrics/rules/generatedRPPrometheusAlertingRules.bicep @@ -234,6 +234,83 @@ resource arohcpAccessClusterSaturationAlerts 'Microsoft.AlertsManagement/prometh } } +resource rpUserJourneyClusterUpgradeMonitorRules 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'rp-user-journey-cluster-upgrade-monitor-rules' + location: location + properties: { + interval: 'PT1M' + rules: [ + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'userJourneyClusterUpgradeStuckInDesired' + enabled: true + labels: { + severity: 'info' + slo: 'cluster-upgrade-reach-partial' + } + annotations: { + correlationId: 'userJourneyClusterUpgradeStuckInDesired/{{ $labels.resource_id }}/{{ $labels.version }}' + description: '''Cluster upgrade target version {{ $labels.version }} has been in desired for over 20 minutes without reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on the affected cluster. +Service Cluster: {{ $labels.cluster }} +''' + info: '''Cluster upgrade target version {{ $labels.version }} has been in desired for over 20 minutes without reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on the affected cluster. +Service Cluster: {{ $labels.cluster }} +''' + runbook_url: 'https://aka.ms/arohcp-runbook-cluster-upgrade' + summary: '[HCPUpgradeStuckInDesired] {{ $labels.resource_id }}' + title: '[HCPUpgradeStuckInDesired] {{ $labels.resource_id }} version:{{ $labels.version }} cluster:{{ $labels.cluster }}' + } + expression: 'hosted_control_plane_upgrade:duration_in_desired:seconds > 1200' + for: 'PT5M' + severity: severityCeiling > 0 ? max(4, severityCeiling) : 4 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'userJourneyClusterUpgradeStuckInProgress' + enabled: true + labels: { + severity: 'info' + slo: 'cluster-upgrade-complete' + } + annotations: { + correlationId: 'userJourneyClusterUpgradeStuckInProgress/{{ $labels.resource_id }}/{{ $labels.version }}' + description: '''Cluster upgrade target version {{ $labels.version }} has been in progress for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on the affected cluster. +Service Cluster: {{ $labels.cluster }} +''' + info: '''Cluster upgrade target version {{ $labels.version }} has been in progress for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on the affected cluster. +Service Cluster: {{ $labels.cluster }} +''' + runbook_url: 'https://aka.ms/arohcp-runbook-cluster-upgrade' + summary: '[HCPUpgradeStuckInProgress] {{ $labels.resource_id }}' + title: '[HCPUpgradeStuckInProgress] {{ $labels.resource_id }} version:{{ $labels.version }} cluster:{{ $labels.cluster }}' + } + expression: 'hosted_control_plane_upgrade:duration_in_progress:seconds > 1800' + for: 'PT5M' + severity: severityCeiling > 0 ? max(4, severityCeiling) : 4 + } + ] + scopes: [ + azureMonitoring + ] + } +} + resource arohcpNodepoolSloErrorAlerts 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { name: 'arohcp_nodepool_slo_error_alerts' location: location diff --git a/dev-infrastructure/modules/metrics/rules/generatedRecordingRules.bicep b/dev-infrastructure/modules/metrics/rules/generatedRecordingRules.bicep index dfc1823f76c..13b69e6a962 100644 --- a/dev-infrastructure/modules/metrics/rules/generatedRecordingRules.bicep +++ b/dev-infrastructure/modules/metrics/rules/generatedRecordingRules.bicep @@ -27,3 +27,37 @@ resource arohcpAccessClusterSloRecordingRules 'Microsoft.AlertsManagement/promet ] } } + +resource arohcpUserJourneyClusterUpgradeRecordingRules 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'arohcp_user_journey_cluster_upgrade_recording_rules' + location: location + properties: { + scopes: [ + azureMonitoring + ] + enabled: true + interval: 'PT1M' + rules: [ + { + record: 'hosted_control_plane_upgrade:upgrade_eligible:info' + expression: '((count by (cluster, resource_id, subscription_id, cluster_uuid) (count by (cluster, resource_id, subscription_id, cluster_uuid, version) (backend_cluster_version_info == 1)) >= 2) and on (cluster, resource_id) (count by (cluster, resource_id) (backend_cluster_version_info{state="completed"} == 1) >= 1)) * 0 + 1' + } + { + record: 'hosted_control_plane_upgrade:version_state_first_seen:timestamp' + expression: 'min without (prometheus_replica) (min by (cluster, resource_id, subscription_id, cluster_uuid, version, state) ((hosted_control_plane_upgrade:version_state_first_seen:timestamp or (timestamp(backend_cluster_version_info{state=~"desired|partial"} == 1) and on (cluster, resource_id) (hosted_control_plane_upgrade:upgrade_eligible:info == 1))) unless on (cluster, resource_id, subscription_id, cluster_uuid, version) (backend_cluster_version_info{state="completed"} == 1)))' + } + { + record: 'hosted_control_plane_upgrade:in_progress:count' + expression: 'count by (cluster) (count by (cluster, resource_id) ((backend_cluster_version_info{state=~"desired|partial"} == 1 unless on (cluster, resource_id, subscription_id, cluster_uuid, version) backend_cluster_version_info{state="completed"} == 1)) >= 1 and on (cluster, resource_id) (hosted_control_plane_upgrade:upgrade_eligible:info == 1)) or 0 * count by (cluster) (backend_cluster_version_info)' + } + { + record: 'hosted_control_plane_upgrade:duration_in_desired:seconds' + expression: '(time() - hosted_control_plane_upgrade:version_state_first_seen:timestamp{state="desired"}) and on (cluster, resource_id, subscription_id, cluster_uuid, version) ((backend_cluster_version_info{state="desired"} == 1 unless on (cluster, resource_id, subscription_id, cluster_uuid, version) (backend_cluster_version_info{state="partial"} == 1 or backend_cluster_version_info{state="completed"} == 1))) and on (cluster, resource_id) (hosted_control_plane_upgrade:upgrade_eligible:info == 1)' + } + { + record: 'hosted_control_plane_upgrade:duration_in_progress:seconds' + expression: '(sum without (state) ((time() - hosted_control_plane_upgrade:version_state_first_seen:timestamp{state="desired"}) and on (cluster, resource_id) (hosted_control_plane_upgrade:upgrade_eligible:info == 1)) unless on (cluster, resource_id, subscription_id, cluster_uuid, version) (backend_cluster_version_info{state="completed"} == 1)) * on (cluster, resource_id, subscription_id, cluster_uuid, version) group_left (state) (backend_cluster_version_info{state="partial"} == 1 or (backend_cluster_version_info{state="desired"} == 1 unless on (cluster, resource_id, subscription_id, cluster_uuid, version) backend_cluster_version_info{state="partial"} == 1))' + } + ] + } +} diff --git a/observability/alerts-rp-services.yaml b/observability/alerts-rp-services.yaml index 514243ecdac..2af317ca77d 100644 --- a/observability/alerts-rp-services.yaml +++ b/observability/alerts-rp-services.yaml @@ -1,6 +1,7 @@ prometheusRules: rulesFolders: - alerts/access-cluster-slo-prometheusRule.yaml + - alerts/userJourneyClusterUpgradeMonitor-prometheusRule.yaml - alerts/nodepool-slo-prometheusRule.yaml untestedRules: [] testDependencies: diff --git a/observability/alerts/userJourneyClusterUpgradeMonitor-prometheusRule.yaml b/observability/alerts/userJourneyClusterUpgradeMonitor-prometheusRule.yaml new file mode 100644 index 00000000000..d3b5c9f14ff --- /dev/null +++ b/observability/alerts/userJourneyClusterUpgradeMonitor-prometheusRule.yaml @@ -0,0 +1,51 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + app.kubernetes.io/name: kube-prometheus + app.kubernetes.io/part-of: kube-prometheus + prometheus: k8s + role: alert-rules + name: user-journey-cluster-upgrade-alerts + namespace: monitoring +spec: + groups: + # Per-upgrade stuck alerts for cluster upgrade SLOs. Sev 4 (info). + # At current upgrade volumes, alert directly on duration instead of fleet burn-rate. + # + # In absence of https://issues.redhat.com/browse/ARO-22686 (reporting of control plane + # version history for versions lower than 4.22), cluster upgrade completeness is directly + # impacted by data plane state — e.g. no worker nodes means the upgrade will not reach + # completed; missing console component in external auth prevents CVO from marking the + # upgrade complete. The complete alert may therefore fire due to missing data plane + # configuration rather than control plane failure. The runbook accommodates this. + - name: rp-user-journey-cluster-upgrade-monitor-rules + rules: + - alert: userJourneyClusterUpgradeStuckInDesired + expr: | + hosted_control_plane_upgrade:duration_in_desired:seconds > 1200 + for: 5m + labels: + severity: info + slo: cluster-upgrade-reach-partial + annotations: + summary: "[HCPUpgradeStuckInDesired] {{ $labels.resource_id }}" + description: | + Cluster upgrade target version {{ $labels.version }} has been in desired for over 20 minutes without reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on the affected cluster. + Service Cluster: {{ $labels.cluster }} + runbook_url: https://aka.ms/arohcp-runbook-cluster-upgrade + correlationId: "userJourneyClusterUpgradeStuckInDesired/{{ $labels.resource_id }}/{{ $labels.version }}" + - alert: userJourneyClusterUpgradeStuckInProgress + expr: | + hosted_control_plane_upgrade:duration_in_progress:seconds > 1800 + for: 5m + labels: + severity: info + slo: cluster-upgrade-complete + annotations: + summary: "[HCPUpgradeStuckInProgress] {{ $labels.resource_id }}" + description: | + Cluster upgrade target version {{ $labels.version }} has been in progress for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on the affected cluster. + Service Cluster: {{ $labels.cluster }} + runbook_url: https://aka.ms/arohcp-runbook-cluster-upgrade + correlationId: "userJourneyClusterUpgradeStuckInProgress/{{ $labels.resource_id }}/{{ $labels.version }}" diff --git a/observability/alerts/userJourneyClusterUpgradeMonitor-prometheusRule_test.yaml b/observability/alerts/userJourneyClusterUpgradeMonitor-prometheusRule_test.yaml new file mode 100644 index 00000000000..829f338231c --- /dev/null +++ b/observability/alerts/userJourneyClusterUpgradeMonitor-prometheusRule_test.yaml @@ -0,0 +1,128 @@ +rule_files: +- userJourneyClusterUpgrade-recordingRule.yaml +- userJourneyClusterUpgradeMonitor-prometheusRule.yaml +evaluation_interval: 1m +tests: +# ======================================== +# Reach partial: successful — no alert +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed"}' + values: "1+0x40" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="desired"}' + values: "1x10 0x25" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="partial"}' + values: "0x10 1x25" + alert_rule_test: + - eval_time: 25m + alertname: userJourneyClusterUpgradeStuckInDesired + exp_alerts: [] + - eval_time: 40m + alertname: userJourneyClusterUpgradeStuckInDesired + exp_alerts: [] +# ======================================== +# Reach partial: stuck in desired >20m — alert fires at SLO breach +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.20", state="completed"}' + values: "1+0x50" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.21", state="desired"}' + values: "1+0x50" + alert_rule_test: + - eval_time: 20m + alertname: userJourneyClusterUpgradeStuckInDesired + exp_alerts: [] + - eval_time: 30m + alertname: userJourneyClusterUpgradeStuckInDesired + exp_alerts: + - exp_labels: + alertname: userJourneyClusterUpgradeStuckInDesired + cluster: "svc-1" + cluster_uuid: "uuid-2" + resource_id: "/sub/2/rg/c2" + subscription_id: "sub-2" + version: "4.19.21" + state: "desired" + severity: info + slo: cluster-upgrade-reach-partial + exp_annotations: + summary: "[HCPUpgradeStuckInDesired] /sub/2/rg/c2" + description: | + Cluster upgrade target version 4.19.21 has been in desired for over 20 minutes without reaching partial. Partial is when the target version becomes active (recognized as active) on the HCP cluster; the upgrade is in progress but not yet complete. Investigate backend_cluster_version_info on the affected cluster. + Service Cluster: svc-1 + runbook_url: "https://aka.ms/arohcp-runbook-cluster-upgrade" + correlationId: "userJourneyClusterUpgradeStuckInDesired//sub/2/rg/c2/4.19.21" +# ======================================== +# Reach partial: install only — not upgrade eligible, no alert +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/c4", subscription_id="sub-3", cluster_uuid="uuid-4", version="4.19.20", state="desired"}' + values: "1+0x50" + alert_rule_test: + - eval_time: 50m + alertname: userJourneyClusterUpgradeStuckInDesired + exp_alerts: [] +# ======================================== +# Upgrade complete: successful — no alert +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed"}' + values: "1+0x40" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="desired"}' + values: "1x10 0x25" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="completed"}' + values: "0x10 1x25" + alert_rule_test: + - eval_time: 40m + alertname: userJourneyClusterUpgradeStuckInProgress + exp_alerts: [] +# ======================================== +# Upgrade complete: stuck in progress >30m — alert fires at SLO breach +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.20", state="completed"}' + values: "1+0x50" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.21", state="desired"}' + values: "1x10 0x50" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.21", state="partial"}' + values: "0x10 1x50" + alert_rule_test: + - eval_time: 30m + alertname: userJourneyClusterUpgradeStuckInProgress + exp_alerts: [] + - eval_time: 40m + alertname: userJourneyClusterUpgradeStuckInProgress + exp_alerts: + - exp_labels: + alertname: userJourneyClusterUpgradeStuckInProgress + cluster: "svc-1" + cluster_uuid: "uuid-2" + resource_id: "/sub/2/rg/c2" + subscription_id: "sub-2" + version: "4.19.21" + state: "partial" + severity: info + slo: cluster-upgrade-complete + exp_annotations: + summary: "[HCPUpgradeStuckInProgress] /sub/2/rg/c2" + description: | + Cluster upgrade target version 4.19.21 has been in progress for over 30 minutes without reaching completed. Completed is when the target version upgrade has finished and is reported as complete on the HCP cluster. Investigate backend_cluster_version_info on the affected cluster. + Service Cluster: svc-1 + runbook_url: "https://aka.ms/arohcp-runbook-cluster-upgrade" + correlationId: "userJourneyClusterUpgradeStuckInProgress//sub/2/rg/c2/4.19.21" +# ======================================== +# Upgrade complete: install only — not upgrade eligible, no alert +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/c4", subscription_id="sub-3", cluster_uuid="uuid-4", version="4.19.20", state="desired"}' + values: "1+0x50" + alert_rule_test: + - eval_time: 50m + alertname: userJourneyClusterUpgradeStuckInProgress + exp_alerts: [] diff --git a/observability/grafana-dashboards/sre/user-journey/cluster-upgrade-slo.json b/observability/grafana-dashboards/sre/user-journey/cluster-upgrade-slo.json new file mode 100644 index 00000000000..b4b63eada83 --- /dev/null +++ b/observability/grafana-dashboards/sre/user-journey/cluster-upgrade-slo.json @@ -0,0 +1,318 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "gridPos": { "h": 4, "w": 24, "x": 0, "y": 0 }, + "id": 1, + "options": { + "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, + "content": "### Cluster Upgrade SLOs\n\nPer-upgrade observability from `backend_cluster_version_info` via `hosted_control_plane_upgrade:*` recording rules.\n\n| SLO | Target | Alert |\n|-----|--------|-------|\n| **Reach partial** | `partial` within 20m | `userJourneyClusterUpgradeStuckInDesired` (>20m in `desired`) |\n| **Upgrade complete** | `completed` within 30m | `userJourneyClusterUpgradeStuckInProgress` (>30m in progress) |\n\n**Runbook:** [https://aka.ms/arohcp-runbook-cluster-upgrade](https://aka.ms/arohcp-runbook-cluster-upgrade)", + "mode": "markdown" + }, + "pluginVersion": "11.6.9", + "title": "Dashboard Info", + "type": "text" + }, + { + "datasource": { "type": "datasource", "uid": "-- Mixed --" }, + "description": "Upgrade-eligible clusters with a version upgrade currently in progress (desired or partial, not yet completed).", + "fieldConfig": { "defaults": {}, "overrides": [] }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 4 }, + "id": 2, + "options": { + "showHeader": true, + "sortBy": [{ "desc": true, "displayName": "Time in Progress (Minutes)" }] + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "$datasource" }, + "editorMode": "code", + "expr": "hosted_control_plane_upgrade:duration_in_progress:seconds{cluster=~\"$cluster\"} / 60", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "In-Flight Upgrades", + "transformations": [ + { "id": "seriesToRows", "options": {} }, + { "id": "merge", "options": {} }, + { + "id": "organize", + "options": { + "excludeByName": { "Time": true, "__name__": true }, + "renameByName": { + "Value": "Time in Progress (Minutes)", + "cluster": "Service Cluster", + "cluster_uuid": "Cluster UUID", + "resource_id": "Resource ID", + "state": "Current State", + "subscription_id": "Subscription", + "version": "Target Version" + } + } + } + ], + "type": "table" + }, + { + "datasource": { "type": "datasource", "uid": "-- Mixed --" }, + "description": "Upgrade-eligible clusters with a version upgrade currently in progress (desired or partial, not yet completed).", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "blue" }] + } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 8, "x": 0, "y": 12 }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "none", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "$datasource" }, + "editorMode": "code", + "expr": "sum(hosted_control_plane_upgrade:in_progress:count{cluster=~\"$cluster\"}) or vector(0)", + "instant": true, + "legendFormat": "In-flight", + "refId": "A" + } + ], + "title": "In-Flight", + "type": "stat" + }, + { + "datasource": { "type": "datasource", "uid": "-- Mixed --" }, + "description": "Upgrades stuck in desired for over 20 minutes without reaching partial. Matches userJourneyClusterUpgradeStuckInDesired.", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green" }, + { "color": "red", "value": 1 } + ] + } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 8, "x": 8, "y": 12 }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "none", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "$datasource" }, + "editorMode": "code", + "expr": "sum(count by (cluster) (hosted_control_plane_upgrade:duration_in_desired:seconds{cluster=~\"$cluster\"} > 1200) or (0 * hosted_control_plane_upgrade:in_progress:count{cluster=~\"$cluster\"})) or vector(0)", + "instant": true, + "legendFormat": "Stuck in desired", + "refId": "A" + } + ], + "title": "Stuck in Desired (>20m)", + "type": "stat" + }, + { + "datasource": { "type": "datasource", "uid": "-- Mixed --" }, + "description": "Upgrades stuck in desired or partial for over 30 minutes without reaching completed. Matches userJourneyClusterUpgradeStuckInProgress.", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green" }, + { "color": "red", "value": 1 } + ] + } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 8, "x": 16, "y": 12 }, + "id": 5, + "options": { + "colorMode": "background", + "graphMode": "none", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "$datasource" }, + "editorMode": "code", + "expr": "sum(count by (cluster) (hosted_control_plane_upgrade:duration_in_progress:seconds{cluster=~\"$cluster\"} > 1800) or (0 * hosted_control_plane_upgrade:in_progress:count{cluster=~\"$cluster\"})) or vector(0)", + "instant": true, + "legendFormat": "Stuck in progress", + "refId": "A" + } + ], + "title": "Stuck in Progress (>30m)", + "type": "stat" + }, + { + "datasource": { "type": "datasource", "uid": "-- Mixed --" }, + "description": "Clusters with upgrades stuck past the 20 minute reach-partial window.", + "fieldConfig": { "defaults": {}, "overrides": [] }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 }, + "id": 6, + "options": { + "showHeader": true, + "sortBy": [{ "desc": true, "displayName": "Time in Desired (Minutes)" }] + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "$datasource" }, + "editorMode": "code", + "expr": "(\n hosted_control_plane_upgrade:duration_in_desired:seconds{cluster=~\"$cluster\"}\n and\n hosted_control_plane_upgrade:duration_in_desired:seconds{cluster=~\"$cluster\"} > 1200\n) / 60", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Stuck in Desired Detail", + "transformations": [ + { "id": "seriesToRows", "options": {} }, + { "id": "merge", "options": {} }, + { + "id": "organize", + "options": { + "excludeByName": { "Time": true, "__name__": true, "state": true }, + "renameByName": { + "Value": "Time in Desired (Minutes)", + "cluster": "Service Cluster", + "cluster_uuid": "Cluster UUID", + "resource_id": "Resource ID", + "subscription_id": "Subscription", + "version": "Target Version" + } + } + } + ], + "type": "table" + }, + { + "datasource": { "type": "datasource", "uid": "-- Mixed --" }, + "description": "Clusters with upgrades stuck past the 30 minute upgrade-complete window.", + "fieldConfig": { "defaults": {}, "overrides": [] }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 24 }, + "id": 7, + "options": { + "showHeader": true, + "sortBy": [{ "desc": true, "displayName": "Time in Progress (Minutes)" }] + }, + "pluginVersion": "11.6.9", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "$datasource" }, + "editorMode": "code", + "expr": "(\n hosted_control_plane_upgrade:duration_in_progress:seconds{cluster=~\"$cluster\"}\n and\n hosted_control_plane_upgrade:duration_in_progress:seconds{cluster=~\"$cluster\"} > 1800\n) / 60", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Stuck in Progress Detail", + "transformations": [ + { "id": "seriesToRows", "options": {} }, + { "id": "merge", "options": {} }, + { + "id": "organize", + "options": { + "excludeByName": { "Time": true, "__name__": true, "state": true }, + "renameByName": { + "Value": "Time in Progress (Minutes)", + "cluster": "Service Cluster", + "cluster_uuid": "Cluster UUID", + "resource_id": "Resource ID", + "subscription_id": "Subscription", + "version": "Target Version" + } + } + } + ], + "type": "table" + } + ], + "schemaVersion": 39, + "tags": [ + "cluster-upgrade", + "aro-hcp", + "slo", + "user-journey" + ], + "templating": { + "list": [ + { + "allowCustomValue": false, + "current": { "text": "All", "value": "$__all" }, + "includeAll": true, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "^Managed_Prometheus_services-.*$", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { "text": "All", "value": "$__all" }, + "datasource": { "type": "prometheus", "uid": "$datasource" }, + "definition": "label_values(backend_cluster_version_info, cluster)", + "includeAll": true, + "multi": true, + "name": "cluster", + "query": "label_values(backend_cluster_version_info, cluster)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { "from": "now-7d", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "Cluster Upgrade SLO", + "uid": "cluster-upgrade-slo", + "version": 1 +} diff --git a/observability/recording-rules-services.yaml b/observability/recording-rules-services.yaml index 183d0e1fb97..b813cea529b 100644 --- a/observability/recording-rules-services.yaml +++ b/observability/recording-rules-services.yaml @@ -1,5 +1,6 @@ prometheusRules: rulesFolders: - recording-rules/access-cluster-slo-recordingRule.yaml + - recording-rules/userJourneyClusterUpgrade-recordingRule.yaml untestedRules: [] outputBicep: ../dev-infrastructure/modules/metrics/rules/generatedRecordingRules.bicep diff --git a/observability/recording-rules/userJourneyClusterUpgrade-recordingRule.yaml b/observability/recording-rules/userJourneyClusterUpgrade-recordingRule.yaml new file mode 100644 index 00000000000..0d59a31a32c --- /dev/null +++ b/observability/recording-rules/userJourneyClusterUpgrade-recordingRule.yaml @@ -0,0 +1,126 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + app.kubernetes.io/name: kube-prometheus + app.kubernetes.io/part-of: kube-prometheus + prometheus: k8s + role: recording-rules + name: user-journey-cluster-upgrade-recording-rules + namespace: monitoring +spec: + groups: + # ======================================== + # Cluster Upgrade recording rules + # ======================================== + # Eligibility, first-seen timestamps, in-progress fleet counts, and per-upgrade + # duration metrics used by stuck alerts and the cluster-upgrade SLO dashboard. + # + # upgrade_eligible:info must be evaluated before version_state_first_seen (same cycle). + # Eligibility: per resource_id, at least two distinct version labels and at least one completed. + # Excludes initial install. Cluster create bounds when install ends (operation_cluster_create.go): + # https://github.com/Azure/ARO-HCP/blob/main/backend/pkg/controllers/operationcontrollers/operation_cluster_create.go#L316-L320 + # + # Naming: level:metric:operations per https://prometheus.io/docs/practices/rules/#naming + - name: arohcp_user_journey_cluster_upgrade_recording_rules + interval: 1m + rules: + - record: hosted_control_plane_upgrade:upgrade_eligible:info + expr: | + ( + ( + count by (cluster, resource_id, subscription_id, cluster_uuid) ( + count by (cluster, resource_id, subscription_id, cluster_uuid, version) ( + backend_cluster_version_info == 1 + ) + ) >= 2 + ) + and on (cluster, resource_id) ( + count by (cluster, resource_id) ( + backend_cluster_version_info{state="completed"} == 1 + ) >= 1 + ) + ) * 0 + 1 + # When each upgrade target version first entered desired or partial (start time for SLO durations). + # Stop recording once the version reaches completed so latched timestamps do not persist forever. + - record: hosted_control_plane_upgrade:version_state_first_seen:timestamp + expr: | + min without(prometheus_replica) ( + min by (cluster, resource_id, subscription_id, cluster_uuid, version, state) ( + ( + hosted_control_plane_upgrade:version_state_first_seen:timestamp + or + ( + timestamp(backend_cluster_version_info{state=~"desired|partial"} == 1) + and on (cluster, resource_id) ( + hosted_control_plane_upgrade:upgrade_eligible:info == 1 + ) + ) + ) + unless on(cluster, resource_id, subscription_id, cluster_uuid, version) ( + backend_cluster_version_info{state="completed"} == 1 + ) + ) + ) + # Count of clusters with an upgrade currently in progress (desired or partial on a target version, without being completed). + - record: hosted_control_plane_upgrade:in_progress:count + expr: | + count by (cluster) ( + count by (cluster, resource_id) ( + ( + backend_cluster_version_info{state=~"desired|partial"} == 1 + unless on(cluster, resource_id, subscription_id, cluster_uuid, version) + backend_cluster_version_info{state="completed"} == 1 + ) + ) >= 1 + and on (cluster, resource_id) ( + hosted_control_plane_upgrade:upgrade_eligible:info == 1 + ) + ) + or + 0 * count by (cluster) (backend_cluster_version_info) + # Seconds the target version has been in desired without reaching partial or completed. + - record: hosted_control_plane_upgrade:duration_in_desired:seconds + expr: | + ( + time() - hosted_control_plane_upgrade:version_state_first_seen:timestamp{state="desired"} + ) + and on(cluster, resource_id, subscription_id, cluster_uuid, version) ( + ( + backend_cluster_version_info{state="desired"} == 1 + unless on(cluster, resource_id, subscription_id, cluster_uuid, version) ( + backend_cluster_version_info{state="partial"} == 1 + or + backend_cluster_version_info{state="completed"} == 1 + ) + ) + ) + and on (cluster, resource_id) ( + hosted_control_plane_upgrade:upgrade_eligible:info == 1 + ) + # Seconds since the target version was selected until it completes (desired or partial on a target version, without being completed). + - record: hosted_control_plane_upgrade:duration_in_progress:seconds + expr: | + ( + sum without(state) ( + ( + time() - hosted_control_plane_upgrade:version_state_first_seen:timestamp{state="desired"} + ) + and on (cluster, resource_id) ( + hosted_control_plane_upgrade:upgrade_eligible:info == 1 + ) + ) + unless on(cluster, resource_id, subscription_id, cluster_uuid, version) ( + backend_cluster_version_info{state="completed"} == 1 + ) + ) + * on(cluster, resource_id, subscription_id, cluster_uuid, version) group_left(state) + ( + backend_cluster_version_info{state="partial"} == 1 + or + ( + backend_cluster_version_info{state="desired"} == 1 + unless on(cluster, resource_id, subscription_id, cluster_uuid, version) + backend_cluster_version_info{state="partial"} == 1 + ) + ) diff --git a/observability/recording-rules/userJourneyClusterUpgrade-recordingRule_test.yaml b/observability/recording-rules/userJourneyClusterUpgrade-recordingRule_test.yaml new file mode 100644 index 00000000000..8a71dfff5d2 --- /dev/null +++ b/observability/recording-rules/userJourneyClusterUpgrade-recordingRule_test.yaml @@ -0,0 +1,228 @@ +rule_files: +- userJourneyClusterUpgrade-recordingRule.yaml +evaluation_interval: 1m +tests: +# ======================================== +# Test 1: Reach partial — desired transitions to partial within 20m (upgrade) +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed"}' + values: "1+0x35" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="desired"}' + values: "1x10 0x25" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="partial"}' + values: "0x10 1x25" + promql_expr_test: + - expr: hosted_control_plane_upgrade:upgrade_eligible:info{cluster="svc-1", resource_id="/sub/1/rg/c1"} + eval_time: 9m + exp_samples: + - labels: 'hosted_control_plane_upgrade:upgrade_eligible:info{cluster="svc-1", cluster_uuid="uuid-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1"}' + value: 1 + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 9m + exp_samples: + - labels: 'hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", cluster_uuid="uuid-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", version="4.19.20", state="desired"}' + value: 540 + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 15m + exp_samples: [] + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 25m + exp_samples: [] + - expr: hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20", state="partial"} + eval_time: 15m + exp_samples: + - labels: 'hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", cluster_uuid="uuid-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", version="4.19.20", state="partial"}' + value: 900 +# ======================================== +# Test 1b: Partial with stale desired — must not count as stuck in desired +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed"}' + values: "1+0x35" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="desired"}' + values: "1+0x35" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="partial"}' + values: "0x10 1x25" + promql_expr_test: + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 25m + exp_samples: [] +# ======================================== +# Test 1c: Completed with stale desired — must not count as stuck in desired or in progress +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed"}' + values: "1+0x35" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="desired"}' + values: "1+0x35" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="completed"}' + values: "0x10 1x25" + promql_expr_test: + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 25m + exp_samples: [] + - expr: hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 25m + exp_samples: [] + - expr: hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"} + eval_time: 25m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"}' + value: 0 +# ======================================== +# Test 2: Stuck in desired — 1 of 1 active upgrades stuck >20m +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.20", state="completed"}' + values: "1+0x50" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.21", state="desired"}' + values: "1+0x50" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c3", subscription_id="sub-2", cluster_uuid="uuid-3", version="4.19.20", state="completed"}' + values: "1+0x50" + promql_expr_test: + - expr: hosted_control_plane_upgrade:upgrade_eligible:info{cluster="svc-1", resource_id="/sub/2/rg/c2"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:upgrade_eligible:info{cluster="svc-1", cluster_uuid="uuid-2", resource_id="/sub/2/rg/c2", subscription_id="sub-2"}' + value: 1 + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/2/rg/c2", version="4.19.21", state="desired"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", cluster_uuid="uuid-2", resource_id="/sub/2/rg/c2", subscription_id="sub-2", version="4.19.21", state="desired"}' + value: 2100 + - expr: hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"}' + value: 1 +# ======================================== +# Test 3: Install only — desired without completed is excluded from SLO +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/3/rg/c4", subscription_id="sub-3", cluster_uuid="uuid-4", version="4.19.20", state="desired"}' + values: "1+0x50" + promql_expr_test: + - expr: hosted_control_plane_upgrade:upgrade_eligible:info{cluster="svc-1", resource_id="/sub/3/rg/c4"} + eval_time: 35m + exp_samples: [] + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/3/rg/c4", version="4.19.20", state="desired"} + eval_time: 35m + exp_samples: [] + - expr: hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"}' + value: 0 +# ======================================== +# Test 4: Complete — desired transitions to completed within 30m (upgrade) +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed"}' + values: "1+0x35" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="desired"}' + values: "1x10 0x25" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="completed"}' + values: "0x10 1x25" + promql_expr_test: + - expr: hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20", state="desired"} + eval_time: 9m + exp_samples: + - labels: 'hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", cluster_uuid="uuid-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", version="4.19.20", state="desired"}' + value: 540 + - expr: hosted_control_plane_upgrade:version_state_first_seen:timestamp{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 15m + exp_samples: [] + - expr: hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 15m + exp_samples: [] + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 15m + exp_samples: [] + - expr: hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"} + eval_time: 15m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"}' + value: 0 +# ======================================== +# Test 4b: Completed with stale partial — must not count as in progress or emit duration +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.19", state="completed"}' + values: "1+0x35" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="partial"}' + values: "1+0x35" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/1/rg/c1", subscription_id="sub-1", cluster_uuid="uuid-1", version="4.19.20", state="completed"}' + values: "1+0x35" + promql_expr_test: + - expr: hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"}' + value: 0 + - expr: hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", resource_id="/sub/1/rg/c1", version="4.19.20"} + eval_time: 35m + exp_samples: [] +# ======================================== +# Test 5: Stuck in progress — partial without completed after 35m +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.20", state="completed"}' + values: "1+0x50" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.21", state="desired"}' + values: "1x10 0x50" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c2", subscription_id="sub-2", cluster_uuid="uuid-2", version="4.19.21", state="partial"}' + values: "0x10 1x50" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/2/rg/c3", subscription_id="sub-2", cluster_uuid="uuid-3", version="4.19.20", state="completed"}' + values: "1+0x50" + promql_expr_test: + - expr: hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", resource_id="/sub/2/rg/c2", version="4.19.21", state="partial"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:duration_in_progress:seconds{cluster="svc-1", cluster_uuid="uuid-2", resource_id="/sub/2/rg/c2", subscription_id="sub-2", version="4.19.21", state="partial"}' + value: 2100 + - expr: hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"}' + value: 1 +# ======================================== +# Test 6: Two versions without completed — not upgrade eligible +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/4/rg/c5", subscription_id="sub-4", cluster_uuid="uuid-5", version="4.19.19", state="partial"}' + values: "1+0x50" + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/4/rg/c5", subscription_id="sub-4", cluster_uuid="uuid-5", version="4.19.20", state="desired"}' + values: "1+0x50" + promql_expr_test: + - expr: hosted_control_plane_upgrade:upgrade_eligible:info{cluster="svc-1", resource_id="/sub/4/rg/c5"} + eval_time: 35m + exp_samples: [] + - expr: hosted_control_plane_upgrade:duration_in_desired:seconds{cluster="svc-1", resource_id="/sub/4/rg/c5", version="4.19.20", state="desired"} + eval_time: 35m + exp_samples: [] + - expr: hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"} + eval_time: 35m + exp_samples: + - labels: 'hosted_control_plane_upgrade:in_progress:count{cluster="svc-1"}' + value: 0 +# ======================================== +# Test 7: Single completed version only — not upgrade eligible +# ======================================== +- interval: 1m + input_series: + - series: 'backend_cluster_version_info{cluster="svc-1", resource_id="/sub/5/rg/c6", subscription_id="sub-5", cluster_uuid="uuid-6", version="4.19.20", state="completed"}' + values: "1+0x50" + promql_expr_test: + - expr: hosted_control_plane_upgrade:upgrade_eligible:info{cluster="svc-1", resource_id="/sub/5/rg/c6"} + eval_time: 35m + exp_samples: []