Skip to content

grafana: make Cluster GPU Power dashboard vendor-neutral (NVIDIA + AMD) #111

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
martin-cala1 merged 1 commit into from
Sep 20, 2026
Merged

grafana: make Cluster GPU Power dashboard vendor-neutral (NVIDIA + AMD) #111

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 12 additions & 12 deletions grafana-cmk/dashboards/Crusoe/cluster-gpu-power.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@
}
]
},
"description": "Aggregate GPU power draw and energy consumption across the cluster. Data sourced from Crusoe Metrics \u2014 DCGM_FI_DEV_POWER_USAGE in watts, DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION in millijoules cumulative. SKU-agnostic: works for any NVIDIA GPU that DCGM reports power for.",
"description": "Aggregate GPU power draw and energy consumption across the cluster. Data sourced from Crusoe Metrics \u2014 DCGM_FI_DEV_POWER_USAGE in watts, DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION in millijoules cumulative. Vendor-neutral: uses NVIDIA DCGM (DCGM_FI_DEV_POWER_USAGE) or AMD device-metrics-exporter (gpu_power_usage), whichever the cluster exposes. AMD has no cumulative energy counter, so energy is derived from average power over the range.",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
Expand Down Expand Up @@ -134,7 +134,7 @@
"type": "prometheus",
"uid": "crusoe-metrics"
},
"expr": "sum(DCGM_FI_DEV_POWER_USAGE{cluster_id=~\"$cluster\"})",
"expr": "sum(gpu_power_usage{cluster_id=~\"$cluster\"}) or sum(DCGM_FI_DEV_POWER_USAGE{cluster_id=~\"$cluster\"})",
"instant": true,
"legendFormat": "",
"refId": "A"
Expand Down Expand Up @@ -196,7 +196,7 @@
"type": "prometheus",
"uid": "crusoe-metrics"
},
"expr": "avg(DCGM_FI_DEV_POWER_USAGE{cluster_id=~\"$cluster\"})",
"expr": "avg(gpu_power_usage{cluster_id=~\"$cluster\"}) or avg(DCGM_FI_DEV_POWER_USAGE{cluster_id=~\"$cluster\"})",
"instant": true,
"legendFormat": "",
"refId": "A"
Expand Down Expand Up @@ -258,7 +258,7 @@
"type": "prometheus",
"uid": "crusoe-metrics"
},
"expr": "sum(DCGM_FI_DEV_POWER_USAGE{cluster_id=~\"$cluster\"})",
"expr": "sum(gpu_power_usage{cluster_id=~\"$cluster\"}) or sum(DCGM_FI_DEV_POWER_USAGE{cluster_id=~\"$cluster\"})",
"instant": false,
"legendFormat": "",
"refId": "A"
Expand Down Expand Up @@ -321,7 +321,7 @@
"type": "prometheus",
"uid": "crusoe-metrics"
},
"expr": "sum(increase(DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION{cluster_id=~\"$cluster\"}[$__range])) / 3600000000",
"expr": "sum(avg_over_time(gpu_power_usage{cluster_id=~\"$cluster\"}[$__range])) * $__range_s / 3600000 or sum(increase(DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION{cluster_id=~\"$cluster\"}[$__range])) / 3600000000",
"instant": true,
"legendFormat": "",
"refId": "A"
Expand Down Expand Up @@ -417,7 +417,7 @@
"type": "prometheus",
"uid": "crusoe-metrics"
},
"expr": "sum(DCGM_FI_DEV_POWER_USAGE{cluster_id=~\"$cluster\"})",
"expr": "sum(gpu_power_usage{cluster_id=~\"$cluster\"}) or sum(DCGM_FI_DEV_POWER_USAGE{cluster_id=~\"$cluster\"})",
"legendFormat": "Total",
"refId": "A"
}
Expand Down Expand Up @@ -512,7 +512,7 @@
"type": "prometheus",
"uid": "crusoe-metrics"
},
"expr": "sum by (vm_name) (DCGM_FI_DEV_POWER_USAGE{cluster_id=~\"$cluster\"})",
"expr": "sum by (vm_name) (gpu_power_usage{cluster_id=~\"$cluster\"}) or sum by (vm_name) (DCGM_FI_DEV_POWER_USAGE{cluster_id=~\"$cluster\"})",
"legendFormat": "{{vm_name}}",
"refId": "A"
}
Expand Down Expand Up @@ -575,7 +575,7 @@
"type": "prometheus",
"uid": "crusoe-metrics"
},
"expr": "sum by (vm_name) (DCGM_FI_DEV_POWER_USAGE{cluster_id=~\"$cluster\"})",
"expr": "sum by (vm_name) (gpu_power_usage{cluster_id=~\"$cluster\"}) or sum by (vm_name) (DCGM_FI_DEV_POWER_USAGE{cluster_id=~\"$cluster\"})",
"instant": true,
"legendFormat": "{{vm_name}}",
"refId": "A"
Expand Down Expand Up @@ -707,7 +707,7 @@
"type": "prometheus",
"uid": "crusoe-metrics"
},
"expr": "sum(DCGM_FI_DEV_POWER_USAGE{cluster_id=~\"$cluster\"})",
"expr": "sum(gpu_power_usage{cluster_id=~\"$cluster\"}) or sum(DCGM_FI_DEV_POWER_USAGE{cluster_id=~\"$cluster\"})",
"legendFormat": "Total Power W",
"refId": "A"
},
Expand All @@ -716,7 +716,7 @@
"type": "prometheus",
"uid": "crusoe-metrics"
},
"expr": "avg(DCGM_FI_DEV_GPU_UTIL{cluster_id=~\"$cluster\"})",
"expr": "avg(gpu_gfx_activity{cluster_id=~\"$cluster\"}) or avg(DCGM_FI_DEV_GPU_UTIL{cluster_id=~\"$cluster\"})",
"legendFormat": "Avg Utilization %",
"refId": "B"
}
Expand Down Expand Up @@ -744,7 +744,7 @@
"type": "prometheus",
"uid": "crusoe-metrics"
},
"definition": "label_values(DCGM_FI_DEV_POWER_USAGE, cluster_id)",
"definition": "label_values({__name__=~\"gpu_power_usage|DCGM_FI_DEV_POWER_USAGE\"}, cluster_id)",
"hide": 0,
"includeAll": true,
"allValue": ".*",
Expand All @@ -753,7 +753,7 @@
"name": "cluster",
"options": [],
"query": {
"query": "label_values(DCGM_FI_DEV_POWER_USAGE, cluster_id)",
"query": "label_values({__name__=~\"gpu_power_usage|DCGM_FI_DEV_POWER_USAGE\"}, cluster_id)",
"refId": "StandardVariableQuery"
},
"refresh": 2,
Expand Down
Loading