From ab4c8dd08dc5d8f5255548539649e1de1dcd2700 Mon Sep 17 00:00:00 2001 From: Garrick Dabbs Date: Fri, 10 Jul 2026 09:56:18 -0700 Subject: [PATCH] Add GRPC operation metrics panels to Maestro dashboard Adds 4 new Grafana panels to visualize Maestro GRPC server metrics with operation-level granularity using the new non-breaking metrics. Panels added: 1. GRPC Request Rate by Operation - req/sec by operation and source 2. GRPC Error Rate by Operation - error % by operation 3. GRPC P95 Latency by Operation - 95th percentile latency 4. GRPC Message Rate by Operation - msg/sec by operation and source Uses new non-breaking metrics with '_by_operation' suffix: - grpc_server_called_total_by_operation - grpc_server_processed_total_by_operation - grpc_server_processed_duration_seconds_by_operation - grpc_server_message_received_total_by_operation No impact on existing dashboards - these are NEW panels using NEW metrics. Requires: openshift-online/maestro PR #555 Related: ARO-26754 Co-Authored-By: Claude Sonnet 4.5 --- .../maestro/maestro-server.json | 396 ++++++++++++++++++ 1 file changed, 396 insertions(+) diff --git a/observability/grafana-dashboards/maestro/maestro-server.json b/observability/grafana-dashboards/maestro/maestro-server.json index 0fa07eb9d9a..592e6ea680e 100644 --- a/observability/grafana-dashboards/maestro/maestro-server.json +++ b/observability/grafana-dashboards/maestro/maestro-server.json @@ -978,6 +978,402 @@ ], "title": "REST API Inbound Request Duration", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (operation, source) (rate(grpc_server_called_total_by_operation{namespace=~\"$namespace\", type=\"Publish\"}[5m]))", + "instant": false, + "legendFormat": "{{operation}} ({{source}})", + "range": true, + "refId": "A" + } + ], + "title": "GRPC Request Rate by Operation", + "type": "timeseries", + "description": "Rate of GRPC Publish requests by operation type (create, update, delete) and source consumer" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (operation) (rate(grpc_server_processed_total_by_operation{namespace=~\"$namespace\", code!=\"OK\"}[5m])) / sum by (operation) (rate(grpc_server_processed_total_by_operation{namespace=~\"$namespace\"}[5m]))", + "instant": false, + "legendFormat": "{{operation}}", + "range": true, + "refId": "A" + } + ], + "title": "GRPC Error Rate by Operation", + "type": "timeseries", + "description": "Error rate (non-OK status codes) by operation type" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (operation, le) (rate(grpc_server_processed_duration_seconds_by_operation_bucket{namespace=~\"$namespace\"}[5m])))", + "instant": false, + "legendFormat": "{{operation}}", + "range": true, + "refId": "A" + } + ], + "title": "GRPC P95 Latency by Operation", + "type": "timeseries", + "description": "95th percentile request latency by operation type" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "msgps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (operation, source) (rate(grpc_server_message_received_total_by_operation{namespace=~\"$namespace\"}[5m]))", + "instant": false, + "legendFormat": "{{operation}} ({{source}})", + "range": true, + "refId": "A" + } + ], + "title": "GRPC Message Rate by Operation", + "type": "timeseries", + "description": "Rate of messages received by operation type and source consumer" } ], "preload": false,