From 7994b5e76b64797174271b888d503b43a2e4d738 Mon Sep 17 00:00:00 2001 From: Laine Smith Date: Thu, 23 Jul 2026 11:17:29 -0400 Subject: [PATCH 01/12] Update README, add temporal_cloud_action_costs_alerts.yaml and temporal_cloud_action_costs.json --- cloud/README.md | 4 +- cloud/temporal_cloud_action_costs.json | 1056 +++++++++++++++++ cloud/temporal_cloud_action_costs_alerts.yaml | 135 +++ 3 files changed, 1194 insertions(+), 1 deletion(-) create mode 100644 cloud/temporal_cloud_action_costs.json create mode 100644 cloud/temporal_cloud_action_costs_alerts.yaml diff --git a/cloud/README.md b/cloud/README.md index 93d75a9..eede573 100644 --- a/cloud/README.md +++ b/cloud/README.md @@ -7,7 +7,9 @@ * [Temporal Cloud metrics](https://docs.temporal.io/production-deployment/cloud/metrics/reference) ## Dashboards -* **Grafana** [here](temporal_cloud_openmetrics.json) +* **Grafana** + * [Cloud Metrics - Workflows, Task Queue, Usage, Schedules, etc](temporal_cloud_openmetrics.json) + * [Cloud Action Cost Metrics](temporal_cloud_action_costs.json) and [Cloud Action Cost Sample Alerts](temporal_cloud_action_costs_alerts.yaml) * **DataDog** integration details and Dashboard access are found [here](https://docs.datadoghq.com/integrations/temporal-cloud/). * Related [Blog post](https://temporal.io/blog/temporal-cloud-metrics-in-datadog). diff --git a/cloud/temporal_cloud_action_costs.json b/cloud/temporal_cloud_action_costs.json new file mode 100644 index 0000000..a93e3d7 --- /dev/null +++ b/cloud/temporal_cloud_action_costs.json @@ -0,0 +1,1056 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "12.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + }, + { + "type": "panel", + "id": "row", + "name": "Row", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "uid": "${datasource}", + "type": "prometheus" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "title": "Temporal Cloud metrics reference", + "url": "https://docs.temporal.io/cloud/metrics/openmetrics/metrics-reference", + "type": "link", + "icon": "external link", + "tooltip": "", + "targetBlank": true, + "keepTime": false, + "includeVars": false, + "asDropdown": false, + "tags": [] + } + ], + "panels": [ + { + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "# Temporal Cloud \u2014 Actions Spent (MTD) & Spike Detection\n\nScope, on purpose: this dashboard answers two questions from OpenMetrics alone \u2014 **\"how many Actions have we used this month?\"** and **\"did something just start using way more Actions than usual?\"** It does not attempt to track spend against a commit or credit balance \u2014 Temporal Cloud doesn't expose price or credit-ledger balance via metrics, only Action volume (`temporal_cloud_v1_total_action_count`, `temporal_cloud_v1_billable_action_count`).\n\nSet the dashboard time range to **This month** for the MTD panels to read correctly.\n\n**Alerting**: the delta panels below (Action Rate \u0394 vs 24h/7d Ago) are paired with Grafana alert rules \u2014 see the companion `temporal_cloud_alert_rules.yaml` \u2014 so a spike notifies your team instead of only being visible to whoever happens to be looking at the dashboard.\n\n\n**Only seeing a few namespaces in the dropdown?** That's expected, not a bug. OpenMetrics uses sparse reporting - a namespace only appears in a given scrape if it generated at least one billable Action in that window. Idle/unused namespaces simply won't show up here, no matter how wide you set the time range, until they have real traffic.\n", + "mode": "markdown" + }, + "pluginVersion": "12.0.0", + "title": "About This Dashboard", + "type": "text" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 10, + "panels": [], + "title": "Summary", + "type": "row" + }, + { + "datasource": { + "uid": "${datasource}", + "type": "prometheus" + }, + "description": "Estimated total billable actions over the last 1 day", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 7 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "uid": "${datasource}", + "type": "prometheus" + }, + "editorMode": "code", + "expr": "sum(avg_over_time(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"}[1d]) * 86400)", + "legendFormat": "Actions", + "range": true, + "refId": "A" + } + ], + "title": "1 Day Actions", + "type": "stat" + }, + { + "datasource": { + "uid": "${datasource}", + "type": "prometheus" + }, + "description": "Estimated total billable actions over the last 7 days", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 7 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "uid": "${datasource}", + "type": "prometheus" + }, + "editorMode": "code", + "expr": "sum(avg_over_time(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"}[7d]) * 604800)", + "legendFormat": "Actions", + "range": true, + "refId": "A" + } + ], + "title": "7 Day Actions", + "type": "stat" + }, + { + "datasource": { + "uid": "${datasource}", + "type": "prometheus" + }, + "description": "Total estimated actions in the selected time range. Set the dashboard time range to 'This month' for an accurate month-to-date figure.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 7 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "uid": "${datasource}", + "type": "prometheus" + }, + "editorMode": "code", + "expr": "sum(avg_over_time(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"}[$__range]) * $__range_s)", + "legendFormat": "Actions", + "range": true, + "refId": "A" + } + ], + "title": "Actions Spent This Month (MTD)", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 20, + "panels": [], + "title": "Action Spike Detection (\"who just started using a lot of actions?\")", + "type": "row" + }, + { + "datasource": { + "uid": "${datasource}", + "type": "prometheus" + }, + "description": "Overlay of the current action rate against the same time yesterday and the same time last week. A line pulling away from its baselines is the first sign of an unannounced change driving usage.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 21, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "uid": "${datasource}", + "type": "prometheus" + }, + "editorMode": "code", + "expr": "sum(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"})", + "legendFormat": "now", + "range": true, + "refId": "A" + }, + { + "datasource": { + "uid": "${datasource}", + "type": "prometheus" + }, + "editorMode": "code", + "expr": "sum(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"} offset 1d)", + "legendFormat": "24h ago", + "range": true, + "refId": "B" + }, + { + "datasource": { + "uid": "${datasource}", + "type": "prometheus" + }, + "editorMode": "code", + "expr": "sum(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"} offset 7d)", + "legendFormat": "7d ago", + "range": true, + "refId": "C" + } + ], + "title": "Action Rate: Now vs 24h Ago vs 7d Ago", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${datasource}", + "type": "prometheus" + }, + "description": "Percent change in current action rate vs the same time yesterday. Large positive swings flag an unannounced usage spike. Paired with an alert rule of the same name in temporal_cloud_alert_rules.yaml.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 25 + }, + { + "color": "red", + "value": 50 + } + ] + }, + "unit": "percent", + "decimals": 1 + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 4, + "x": 12, + "y": 17 + }, + "id": 22, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "uid": "${datasource}", + "type": "prometheus" + }, + "editorMode": "code", + "expr": "(sum(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"}) - sum(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"} offset 1d)) / sum(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"} offset 1d) * 100", + "legendFormat": "% change", + "range": true, + "refId": "A" + } + ], + "title": "Action Rate \u0394 vs 24h Ago", + "type": "stat" + }, + { + "datasource": { + "uid": "${datasource}", + "type": "prometheus" + }, + "description": "Percent change in current action rate vs the same time last week. Paired with an alert rule of the same name in temporal_cloud_alert_rules.yaml.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 25 + }, + { + "color": "red", + "value": 75 + } + ] + }, + "unit": "percent", + "decimals": 1 + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 4, + "x": 16, + "y": 17 + }, + "id": 23, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "uid": "${datasource}", + "type": "prometheus" + }, + "editorMode": "code", + "expr": "(sum(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"}) - sum(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"} offset 7d)) / sum(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"} offset 7d) * 100", + "legendFormat": "% change", + "range": true, + "refId": "A" + } + ], + "title": "Action Rate \u0394 vs 7d Ago", + "type": "stat" + }, + { + "datasource": { + "uid": "${datasource}", + "type": "prometheus" + }, + "description": "Current number of open workflows in scope", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 4, + "x": 20, + "y": 17 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "uid": "${datasource}", + "type": "prometheus" + }, + "editorMode": "code", + "expr": "sum(temporal_cloud_v1_namespace_open_workflows{temporal_namespace=~\"$temporal_namespace\"})", + "legendFormat": "open", + "range": true, + "refId": "A" + } + ], + "title": "Open Workflows", + "type": "stat" + }, + { + "datasource": { + "uid": "${datasource}", + "type": "prometheus" + }, + "description": "Breaks the action rate down by Workflow Type so a newly-released or newly-busy workflow shows up immediately instead of hiding inside the account total.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "uid": "${datasource}", + "type": "prometheus" + }, + "editorMode": "code", + "expr": "topk(8, sum by (temporal_workflow_type) (temporal_cloud_v1_billable_action_count{temporal_namespace=~\"$temporal_namespace\"}))", + "legendFormat": "{{temporal_workflow_type}}", + "range": true, + "refId": "A" + } + ], + "title": "Billable Action Rate by Workflow Type (Top 8)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${datasource}", + "type": "prometheus" + }, + "description": "Breaks the action rate down by Action Type (Workflow started, Activity, Signal, Timer, etc.) to show which billable operation is driving a spike.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 26, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "uid": "${datasource}", + "type": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (action_type) (temporal_cloud_v1_billable_action_count{temporal_namespace=~\"$temporal_namespace\"})", + "legendFormat": "{{action_type}}", + "range": true, + "refId": "A" + } + ], + "title": "Billable Action Rate by Action Type", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${datasource}", + "type": "prometheus" + }, + "description": "Ranked table of Workflow Types by estimated billable actions over the selected time range. Sort or widen the time range to spot which workflow type is the biggest driver of usage.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 27, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": [ + "Value" + ], + "reducer": [ + "sum" + ], + "show": true + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Actions/sec" + } + ] + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "uid": "${datasource}", + "type": "prometheus" + }, + "editorMode": "code", + "expr": "topk(20, sum by (temporal_workflow_type, temporal_namespace) (avg_over_time(temporal_cloud_v1_billable_action_count{temporal_namespace=~\"$temporal_namespace\"}[$__range]) * $__range_s))", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Top Workflow Types \u2014 Actions in Selected Range", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": { + "temporal_workflow_type": 0, + "temporal_namespace": 1, + "Value": 2 + }, + "renameByName": { + "Value": "Estimated Actions", + "temporal_namespace": "Namespace", + "temporal_workflow_type": "Workflow Type" + } + } + } + ], + "type": "table" + } + ], + "refresh": "5m", + "schemaVersion": 41, + "tags": [ + "temporal", + "cloud", + "actions", + "mtd" + ], + "templating": { + "list": [ + { + "allowCustomValue": false, + "current": {}, + "description": "Prometheus Data source", + "label": "Data source", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "allowCustomValue": false, + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "${datasource}", + "type": "prometheus" + }, + "definition": "label_values(temporal_cloud_v1_total_action_count,temporal_namespace)", + "includeAll": true, + "name": "temporal_namespace", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(temporal_cloud_v1_total_action_count,temporal_namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query", + "multi": true + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Temporal Cloud - Actions MTD & Spike Detection", + "uid": null, + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/cloud/temporal_cloud_action_costs_alerts.yaml b/cloud/temporal_cloud_action_costs_alerts.yaml new file mode 100644 index 0000000..8e31317 --- /dev/null +++ b/cloud/temporal_cloud_action_costs_alerts.yaml @@ -0,0 +1,135 @@ +# Temporal Cloud action-spike alert rules for Grafana (unified alerting). +# +# This is a provisioning file, not a dashboard - it is imported separately from +# temporal_cloud_action_costs.json. +# +# Instructions: +# 1. Replace every PROM_DS_UID placeholder with your own Prometheus data source's UID (find it under Connections -> Data sources -> [your source] -> Settings - it's in the page URL +# and in the data source's JSON model). +# 2. Load the file into Grafana. +# 3. Create a contact point (Slack webhook, email, PagerDuty, etc.) in Alerting -> Contact points, and a notification policy that routes alerts labeled category: +# temporal-cost to it. The rules intentionally don't hardcode a contact point, since that's specific to your team. +# +# Rules included +# Action rate spike vs 24h ago: Fires when current action rate is >50% above the same time yesterday, sustained for 10 min +# Action rate spike vs 7d ago: Fires when current action rate is >75% above the same time last week, sustained for 10 min +# +# Before use, replace every occurrence of PROM_DS_UID below with the UID of your +# own Prometheus data source (Connections > Data sources > your source > Settings). +# +# The thresholds here (50%, 75%, 80%) are reasonable starting points, not tuned to +# any specific account. A namespace with naturally spiky, low-volume traffic will +# trip the 24h/7d rules on ordinary variance; a very high, steady-volume namespace +# may want tighter thresholds to catch smaller-but-still-costly increases. Watch +# these for a week or two after setup and adjust the "params" values to match your +# own traffic pattern. + +apiVersion: 1 + +groups: + - orgId: 1 + name: temporal-cloud-action-spikes + folder: Temporal Cloud + interval: 5m + rules: + - uid: tcloud-action-spike-24h + title: Temporal Cloud - action rate spike vs 24h ago + condition: C + for: 10m + noDataState: NoData + execErrState: Error + labels: + category: temporal-cost + severity: warning + annotations: + summary: "Action rate is {{ $values.A }}% higher than the same time yesterday." + description: > + Current Temporal Cloud action rate has moved more than 50% away from + the same time 24 hours ago. This usually means a deploy, new workflow + type, or bug started driving a lot more billable Actions than usual. + Check the "Billable Action Rate by Workflow Type" panel on the + Actions MTD & Spike Detection dashboard to see which Workflow Type + is responsible. + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PROM_DS_UID + model: + editorMode: code + expr: >- + (sum(temporal_cloud_v1_total_action_count{temporal_namespace=~".*"}) + - sum(temporal_cloud_v1_total_action_count{temporal_namespace=~".*"} offset 1d)) + / sum(temporal_cloud_v1_total_action_count{temporal_namespace=~".*"} offset 1d) * 100 + instant: true + range: false + refId: A + - refId: B + datasourceUid: __expr__ + model: + type: reduce + expression: A + reducer: last + refId: B + - refId: C + datasourceUid: __expr__ + model: + type: threshold + expression: B + conditions: + # Percent-change threshold - tune to your own traffic pattern (see note above). + - evaluator: + type: gt + params: [50] + refId: C + + - uid: tcloud-action-spike-7d + title: Temporal Cloud - action rate spike vs 7d ago + condition: C + for: 10m + noDataState: NoData + execErrState: Error + labels: + category: temporal-cost + severity: warning + annotations: + summary: "Action rate is {{ $values.A }}% higher than the same time last week." + description: > + Current Temporal Cloud action rate has moved more than 75% away from + the same time 7 days ago. Week-over-week naturally varies more than + day-over-day, so this threshold is looser than the 24h rule - treat + it as a slower-moving confirmation of a sustained usage change + rather than a same-day spike. + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PROM_DS_UID + model: + editorMode: code + expr: >- + (sum(temporal_cloud_v1_total_action_count{temporal_namespace=~".*"}) + - sum(temporal_cloud_v1_total_action_count{temporal_namespace=~".*"} offset 7d)) + / sum(temporal_cloud_v1_total_action_count{temporal_namespace=~".*"} offset 7d) * 100 + instant: true + range: false + refId: A + - refId: B + datasourceUid: __expr__ + model: + type: reduce + expression: A + reducer: last + refId: B + - refId: C + datasourceUid: __expr__ + model: + type: threshold + expression: B + conditions: + - evaluator: + type: gt + params: [75] + refId: C \ No newline at end of file From 9b868dd9cacb4185eec1c1ae6c0a3483fc8538af Mon Sep 17 00:00:00 2001 From: Laine Smith <39713102+lainecsmith@users.noreply.github.com> Date: Thu, 23 Jul 2026 14:15:38 -0400 Subject: [PATCH 02/12] Update alert rule references in JSON descriptions --- cloud/temporal_cloud_action_costs.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cloud/temporal_cloud_action_costs.json b/cloud/temporal_cloud_action_costs.json index a93e3d7..85c4edb 100644 --- a/cloud/temporal_cloud_action_costs.json +++ b/cloud/temporal_cloud_action_costs.json @@ -477,7 +477,7 @@ "uid": "${datasource}", "type": "prometheus" }, - "description": "Percent change in current action rate vs the same time yesterday. Large positive swings flag an unannounced usage spike. Paired with an alert rule of the same name in temporal_cloud_alert_rules.yaml.", + "description": "Percent change in current action rate vs the same time yesterday. Large positive swings flag an unannounced usage spike. Paired with an alert rule of the same name in temporal_cloud_action_costs_alerts.yaml.", "fieldConfig": { "defaults": { "color": { @@ -551,7 +551,7 @@ "uid": "${datasource}", "type": "prometheus" }, - "description": "Percent change in current action rate vs the same time last week. Paired with an alert rule of the same name in temporal_cloud_alert_rules.yaml.", + "description": "Percent change in current action rate vs the same time last week. Paired with an alert rule of the same name in temporal_cloud_action_costs_alerts.yaml.", "fieldConfig": { "defaults": { "color": { @@ -1053,4 +1053,4 @@ "uid": null, "version": 1, "weekStart": "" -} \ No newline at end of file +} From 10228666814e53839e0a39983c933fa7cba84b93 Mon Sep 17 00:00:00 2001 From: Laine Smith <39713102+lainecsmith@users.noreply.github.com> Date: Thu, 23 Jul 2026 14:16:32 -0400 Subject: [PATCH 03/12] Update display name for Actions/sec to Estimated Actions/sec --- cloud/temporal_cloud_action_costs.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/temporal_cloud_action_costs.json b/cloud/temporal_cloud_action_costs.json index 85c4edb..b81f3f5 100644 --- a/cloud/temporal_cloud_action_costs.json +++ b/cloud/temporal_cloud_action_costs.json @@ -947,7 +947,7 @@ "sortBy": [ { "desc": true, - "displayName": "Actions/sec" + "displayName": "Estimated Actions/sec" } ] }, From 26f4b52e3bfd5b7808bb82cdcb7fe666dd5cf3e5 Mon Sep 17 00:00:00 2001 From: Laine Smith <39713102+lainecsmith@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:45:35 -0400 Subject: [PATCH 04/12] Fix for PR finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- cloud/temporal_cloud_action_costs.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/temporal_cloud_action_costs.json b/cloud/temporal_cloud_action_costs.json index b81f3f5..e82d61b 100644 --- a/cloud/temporal_cloud_action_costs.json +++ b/cloud/temporal_cloud_action_costs.json @@ -109,7 +109,7 @@ "showLineNumbers": false, "showMiniMap": false }, - "content": "# Temporal Cloud \u2014 Actions Spent (MTD) & Spike Detection\n\nScope, on purpose: this dashboard answers two questions from OpenMetrics alone \u2014 **\"how many Actions have we used this month?\"** and **\"did something just start using way more Actions than usual?\"** It does not attempt to track spend against a commit or credit balance \u2014 Temporal Cloud doesn't expose price or credit-ledger balance via metrics, only Action volume (`temporal_cloud_v1_total_action_count`, `temporal_cloud_v1_billable_action_count`).\n\nSet the dashboard time range to **This month** for the MTD panels to read correctly.\n\n**Alerting**: the delta panels below (Action Rate \u0394 vs 24h/7d Ago) are paired with Grafana alert rules \u2014 see the companion `temporal_cloud_alert_rules.yaml` \u2014 so a spike notifies your team instead of only being visible to whoever happens to be looking at the dashboard.\n\n\n**Only seeing a few namespaces in the dropdown?** That's expected, not a bug. OpenMetrics uses sparse reporting - a namespace only appears in a given scrape if it generated at least one billable Action in that window. Idle/unused namespaces simply won't show up here, no matter how wide you set the time range, until they have real traffic.\n", + "content": "# Temporal Cloud \u2014 Actions Spent (MTD) & Spike Detection\n\nScope, on purpose: this dashboard answers two questions from OpenMetrics alone \u2014 **\"how many Actions have we used this month?\"** and **\"did something just start using way more Actions than usual?\"** It does not attempt to track spend against a commit or credit balance \u2014 Temporal Cloud doesn't expose price or credit-ledger balance via metrics, only Action volume (`temporal_cloud_v1_total_action_count`, `temporal_cloud_v1_billable_action_count`).\n\nSet the dashboard time range to **This month** for the MTD panels to read correctly.\n\n**Alerting**: the delta panels below (Action Rate \u0394 vs 24h/7d Ago) are paired with Grafana alert rules \u2014 see the companion `temporal_cloud_action_costs_alerts.yaml` \u2014 so a spike notifies your team instead of only being visible to whoever happens to be looking at the dashboard.\n\n\n**Only seeing a few namespaces in the dropdown?** That's expected, not a bug. OpenMetrics uses sparse reporting - a namespace only appears in a given scrape if it generated at least one billable Action in that window. Idle/unused namespaces simply won't show up here, no matter how wide you set the time range, until they have real traffic.\n", "mode": "markdown" }, "pluginVersion": "12.0.0", From 5b763f3c9a137c398f8117079bb033496e534ad8 Mon Sep 17 00:00:00 2001 From: Laine Smith <39713102+lainecsmith@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:46:45 -0400 Subject: [PATCH 05/12] Fix for PR finding to avoid potential divide by 0 issues Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- cloud/temporal_cloud_action_costs.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/temporal_cloud_action_costs.json b/cloud/temporal_cloud_action_costs.json index e82d61b..61cfb8b 100644 --- a/cloud/temporal_cloud_action_costs.json +++ b/cloud/temporal_cloud_action_costs.json @@ -537,7 +537,7 @@ "type": "prometheus" }, "editorMode": "code", - "expr": "(sum(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"}) - sum(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"} offset 1d)) / sum(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"} offset 1d) * 100", + "expr": "(sum(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"}) - sum(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"} offset 1d)) / clamp_min(sum(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"} offset 1d), 1e-9) * 100", "legendFormat": "% change", "range": true, "refId": "A" From 8437ac3a1cfee310d1944a669024d863d2cebbc4 Mon Sep 17 00:00:00 2001 From: Laine Smith <39713102+lainecsmith@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:47:11 -0400 Subject: [PATCH 06/12] Fix for PR finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- cloud/temporal_cloud_action_costs.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/temporal_cloud_action_costs.json b/cloud/temporal_cloud_action_costs.json index 61cfb8b..25c4b3c 100644 --- a/cloud/temporal_cloud_action_costs.json +++ b/cloud/temporal_cloud_action_costs.json @@ -947,7 +947,7 @@ "sortBy": [ { "desc": true, - "displayName": "Estimated Actions/sec" + "displayName": "Estimated Actions" } ] }, From f174e12a04cc80b6f5615d2d5342ef7482ba28b3 Mon Sep 17 00:00:00 2001 From: Laine Smith <39713102+lainecsmith@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:47:27 -0400 Subject: [PATCH 07/12] Fix for PR finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- cloud/temporal_cloud_action_costs_alerts.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/temporal_cloud_action_costs_alerts.yaml b/cloud/temporal_cloud_action_costs_alerts.yaml index 8e31317..c25b783 100644 --- a/cloud/temporal_cloud_action_costs_alerts.yaml +++ b/cloud/temporal_cloud_action_costs_alerts.yaml @@ -11,7 +11,7 @@ # temporal-cost to it. The rules intentionally don't hardcode a contact point, since that's specific to your team. # # Rules included -# Action rate spike vs 24h ago: Fires when current action rate is >50% above the same time yesterday, sustained for 10 min +# Action rate spike vs 24h ago: Fires when current action rate is >50% above the same time yesterday, sustained for 10 min # Action rate spike vs 7d ago: Fires when current action rate is >75% above the same time last week, sustained for 10 min # # Before use, replace every occurrence of PROM_DS_UID below with the UID of your From 72b88b65bc9070343f50d0a46454850699a0876d Mon Sep 17 00:00:00 2001 From: Laine Smith <39713102+lainecsmith@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:47:54 -0400 Subject: [PATCH 08/12] Fix for PR finding to avoid divide by 0 Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- cloud/temporal_cloud_action_costs_alerts.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/temporal_cloud_action_costs_alerts.yaml b/cloud/temporal_cloud_action_costs_alerts.yaml index c25b783..0b1a010 100644 --- a/cloud/temporal_cloud_action_costs_alerts.yaml +++ b/cloud/temporal_cloud_action_costs_alerts.yaml @@ -61,7 +61,7 @@ groups: expr: >- (sum(temporal_cloud_v1_total_action_count{temporal_namespace=~".*"}) - sum(temporal_cloud_v1_total_action_count{temporal_namespace=~".*"} offset 1d)) - / sum(temporal_cloud_v1_total_action_count{temporal_namespace=~".*"} offset 1d) * 100 + / clamp_min(sum(temporal_cloud_v1_total_action_count{temporal_namespace=~".*"} offset 1d), 1e-9) * 100 instant: true range: false refId: A From 034fd750e7c358afc67da82b9826ec8521c48614 Mon Sep 17 00:00:00 2001 From: Laine Smith <39713102+lainecsmith@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:48:22 -0400 Subject: [PATCH 09/12] Fix for PR finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- cloud/temporal_cloud_action_costs.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/temporal_cloud_action_costs.json b/cloud/temporal_cloud_action_costs.json index 25c4b3c..06e7b9a 100644 --- a/cloud/temporal_cloud_action_costs.json +++ b/cloud/temporal_cloud_action_costs.json @@ -936,7 +936,7 @@ "footer": { "countRows": false, "fields": [ - "Value" + "Estimated Actions" ], "reducer": [ "sum" From 663961626b6d8a72f84ff214a48c5bc28b27168e Mon Sep 17 00:00:00 2001 From: Laine Smith <39713102+lainecsmith@users.noreply.github.com> Date: Fri, 14 Aug 2026 14:26:25 -0400 Subject: [PATCH 10/12] Refine action rate alert rules in Grafana Updated alert rules for action rate spikes and limits in Grafana. Adjusted summaries and descriptions for clarity and accuracy. --- cloud/temporal_cloud_action_costs_alerts.yaml | 119 +++++++++++++----- 1 file changed, 89 insertions(+), 30 deletions(-) diff --git a/cloud/temporal_cloud_action_costs_alerts.yaml b/cloud/temporal_cloud_action_costs_alerts.yaml index 0b1a010..5b9cb76 100644 --- a/cloud/temporal_cloud_action_costs_alerts.yaml +++ b/cloud/temporal_cloud_action_costs_alerts.yaml @@ -1,22 +1,17 @@ # Temporal Cloud action-spike alert rules for Grafana (unified alerting). # # This is a provisioning file, not a dashboard - it is imported separately from -# temporal_cloud_action_costs.json. -# -# Instructions: -# 1. Replace every PROM_DS_UID placeholder with your own Prometheus data source's UID (find it under Connections -> Data sources -> [your source] -> Settings - it's in the page URL -# and in the data source's JSON model). -# 2. Load the file into Grafana. -# 3. Create a contact point (Slack webhook, email, PagerDuty, etc.) in Alerting -> Contact points, and a notification policy that routes alerts labeled category: -# temporal-cost to it. The rules intentionally don't hardcode a contact point, since that's specific to your team. -# -# Rules included -# Action rate spike vs 24h ago: Fires when current action rate is >50% above the same time yesterday, sustained for 10 min -# Action rate spike vs 7d ago: Fires when current action rate is >75% above the same time last week, sustained for 10 min +# temporal_cloud_actions_mtd.json. See the README in this folder for setup steps. # # Before use, replace every occurrence of PROM_DS_UID below with the UID of your # own Prometheus data source (Connections > Data sources > your source > Settings). # +# These rules link back to specific panels on temporal_cloud_actions_mtd.json via +# __dashboardUid__/__panelId__ annotations, so a firing alert can jump straight to +# the relevant chart. This only works because that dashboard's uid is pinned to +# "temporal-cloud-action-costs" - if you rename or re-key the dashboard's uid on +# import, update the __dashboardUid__ values below to match. +# # The thresholds here (50%, 75%, 80%) are reasonable starting points, not tuned to # any specific account. A namespace with naturally spiky, low-volume traffic will # trip the 24h/7d rules on ordinary variance; a very high, steady-volume namespace @@ -42,14 +37,21 @@ groups: category: temporal-cost severity: warning annotations: - summary: "Action rate is {{ $values.A }}% higher than the same time yesterday." + summary: '{{ if $values.A }}Action rate for namespace {{ $labels.temporal_namespace }} is {{ printf "%.1f" $values.A.Value }}% above{{ else }}Action rate change unknown vs{{ end }} the same time yesterday.' + __dashboardUid__: temporal-cloud-action-costs + __panelId__: "25" description: > - Current Temporal Cloud action rate has moved more than 50% away from - the same time 24 hours ago. This usually means a deploy, new workflow - type, or bug started driving a lot more billable Actions than usual. - Check the "Billable Action Rate by Workflow Type" panel on the - Actions MTD & Spike Detection dashboard to see which Workflow Type - is responsible. + The action rate for namespace {{ $labels.temporal_namespace }} has + increased by more than 50% compared to the same time 24 hours ago. + This rule fires per-namespace specifically so a spike in a smaller + namespace isn't diluted/hidden by a much larger steady namespace in + the same account. This usually means a deploy, new workflow type, or + bug started driving a lot more billable Actions than usual in this + namespace. (This rule only catches increases - a sudden drop in + action rate, e.g. from workers going down, won't trigger it.) Check + the "Billable Action Rate by Workflow Type" panel, filtered to this + namespace, on the Actions MTD & Spike Detection dashboard to see + which Workflow Type is responsible. data: - refId: A relativeTimeRange: @@ -59,9 +61,9 @@ groups: model: editorMode: code expr: >- - (sum(temporal_cloud_v1_total_action_count{temporal_namespace=~".*"}) - - sum(temporal_cloud_v1_total_action_count{temporal_namespace=~".*"} offset 1d)) - / clamp_min(sum(temporal_cloud_v1_total_action_count{temporal_namespace=~".*"} offset 1d), 1e-9) * 100 + (sum by (temporal_namespace) (temporal_cloud_v1_total_action_count{temporal_namespace=~".*"}) + - (sum by (temporal_namespace) (temporal_cloud_v1_total_action_count{temporal_namespace=~".*"} offset 1d) or (sum by (temporal_namespace) (temporal_cloud_v1_total_action_count{temporal_namespace=~".*"}) * 0))) + / clamp_min(sum by (temporal_namespace) (temporal_cloud_v1_total_action_count{temporal_namespace=~".*"} offset 1d) or (sum by (temporal_namespace) (temporal_cloud_v1_total_action_count{temporal_namespace=~".*"}) * 0), 1) * 100 instant: true range: false refId: A @@ -94,13 +96,19 @@ groups: category: temporal-cost severity: warning annotations: - summary: "Action rate is {{ $values.A }}% higher than the same time last week." + summary: '{{ if $values.A }}Action rate for namespace {{ $labels.temporal_namespace }} is {{ printf "%.1f" $values.A.Value }}% above{{ else }}Action rate change unknown vs{{ end }} the same time last week.' + __dashboardUid__: temporal-cloud-action-costs + __panelId__: "25" description: > - Current Temporal Cloud action rate has moved more than 75% away from - the same time 7 days ago. Week-over-week naturally varies more than + The action rate for namespace {{ $labels.temporal_namespace }} has + increased by more than 75% compared to the same time 7 days ago. + This rule fires per-namespace specifically so a spike in a smaller + namespace isn't diluted/hidden by a much larger steady namespace in + the same account. Week-over-week naturally varies more than day-over-day, so this threshold is looser than the 24h rule - treat it as a slower-moving confirmation of a sustained usage change - rather than a same-day spike. + rather than a same-day spike. (This rule only catches increases, + not drops.) data: - refId: A relativeTimeRange: @@ -110,9 +118,9 @@ groups: model: editorMode: code expr: >- - (sum(temporal_cloud_v1_total_action_count{temporal_namespace=~".*"}) - - sum(temporal_cloud_v1_total_action_count{temporal_namespace=~".*"} offset 7d)) - / sum(temporal_cloud_v1_total_action_count{temporal_namespace=~".*"} offset 7d) * 100 + (sum by (temporal_namespace) (temporal_cloud_v1_total_action_count{temporal_namespace=~".*"}) + - (sum by (temporal_namespace) (temporal_cloud_v1_total_action_count{temporal_namespace=~".*"} offset 7d) or (sum by (temporal_namespace) (temporal_cloud_v1_total_action_count{temporal_namespace=~".*"}) * 0))) + / clamp_min(sum by (temporal_namespace) (temporal_cloud_v1_total_action_count{temporal_namespace=~".*"} offset 7d) or (sum by (temporal_namespace) (temporal_cloud_v1_total_action_count{temporal_namespace=~".*"}) * 0), 1) * 100 instant: true range: false refId: A @@ -132,4 +140,55 @@ groups: - evaluator: type: gt params: [75] - refId: C \ No newline at end of file + refId: C + + - uid: tcloud-action-limit-approaching + title: Temporal Cloud - approaching action rate limit + condition: C + for: 5m + noDataState: NoData + execErrState: Error + labels: + category: temporal-cost + severity: critical + annotations: + summary: '{{ if $values.A }}Namespace {{ $labels.temporal_namespace }} is at {{ printf "%.1f" $values.A.Value }}% of its action rate limit{{ else }}Action rate vs limit unknown{{ end }}.' + __dashboardUid__: temporal-cloud-action-costs + description: > + Namespace {{ $labels.temporal_namespace }}'s actions-per-second is + above 80% of its own configured temporal_cloud_v1_action_limit. This + rule is per-namespace since the limit itself is per-namespace - an + account-wide comparison would let one saturated namespace hide + behind other idle ones. Sustained proximity to the limit risks + throttling (temporal_cloud_v1_total_action_throttled_count). + data: + - refId: A + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: PROM_DS_UID + model: + editorMode: code + expr: >- + sum by (temporal_namespace) (temporal_cloud_v1_total_action_count{temporal_namespace=~".*", is_background="false"}) + / sum by (temporal_namespace) (temporal_cloud_v1_action_limit{temporal_namespace=~".*"}) * 100 + instant: true + range: false + refId: A + - refId: B + datasourceUid: __expr__ + model: + type: reduce + expression: A + reducer: last + refId: B + - refId: C + datasourceUid: __expr__ + model: + type: threshold + expression: B + conditions: + - evaluator: + type: gt + params: [80] + refId: C From 0bc7fb25a290332f1ccf91b5e663a5b9e44a2035 Mon Sep 17 00:00:00 2001 From: Laine Smith <39713102+lainecsmith@users.noreply.github.com> Date: Fri, 14 Aug 2026 14:33:22 -0400 Subject: [PATCH 11/12] Refactor temporal_cloud_action_costs.json structure Removed unused inputs and updated descriptions and expressions for various panels to improve clarity and accuracy. --- cloud/temporal_cloud_action_costs.json | 42 ++++++++++---------------- 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/cloud/temporal_cloud_action_costs.json b/cloud/temporal_cloud_action_costs.json index 06e7b9a..fe760d9 100644 --- a/cloud/temporal_cloud_action_costs.json +++ b/cloud/temporal_cloud_action_costs.json @@ -1,14 +1,4 @@ { - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], "__elements": {}, "__requires": [ { @@ -109,7 +99,7 @@ "showLineNumbers": false, "showMiniMap": false }, - "content": "# Temporal Cloud \u2014 Actions Spent (MTD) & Spike Detection\n\nScope, on purpose: this dashboard answers two questions from OpenMetrics alone \u2014 **\"how many Actions have we used this month?\"** and **\"did something just start using way more Actions than usual?\"** It does not attempt to track spend against a commit or credit balance \u2014 Temporal Cloud doesn't expose price or credit-ledger balance via metrics, only Action volume (`temporal_cloud_v1_total_action_count`, `temporal_cloud_v1_billable_action_count`).\n\nSet the dashboard time range to **This month** for the MTD panels to read correctly.\n\n**Alerting**: the delta panels below (Action Rate \u0394 vs 24h/7d Ago) are paired with Grafana alert rules \u2014 see the companion `temporal_cloud_action_costs_alerts.yaml` \u2014 so a spike notifies your team instead of only being visible to whoever happens to be looking at the dashboard.\n\n\n**Only seeing a few namespaces in the dropdown?** That's expected, not a bug. OpenMetrics uses sparse reporting - a namespace only appears in a given scrape if it generated at least one billable Action in that window. Idle/unused namespaces simply won't show up here, no matter how wide you set the time range, until they have real traffic.\n", + "content": "# Temporal Cloud \u2014 Actions Spent (MTD) & Spike Detection\n\nScope, on purpose: this dashboard answers two questions from OpenMetrics alone \u2014 **\"how many Actions have we used this month?\"** and **\"did something just start using way more Actions than usual?\"** It does not attempt to track spend against a commit or credit balance \u2014 Temporal Cloud doesn't expose price or credit-ledger balance via metrics, only Action volume (`temporal_cloud_v1_total_action_count`, `temporal_cloud_v1_billable_action_count`).\n\nThe dashboard defaults to **month-to-date** (start of month through now). If you change the time picker, avoid Grafana's built-in \"This month\" quick-pick \u2014 it rounds the end of the range up to the last day of the month rather than stopping at now, so the label will visually claim more days than have actually happened (the totals themselves are still accurate either way, since only real data gets summed - but the labeled range would be misleading). Use a custom range with \"to: now\" instead.\n\n**Alerting**: the delta panels below (Action Rate \u0394 vs 24h/7d Ago) are paired with Grafana alert rules \u2014 see the companion `temporal_cloud_action_costs_alerts.yaml` \u2014 so a spike notifies your team instead of only being visible to whoever happens to be looking at the dashboard. **Scope note:** the alert rules evaluate per-namespace (so a spike in a small namespace isn't masked by a much larger, steady one), but the two \u0394 stat panels on this dashboard are still account-wide aggregates. If you get paged for a specific namespace, filter the `Namespace` variable above to that namespace before checking these panels \u2014 looking at them with \"All\" selected can understate or hide the very spike that paged you.\n\n\n**Only seeing a few namespaces in the dropdown?** That's expected, not a bug. OpenMetrics uses sparse reporting - a namespace only appears in a given scrape if it generated at least one billable Action in that window. Idle/unused namespaces simply won't show up here, no matter how wide you set the time range, until they have real traffic.\n", "mode": "markdown" }, "pluginVersion": "12.0.0", @@ -189,7 +179,7 @@ "type": "prometheus" }, "editorMode": "code", - "expr": "sum(avg_over_time(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"}[1d]) * 86400)", + "expr": "sum(sum_over_time(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"}[1d:1m])) * 60", "legendFormat": "Actions", "range": true, "refId": "A" @@ -258,7 +248,7 @@ "type": "prometheus" }, "editorMode": "code", - "expr": "sum(avg_over_time(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"}[7d]) * 604800)", + "expr": "sum(sum_over_time(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"}[7d:1m])) * 60", "legendFormat": "Actions", "range": true, "refId": "A" @@ -272,7 +262,7 @@ "uid": "${datasource}", "type": "prometheus" }, - "description": "Total estimated actions in the selected time range. Set the dashboard time range to 'This month' for an accurate month-to-date figure.", + "description": "Total estimated actions in the selected time range. Defaults to month-to-date (now/M to now). If you customize the time range, avoid Grafana's built-in 'This month' quick-pick - it rounds the end of the range up to month-end rather than stopping at now, so the displayed range would visually claim days that haven't happened yet, even though the total itself stays accurate.", "fieldConfig": { "defaults": { "color": { @@ -328,7 +318,7 @@ "type": "prometheus" }, "editorMode": "code", - "expr": "sum(avg_over_time(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"}[$__range]) * $__range_s)", + "expr": "sum(sum_over_time(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"}[$__range:1m])) * 60", "legendFormat": "Actions", "range": true, "refId": "A" @@ -477,7 +467,7 @@ "uid": "${datasource}", "type": "prometheus" }, - "description": "Percent change in current action rate vs the same time yesterday. Large positive swings flag an unannounced usage spike. Paired with an alert rule of the same name in temporal_cloud_action_costs_alerts.yaml.", + "description": "Percent change in current action rate vs the same time yesterday. Large positive swings flag an unannounced usage spike. Paired with the matching per-namespace alert rule in temporal_cloud_action_costs_alerts.yaml (this panel itself is account-wide, not per-namespace - see note below).", "fieldConfig": { "defaults": { "color": { @@ -537,7 +527,7 @@ "type": "prometheus" }, "editorMode": "code", - "expr": "(sum(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"}) - sum(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"} offset 1d)) / clamp_min(sum(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"} offset 1d), 1e-9) * 100", + "expr": "(sum(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"}) - (sum(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"} offset 1d) or vector(0))) / clamp_min(sum(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"} offset 1d) or vector(0), 1) * 100", "legendFormat": "% change", "range": true, "refId": "A" @@ -551,7 +541,7 @@ "uid": "${datasource}", "type": "prometheus" }, - "description": "Percent change in current action rate vs the same time last week. Paired with an alert rule of the same name in temporal_cloud_action_costs_alerts.yaml.", + "description": "Percent change in current action rate vs the same time last week. Paired with the matching per-namespace alert rule in temporal_cloud_action_costs_alerts.yaml (this panel itself is account-wide, not per-namespace - see note below).", "fieldConfig": { "defaults": { "color": { @@ -611,7 +601,7 @@ "type": "prometheus" }, "editorMode": "code", - "expr": "(sum(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"}) - sum(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"} offset 7d)) / sum(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"} offset 7d) * 100", + "expr": "(sum(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"}) - (sum(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"} offset 7d) or vector(0))) / clamp_min(sum(temporal_cloud_v1_total_action_count{temporal_namespace=~\"$temporal_namespace\"} offset 7d) or vector(0), 1) * 100", "legendFormat": "% change", "range": true, "refId": "A" @@ -780,8 +770,8 @@ "type": "prometheus" }, "editorMode": "code", - "expr": "topk(8, sum by (temporal_workflow_type) (temporal_cloud_v1_billable_action_count{temporal_namespace=~\"$temporal_namespace\"}))", - "legendFormat": "{{temporal_workflow_type}}", + "expr": "sum by (temporal_workflow_type, temporal_namespace) (temporal_cloud_v1_billable_action_count{temporal_namespace=~\"$temporal_namespace\"})\nand\ntopk(8, sum by (temporal_workflow_type, temporal_namespace) (\n sum_over_time(temporal_cloud_v1_billable_action_count{temporal_namespace=~\"$temporal_namespace\"}[$__range:1m] @ end())\n))", + "legendFormat": "{{temporal_workflow_type}} ({{temporal_namespace}})", "range": true, "refId": "A" } @@ -894,7 +884,7 @@ "uid": "${datasource}", "type": "prometheus" }, - "description": "Ranked table of Workflow Types by estimated billable actions over the selected time range. Sort or widen the time range to spot which workflow type is the biggest driver of usage.", + "description": "Ranked table of the top 20 Workflow Types by estimated billable actions over the selected time range. Sort or widen the time range to spot which workflow type is the biggest driver of usage. Note: the footer total reflects only these top 20 rows, not the full account \u2014 if you have more than 20 distinct Workflow Types, this will undercount the true account-wide total.", "fieldConfig": { "defaults": { "color": { @@ -959,7 +949,7 @@ "type": "prometheus" }, "editorMode": "code", - "expr": "topk(20, sum by (temporal_workflow_type, temporal_namespace) (avg_over_time(temporal_cloud_v1_billable_action_count{temporal_namespace=~\"$temporal_namespace\"}[$__range]) * $__range_s))", + "expr": "topk(20, sum by (temporal_workflow_type, temporal_namespace) (\n sum_over_time(temporal_cloud_v1_billable_action_count{temporal_namespace=~\"$temporal_namespace\"}[$__range:1m])\n) * 60)", "format": "table", "instant": true, "interval": "", @@ -968,7 +958,7 @@ "refId": "A" } ], - "title": "Top Workflow Types \u2014 Actions in Selected Range", + "title": "Top 20 Workflow Types \u2014 Actions in Selected Range", "transformations": [ { "id": "organize", @@ -1044,13 +1034,13 @@ ] }, "time": { - "from": "now-7d", + "from": "now/M", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Temporal Cloud - Actions MTD & Spike Detection", - "uid": null, + "uid": "temporal-cloud-action-costs", "version": 1, "weekStart": "" } From d3980b6409713c5345a2f5c2d7eb9a9b2f0880c4 Mon Sep 17 00:00:00 2001 From: Laine Smith <39713102+lainecsmith@users.noreply.github.com> Date: Fri, 14 Aug 2026 14:33:48 -0400 Subject: [PATCH 12/12] Update references in alert rules file --- cloud/temporal_cloud_action_costs_alerts.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloud/temporal_cloud_action_costs_alerts.yaml b/cloud/temporal_cloud_action_costs_alerts.yaml index 5b9cb76..2ae5c45 100644 --- a/cloud/temporal_cloud_action_costs_alerts.yaml +++ b/cloud/temporal_cloud_action_costs_alerts.yaml @@ -1,12 +1,12 @@ # Temporal Cloud action-spike alert rules for Grafana (unified alerting). # # This is a provisioning file, not a dashboard - it is imported separately from -# temporal_cloud_actions_mtd.json. See the README in this folder for setup steps. +# temporal_cloud_action_costs.json. See the README in this folder for setup steps. # # Before use, replace every occurrence of PROM_DS_UID below with the UID of your # own Prometheus data source (Connections > Data sources > your source > Settings). # -# These rules link back to specific panels on temporal_cloud_actions_mtd.json via +# These rules link back to specific panels on temporal_cloud_action_costs.json via # __dashboardUid__/__panelId__ annotations, so a firing alert can jump straight to # the relevant chart. This only works because that dashboard's uid is pinned to # "temporal-cloud-action-costs" - if you rename or re-key the dashboard's uid on