From 7812ac6ff92703582e4e2600ef902199699468bc Mon Sep 17 00:00:00 2001 From: "pardhubharadhwaz.vangipuram" Date: Mon, 24 Aug 2026 17:06:37 +0530 Subject: [PATCH] feat(self-hosted): namespace selection, runtime tag fix, non-k8s metrics bridge Validated against a live Temporal Server 1.31.2 deployment: - Add a multi-select Temporal Namespace variable and apply namespace filtering to every namespace-bearing panel (service, workflow/task, matching/backlog, persistence); leave cache/runtime panels global since they carry no namespace. - Group server runtime metrics (restarts, num_goroutines, memory_heap) by service_name, the tag actually emitted (not temporal_service_type); sync docs. - Add temporal-self-hosted-metrics-bridge.sh + systemd unit to forward server /metrics to a Sumo HTTP source for EC2/VM/Docker (non-Kubernetes) deployments. --- dashboards/sumo/metric-to-panel-mapping.md | 2 +- .../sumo/temporal-self-hosted/README.md | 28 +++++-- .../temporal-server-metrics-dashboard.json | 80 +++++++++---------- sumo/metric-field-mapping.md | 12 +-- ...emporal-self-hosted-metrics-bridge.service | 28 +++++++ sumo/temporal-self-hosted-metrics-bridge.sh | 47 +++++++++++ sumo/validation-queries.md | 4 +- 7 files changed, 144 insertions(+), 57 deletions(-) create mode 100644 sumo/temporal-self-hosted-metrics-bridge.service create mode 100755 sumo/temporal-self-hosted-metrics-bridge.sh diff --git a/dashboards/sumo/metric-to-panel-mapping.md b/dashboards/sumo/metric-to-panel-mapping.md index 39abf5c..f5764f7 100644 --- a/dashboards/sumo/metric-to-panel-mapping.md +++ b/dashboards/sumo/metric-to-panel-mapping.md @@ -42,7 +42,7 @@ dimension; scraped server metrics do not carry one. | Matching / task queue | `approximate_backlog_count`, `approximate_backlog_age_seconds`, `no_poller_tasks`, `poll_success`, `poll_success_sync`, `poll_timeouts` | | Persistence | `persistence_requests`, `persistence_errors`, `persistence_errors_resource_exhausted`, `persistence_latency_sum`/`_count` | | History cache | `cache_size`, `cache_usage`, `cache_pinned_usage` by `cache_type` | -| Server runtime | `restarts`, `num_goroutines`, `memory_heap` by `temporal_service_type` | +| Server runtime | `restarts`, `num_goroutines`, `memory_heap` by `service_name` | | Kubernetes correlation | `kube_pod_container_status_restarts_total`, `container_cpu_usage_seconds_total`, `container_memory_working_set_bytes` | Latency uses the average form `rate(*_latency_sum)/rate(*_latency_count)`: diff --git a/dashboards/sumo/temporal-self-hosted/README.md b/dashboards/sumo/temporal-self-hosted/README.md index e93cfa0..b44f8a4 100644 --- a/dashboards/sumo/temporal-self-hosted/README.md +++ b/dashboards/sumo/temporal-self-hosted/README.md @@ -25,7 +25,7 @@ Sections: - **Persistence** — `persistence_requests`, `persistence_errors`, error ratio, average `persistence_latency`, `persistence_errors_resource_exhausted`. - **History Cache & Server Runtime** — `cache_usage`/`cache_size` by `cache_type`, - `restarts`/`num_goroutines`/`memory_heap` by `temporal_service_type`. + `restarts`/`num_goroutines`/`memory_heap` by `service_name`. - **Infrastructure Correlation** — Kubernetes pod restarts, CPU, and memory for Temporal pods (Sumo Kubernetes Collection). @@ -35,10 +35,14 @@ dashboards. ## Dependencies -The dashboard expects native Temporal Server metrics scraped in-cluster by the -Sumo Kubernetes Collection from the server `/metrics` endpoint via -`prometheus.io/*` pod annotations. A dedicated Cloud-style metrics collector is -**not** required. See `sumo/temporal-self-hosted-collection-values.example.yaml`. +The dashboard expects native Temporal Server metrics collected into Sumo: +- **Kubernetes:** scrape in-cluster with the Sumo Kubernetes Collection from the + server `/metrics` endpoint via `prometheus.io/*` pod annotations. A dedicated + Cloud-style collector is **not** required. See + `sumo/temporal-self-hosted-collection-values.example.yaml`. +- **EC2 / VM / Docker (non-Kubernetes):** forward the server `/metrics` to a Sumo + HTTP Logs & Metrics source with `sumo/temporal-self-hosted-metrics-bridge.sh` + (run continuously via `temporal-self-hosted-metrics-bridge.service`). Primary dimensions: - `service_name` — the native role tag (`frontend`, `history`, `matching`, @@ -47,12 +51,20 @@ Primary dimensions: - `operation` — RPC or persistence operation. - `namespace` — Temporal namespace. - `cache_type` — history cache family. -- `temporal_service_type` — role tag on server runtime metrics (`restarts`, etc.). +- `service_name` also tags server runtime metrics (`restarts`, `num_goroutines`, `memory_heap`) in current server versions (verified on 1.31.2). ## Variables -- `service_name` (Service Role) — default `*`; also `frontend`/`history`/`matching`/`worker`. -- `namespace` (Temporal Namespace) — default `*`. +- `service_name` (Service Role) — default `*`; also `frontend`/`history`/`matching`/`worker`/`server`. +- `namespace` (Temporal Namespace) — **multi-select** dropdown, default `*`, populated + from live values (`metric=persistence_requests`, key `namespace`). Select one or more + namespaces (e.g. `ameriprise_agentic`) to scope the namespace-bearing panels. + +Namespace filtering is applied only to panels whose metrics carry a `namespace` +label (service, workflow/task, matching/backlog, persistence). Server-wide panels +that have no namespace dimension — history cache (`cache_*`) and runtime +(`restarts`, `num_goroutines`, `memory_heap`) — are intentionally left global so a +namespace selection does not blank them. There is intentionally **no** `service` variable: scraped server metrics do not carry a `service` dimension, so filtering by one would blank every panel. diff --git a/dashboards/sumo/temporal-self-hosted/temporal-server-metrics-dashboard.json b/dashboards/sumo/temporal-self-hosted/temporal-server-metrics-dashboard.json index 105cf38..94a0111 100644 --- a/dashboards/sumo/temporal-self-hosted/temporal-server-metrics-dashboard.json +++ b/dashboards/sumo/temporal-self-hosted/temporal-server-metrics-dashboard.json @@ -1,7 +1,7 @@ { "type": "DashboardV2SyncDefinition", "name": "Temporal Self-Hosted Server Metrics", - "description": "Self-hosted Temporal Server observability: service-role health, workflow/task traffic, topology, matching/backlog, persistence, history cache, server runtime, and Kubernetes correlation. Metric families verified against Temporal metric_defs.go and the official server dashboards.", + "description": "Self-hosted Temporal Server observability with per-namespace selection: service-role health, workflow/task traffic, topology, matching/backlog, persistence, history cache, server runtime, and Kubernetes correlation. Metric families and tags validated against Temporal Server 1.31.2.", "title": "Temporal Self-Hosted Server Metrics", "theme": "Light", "topologyLabelMap": { @@ -211,7 +211,7 @@ { "queryKey": "A", "queryType": "Metrics", - "queryString": "metric=service_requests service_name={{service_name}} | sum by service_name", + "queryString": "metric=service_requests service_name={{service_name}} namespace={{namespace}} | sum by service_name", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -234,7 +234,7 @@ { "queryKey": "A", "queryType": "Metrics", - "queryString": "metric=service_errors service_name={{service_name}} | sum by service_name", + "queryString": "metric=service_errors service_name={{service_name}} namespace={{namespace}} | sum by service_name", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -257,7 +257,7 @@ { "queryKey": "A", "queryType": "Metrics", - "queryString": "metric=service_errors service_name={{service_name}} | sum", + "queryString": "metric=service_errors service_name={{service_name}} namespace={{namespace}} | sum", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -267,7 +267,7 @@ { "queryKey": "B", "queryType": "Metrics", - "queryString": "metric=service_requests service_name={{service_name}} | sum", + "queryString": "metric=service_requests service_name={{service_name}} namespace={{namespace}} | sum", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -300,7 +300,7 @@ { "queryKey": "A", "queryType": "Metrics", - "queryString": "metric=service_latency_sum service_name={{service_name}} | sum by operation", + "queryString": "metric=service_latency_sum service_name={{service_name}} namespace={{namespace}} | sum by operation", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -310,7 +310,7 @@ { "queryKey": "B", "queryType": "Metrics", - "queryString": "metric=service_latency_count service_name={{service_name}} | sum by operation", + "queryString": "metric=service_latency_count service_name={{service_name}} namespace={{namespace}} | sum by operation", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -343,7 +343,7 @@ { "queryKey": "A", "queryType": "Metrics", - "queryString": "metric=service_pending_requests service_name={{service_name}} | avg by service_name", + "queryString": "metric=service_pending_requests service_name={{service_name}} namespace={{namespace}} | avg by service_name", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -366,7 +366,7 @@ { "queryKey": "A", "queryType": "Metrics", - "queryString": "metric=client_errors service_name={{service_name}} | sum by operation", + "queryString": "metric=client_errors service_name={{service_name}} namespace={{namespace}} | sum by operation", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -398,7 +398,7 @@ { "queryKey": "A", "queryType": "Metrics", - "queryString": "metric=service_requests service_name=frontend | sum by operation", + "queryString": "metric=service_requests service_name=frontend namespace={{namespace}} | sum by operation", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -421,7 +421,7 @@ { "queryKey": "A", "queryType": "Metrics", - "queryString": "metric=action service_name=frontend | rate", + "queryString": "metric=action service_name=frontend namespace={{namespace}} | rate", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -444,7 +444,7 @@ { "queryKey": "A", "queryType": "Metrics", - "queryString": "metric=workflow_success | sum", + "queryString": "metric=workflow_success namespace={{namespace}} | sum", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -467,7 +467,7 @@ { "queryKey": "A", "queryType": "Metrics", - "queryString": "metric=workflow_failed | sum", + "queryString": "metric=workflow_failed namespace={{namespace}} | sum", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -490,7 +490,7 @@ { "queryKey": "A", "queryType": "Metrics", - "queryString": "metric=schedule_to_start_timeout | sum by operation", + "queryString": "metric=schedule_to_start_timeout namespace={{namespace}} | sum by operation", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -513,7 +513,7 @@ { "queryKey": "A", "queryType": "Metrics", - "queryString": "metric=start_to_close_timeout | sum by operation", + "queryString": "metric=start_to_close_timeout namespace={{namespace}} | sum by operation", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -545,7 +545,7 @@ { "queryKey": "A", "queryType": "Metrics", - "queryString": "metric=service_requests service_name=frontend | sum by operation", + "queryString": "metric=service_requests service_name=frontend namespace={{namespace}} | sum by operation", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -568,7 +568,7 @@ { "queryKey": "A", "queryType": "Metrics", - "queryString": "metric=service_requests service_name=history | sum by operation", + "queryString": "metric=service_requests service_name=history namespace={{namespace}} | sum by operation", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -591,7 +591,7 @@ { "queryKey": "A", "queryType": "Metrics", - "queryString": "metric=service_requests service_name=matching | sum by operation", + "queryString": "metric=service_requests service_name=matching namespace={{namespace}} | sum by operation", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -614,7 +614,7 @@ { "queryKey": "A", "queryType": "Metrics", - "queryString": "metric=service_requests service_name=worker | sum by operation", + "queryString": "metric=service_requests service_name=worker namespace={{namespace}} | sum by operation", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -715,7 +715,7 @@ { "queryKey": "A", "queryType": "Metrics", - "queryString": "metric=poll_success | sum", + "queryString": "metric=poll_success namespace={{namespace}} | sum", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -738,7 +738,7 @@ { "queryKey": "A", "queryType": "Metrics", - "queryString": "metric=poll_success_sync | sum", + "queryString": "metric=poll_success_sync namespace={{namespace}} | sum", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -761,7 +761,7 @@ { "queryKey": "A", "queryType": "Metrics", - "queryString": "metric=poll_timeouts | sum", + "queryString": "metric=poll_timeouts namespace={{namespace}} | sum", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -793,7 +793,7 @@ { "queryKey": "A", "queryType": "Metrics", - "queryString": "metric=persistence_requests | sum by operation", + "queryString": "metric=persistence_requests namespace={{namespace}} | sum by operation", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -816,7 +816,7 @@ { "queryKey": "A", "queryType": "Metrics", - "queryString": "metric=persistence_errors | sum by operation", + "queryString": "metric=persistence_errors namespace={{namespace}} | sum by operation", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -839,7 +839,7 @@ { "queryKey": "A", "queryType": "Metrics", - "queryString": "metric=persistence_errors | sum", + "queryString": "metric=persistence_errors namespace={{namespace}} | sum", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -849,7 +849,7 @@ { "queryKey": "B", "queryType": "Metrics", - "queryString": "metric=persistence_requests | sum", + "queryString": "metric=persistence_requests namespace={{namespace}} | sum", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -882,7 +882,7 @@ { "queryKey": "A", "queryType": "Metrics", - "queryString": "metric=persistence_latency_sum | sum by operation", + "queryString": "metric=persistence_latency_sum namespace={{namespace}} | sum by operation", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -892,7 +892,7 @@ { "queryKey": "B", "queryType": "Metrics", - "queryString": "metric=persistence_latency_count | sum by operation", + "queryString": "metric=persistence_latency_count namespace={{namespace}} | sum by operation", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -925,7 +925,7 @@ { "queryKey": "A", "queryType": "Metrics", - "queryString": "metric=persistence_errors_resource_exhausted | sum", + "queryString": "metric=persistence_errors_resource_exhausted namespace={{namespace}} | sum", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -949,7 +949,7 @@ "id": null, "key": "panel-33", "title": "History Cache Usage", - "description": "cache_usage by cache_type (validate cache_type values for the release).", + "description": "cache_usage by cache_type (server-wide; no namespace dimension).", "visualSettings": "{\"general\": {\"mode\": \"timeSeries\", \"type\": \"line\"}, \"series\": {}}", "keepVisualSettingsConsistentWithParent": true, "panelType": "SumoSearchPanel", @@ -972,7 +972,7 @@ "id": null, "key": "panel-34", "title": "History Cache Size", - "description": "cache_size by cache_type.", + "description": "cache_size by cache_type (server-wide).", "visualSettings": "{\"general\": {\"mode\": \"timeSeries\", \"type\": \"line\"}, \"series\": {}}", "keepVisualSettingsConsistentWithParent": true, "panelType": "SumoSearchPanel", @@ -995,7 +995,7 @@ "id": null, "key": "panel-35", "title": "Server Restarts", - "description": "Process restarts (restarts) by temporal_service_type; corroborate with K8s restart/OOM.", + "description": "Process restarts (restarts) by service_name; corroborate with K8s restart/OOM.", "visualSettings": "{\"general\": {\"mode\": \"timeSeries\", \"type\": \"column\", \"displayType\": \"stacked\"}, \"series\": {}}", "keepVisualSettingsConsistentWithParent": true, "panelType": "SumoSearchPanel", @@ -1003,7 +1003,7 @@ { "queryKey": "A", "queryType": "Metrics", - "queryString": "metric=restarts | sum by temporal_service_type", + "queryString": "metric=restarts | sum by service_name", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -1018,7 +1018,7 @@ "id": null, "key": "panel-36", "title": "Goroutines", - "description": "num_goroutines by temporal_service_type.", + "description": "num_goroutines by service_name.", "visualSettings": "{\"general\": {\"mode\": \"timeSeries\", \"type\": \"line\"}, \"series\": {}}", "keepVisualSettingsConsistentWithParent": true, "panelType": "SumoSearchPanel", @@ -1026,7 +1026,7 @@ { "queryKey": "A", "queryType": "Metrics", - "queryString": "metric=num_goroutines | avg by temporal_service_type", + "queryString": "metric=num_goroutines | avg by service_name", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -1041,7 +1041,7 @@ "id": null, "key": "panel-37", "title": "Heap Memory", - "description": "memory_heap by temporal_service_type; compare against pod memory limits.", + "description": "memory_heap by service_name; compare against pod memory limits.", "visualSettings": "{\"general\": {\"mode\": \"timeSeries\", \"type\": \"line\"}, \"series\": {}}", "keepVisualSettingsConsistentWithParent": true, "panelType": "SumoSearchPanel", @@ -1049,7 +1049,7 @@ { "queryKey": "A", "queryType": "Metrics", - "queryString": "metric=memory_heap | avg by temporal_service_type", + "queryString": "metric=memory_heap | avg by service_name", "metricsQueryMode": "Advanced", "metricsQueryData": null, "tracesQueryData": null, @@ -1146,7 +1146,7 @@ "defaultValue": "*", "sourceDefinition": { "variableSourceType": "CsvVariableSourceDefinition", - "values": "*,frontend,history,matching,worker" + "values": "*,frontend,history,matching,worker,server" }, "allowMultiSelect": false, "includeAllOption": true, @@ -1158,10 +1158,10 @@ "defaultValue": "*", "sourceDefinition": { "variableSourceType": "MetadataVariableSourceDefinition", - "filter": "metric=service_requests", + "filter": "metric=persistence_requests", "key": "namespace" }, - "allowMultiSelect": false, + "allowMultiSelect": true, "includeAllOption": true, "hideFromUI": false } diff --git a/sumo/metric-field-mapping.md b/sumo/metric-field-mapping.md index 12d1d4c..da95f4c 100644 --- a/sumo/metric-field-mapping.md +++ b/sumo/metric-field-mapping.md @@ -82,16 +82,16 @@ dashboards. | `persistence_errors_resource_exhausted` | Capacity/resource-exhaustion persistence errors, separated from general errors. | | `persistence_latency` (`_bucket`/`_sum`/`_count`) | Persistence latency histogram. | | `cache_size` / `cache_usage` / `cache_pinned_usage` | History cache families, grouped by `cache_type`. | -| `restarts` | Server process restart counter (by `temporal_service_type`). | +| `restarts` | Server process restart counter (by `service_name`). | | `num_goroutines` / `memory_heap` / `memory_heapinuse` | Go runtime pressure; compare against pod CPU/memory limits. | | `action` | Server action counter. Self-hosted has no SaaS action-limit/throttle telemetry. | | `schedule_to_start_timeout` / `start_to_close_timeout` | Task pickup / execution timeout counters (by `operation`). | -Recommended dimensions: `service_name` (`frontend`/`history`/`matching`/`worker`), -`operation`, `namespace`, `cache_type`, `temporal_service_type`. Temporal's native -role tag is `service_name`; `service_role` also exists but Temporal's own -dashboards group by `service_name`. Do **not** rely on an invented `temporal_role` -tag. +Recommended dimensions: `service_name` (`frontend`/`history`/`matching`/`worker`/`server`), +`operation`, `namespace`, `cache_type`. The native role tag is `service_name` — it also +tags server runtime metrics (`restarts`, `num_goroutines`, `memory_heap`), verified on +Temporal Server 1.31.2. `service_role` also exists but Temporal's own dashboards group by +`service_name`; do **not** rely on an invented `temporal_role` tag. Collection and naming caveats for the EKS path: diff --git a/sumo/temporal-self-hosted-metrics-bridge.service b/sumo/temporal-self-hosted-metrics-bridge.service new file mode 100644 index 0000000..36da3a8 --- /dev/null +++ b/sumo/temporal-self-hosted-metrics-bridge.service @@ -0,0 +1,28 @@ +# systemd unit for the Temporal Self-Hosted -> Sumo Logic metrics bridge. +# Install on the host that can reach the Temporal server metrics port (ideally the +# Temporal host itself, using localhost:6051 — fast and complete, no WAN truncation). +# +# Install: +# sudo install -m 0755 temporal-self-hosted-metrics-bridge.sh /usr/local/bin/temporal-self-hosted-metrics-bridge.sh +# printf 'SUMO_METRICS_URL=%s\n' 'https://endpointN.collection.sumologic.com/receiver/v1/http/XXXX' | sudo tee /etc/temporal-sumo-bridge.env >/dev/null +# sudo chmod 600 /etc/temporal-sumo-bridge.env +# sudo cp temporal-self-hosted-metrics-bridge.service /etc/systemd/system/ +# sudo systemctl daemon-reload && sudo systemctl enable --now temporal-self-hosted-metrics-bridge +# journalctl -u temporal-self-hosted-metrics-bridge -f +[Unit] +Description=Temporal Self-Hosted -> Sumo Logic metrics bridge +After=network-online.target +Wants=network-online.target + +[Service] +Environment=METRICS_ENDPOINT=http://localhost:6051/metrics +Environment=INTERVAL_SECONDS=60 +# SUMO_METRICS_URL is provided here (kept out of git, chmod 600): +EnvironmentFile=/etc/temporal-sumo-bridge.env +ExecStart=/usr/local/bin/temporal-self-hosted-metrics-bridge.sh +Restart=always +RestartSec=10 +DynamicUser=yes + +[Install] +WantedBy=multi-user.target diff --git a/sumo/temporal-self-hosted-metrics-bridge.sh b/sumo/temporal-self-hosted-metrics-bridge.sh new file mode 100755 index 0000000..9560756 --- /dev/null +++ b/sumo/temporal-self-hosted-metrics-bridge.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Continuously scrape a self-hosted Temporal Server /metrics endpoint and forward +# the dashboard-relevant metric families to a Sumo Logic HTTP Logs & Metrics source +# in Prometheus format. +# +# Use this for EC2/VM/Docker self-hosted deployments that are NOT on Kubernetes. +# (On Kubernetes, prefer the Sumo Kubernetes Collection annotation scrape in +# temporal-self-hosted-collection-values.example.yaml instead of this bridge.) +# +# Required env: +# SUMO_METRICS_URL Sumo HTTP Logs & Metrics source URL. KEEP SECRET; never commit. +# Optional env: +# METRICS_ENDPOINT Temporal server Prometheus endpoint. Default http://localhost:6051/metrics +# INTERVAL_SECONDS Scrape/forward interval. Default 60 +# +# Notes: +# - --compressed is required: the endpoint is large (100k+ lines on busy multi-namespace +# servers) and an uncompressed pull over a WAN can truncate mid-stream. +# - Only the families the self-hosted dashboard uses are forwarded, and histogram +# *_bucket series are dropped, to keep ingestion cardinality/cost low. Latency panels +# use *_sum / *_count, which are kept. +set -euo pipefail + +: "${SUMO_METRICS_URL:?set SUMO_METRICS_URL (Sumo HTTP source; keep it out of git)}" +ENDPOINT="${METRICS_ENDPOINT:-http://localhost:6051/metrics}" +INTERVAL="${INTERVAL_SECONDS:-60}" + +FAMILIES='service_requests|service_errors|service_pending_requests|service_latency_sum|service_latency_count|client_errors|action|workflow_success|workflow_failed|schedule_to_start_timeout|start_to_close_timeout|approximate_backlog_count|approximate_backlog_age_seconds|no_poller_tasks|poll_success|poll_success_sync|poll_timeouts|persistence_requests|persistence_errors|persistence_errors_resource_exhausted|persistence_latency_sum|persistence_latency_count|cache_size|cache_usage|restarts|num_goroutines|memory_heap' + +tmp="$(mktemp)" +trap 'rm -f "$tmp"' EXIT + +echo "$(date -u +%FT%TZ) bridge start: $ENDPOINT -> Sumo every ${INTERVAL}s" +while true; do + if curl -fsS --compressed --max-time "$INTERVAL" "$ENDPOINT" 2>/dev/null \ + | grep -E "^(${FAMILIES})(\{| )" > "$tmp"; then + if curl -fsS -X POST -H "Content-Type: application/vnd.sumologic.prometheus" \ + --data-binary @"$tmp" "$SUMO_METRICS_URL" >/dev/null 2>&1; then + echo "$(date -u +%FT%TZ) pushed $(wc -l < "$tmp") series" + else + echo "$(date -u +%FT%TZ) push failed" + fi + else + echo "$(date -u +%FT%TZ) scrape failed from $ENDPOINT" + fi + sleep "$INTERVAL" +done diff --git a/sumo/validation-queries.md b/sumo/validation-queries.md index 25968a2..0f38531 100644 --- a/sumo/validation-queries.md +++ b/sumo/validation-queries.md @@ -187,8 +187,8 @@ metric=cache_usage | max by cache_type Server runtime: ```text -metric=restarts | sum by temporal_service_type -metric=num_goroutines | avg by temporal_service_type +metric=restarts | sum by service_name +metric=num_goroutines | avg by service_name ``` A practical smoke gate should verify: