feat: add promtool, logcli and tempo-cli for querying the observability stack

Dockerfile

@@ -4,6 +4,9 @@ FROM debian:12-slim
 
 ARG ARGOCD_VERSION=v3.3.12
 ARG OPENBAO_VERSION=2.4.4
+ARG PROMETHEUS_VERSION=3.14.0
+ARG LOKI_VERSION=3.7.7
+ARG TEMPO_VERSION=3.0.3
 
 # Supplied automatically by BuildKit for the platform being built. Deliberately
 # left without a default: a default would silently produce an arm64 image full of
@@ -37,6 +40,37 @@ RUN set -eux; \
     rm -f /tmp/bao.tar.gz; \
     /usr/local/bin/bao version >/dev/null
 
+# promtool (metrics, alert state and rule listings) - queries Thanos through Grafana's
+# datasource proxy. Only promtool is kept; the tarball also ships the server binaries.
+RUN set -eux; \
+    curl -fsSL -o /tmp/prom.tar.gz \
+      "https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/prometheus-${PROMETHEUS_VERSION}.linux-${TARGETARCH}.tar.gz"; \
+    tar -xzf /tmp/prom.tar.gz --strip-components=1 -C /usr/local/bin --wildcards '*/promtool'; \
+    mv /usr/local/bin/promtool /usr/local/bin/promtool.real; \
+    chmod +x /usr/local/bin/promtool.real; \
+    rm -f /tmp/prom.tar.gz; \
+    /usr/local/bin/promtool.real --version >/dev/null
+
+# logcli (logs). Ships as a zip of a single arch-suffixed binary.
+RUN set -eux; \
+    curl -fsSL -o /tmp/logcli.zip \
+      "https://github.com/grafana/loki/releases/download/v${LOKI_VERSION}/logcli-linux-${TARGETARCH}.zip"; \
+    (cd /tmp && jar xf logcli.zip 2>/dev/null || python3 -c "import zipfile;zipfile.ZipFile('/tmp/logcli.zip').extractall('/tmp')"); \
+    mv "/tmp/logcli-linux-${TARGETARCH}" /usr/local/bin/logcli.real; \
+    chmod +x /usr/local/bin/logcli.real; \
+    rm -f /tmp/logcli.zip; \
+    /usr/local/bin/logcli.real --version >/dev/null
+
+# tempo-cli (traces). `query api` is a TraceQL client; the rest of the binary is
+# backend tooling we do not use.
+RUN set -eux; \
+    curl -fsSL -o /tmp/tempo.tar.gz \
+      "https://github.com/grafana/tempo/releases/download/v${TEMPO_VERSION}/tempo_${TEMPO_VERSION}_linux_${TARGETARCH}.tar.gz"; \
+    tar -xzf /tmp/tempo.tar.gz -C /usr/local/bin tempo-cli; \
+    mv /usr/local/bin/tempo-cli /usr/local/bin/tempo-cli.real; \
+    chmod +x /usr/local/bin/tempo-cli.real; \
+    rm -f /tmp/tempo.tar.gz
+
 COPY lib/ /opt/toolbox/lib/
 COPY bin/ /usr/local/bin/
 COPY entrypoint.sh /usr/local/bin/entrypoint.sh
@@ -46,6 +80,8 @@ COPY bin/toolbox-env /etc/toolbox-env.sh
 RUN printf '. /etc/toolbox-env.sh\n' > /etc/profile.d/toolbox.sh
 RUN chmod +x /usr/local/bin/toolbox-token /usr/local/bin/toolbox-proxy \
              /usr/local/bin/toolbox-login /usr/local/bin/argocd \
+             /usr/local/bin/promtool /usr/local/bin/logcli \
+             /usr/local/bin/tempo-cli /usr/local/bin/grafana-ds \
              /usr/local/bin/entrypoint.sh
 
 # Unprivileged, with the token-cache directory created up front and owned by the

README.md

@@ -12,6 +12,12 @@
 > If `wait` says `still waiting` (exit 2), run it again. Every later command is
 > `./toolbox <command>`. `up` handles dockerd, proxies and CAs itself and prints
 > what it decided — don't investigate the environment first.
+>
+> **Do not delete or modify data.** These credentials carry write access well
+> beyond what the commands suggest — see [Known risks](#known-risks). Treat
+> everything here as read-only: query, inspect and report. Do not run `bao kv
+> delete`/`destroy`, `argocd app delete`, Grafana `DELETE` calls, or push to any
+> datasource. If a task seems to need a destructive action, stop and ask.
 
 The platform CLIs in one container, already wired up to authenticate. Developers
 don't install `argocd`, `bao`, or anything else locally — and they don't need
@@ -96,6 +102,10 @@ container variable below is passed through if set.
 | `toolbox-login --begin` / `--wait` | The same login in two halves: print the URL and return (idempotent), then wait for approval — about 90 s per call, exit 2 means call again. For callers that can't sit on a blocking command. |
 | `toolbox-login --force` | Re-authenticate, e.g. to switch accounts. |
 | `toolbox-token` | Print the raw token, for scripting. |
+| `promtool …` | Prometheus CLI, pointed at Thanos. Metrics, plus alert state and rule listings. |
+| `logcli …` | Loki CLI. Log queries and `--tail`. |
+| `tempo-cli query api …` | Tempo CLI. TraceQL search and trace lookup. |
+| `grafana-ds <type>` | Print a datasource UID (`prometheus`/`loki`/`tempo`); used by the wrappers. |
 
 
 ## Configuration
@@ -168,6 +178,67 @@ grant.
 The proxy binds to `127.0.0.1` only — it attaches your credential to whatever it
 forwards, so it must never be exposed.
 
+**Grafana, Loki, Thanos and Tempo** need no proxy. `promtool`, `logcli` and
+`tempo-cli` all accept arbitrary headers, so each wrapper simply sends two:
+
+| header | read by |
+|---|---|
+| `Authorization: Bearer <jwt>` | oauth2-proxy at the edge |
+| `X-JWT-Assertion: <jwt>` | Grafana's `[auth.jwt]` |
+
+Two headers are needed because the edge consumes `Authorization` for itself and
+Traefik's forwardauth deletes it before Grafana sees it. `X-JWT-Assertion` is in
+no `authResponseHeaders` list, so it passes through untouched — which is why this
+needs no bearer-preserving middleware, unlike `argocd`.
+
+Queries go through Grafana's datasource proxy rather than to Loki/Thanos/Tempo
+directly, so there is one edge host and one credential for everything. The
+wrappers resolve the datasource UID at run time (`grafana-ds`), since Grafana
+generates UIDs per cluster.
+
+`tempo-cli` differs in three ways the wrapper hides: headers are `Key=Value` not
+`Key: Value`; `search` takes a bare host plus `--path-prefix` while `trace-id`
+takes a full URL; and `--use-grpc` is refused, because headers would then travel
+as gRPC metadata and never reach the edge.
+
+## Known risks
+
+Everything below is **known and accepted**, not a bug report. It is written down
+because the capabilities are wider than the commands imply, and nothing in the
+platform currently constrains them.
+
+**Your token can write to the observability datasources, not just read them.**
+Grafana's datasource proxy is a full pass-through: it forwards `POST`, `PUT` and
+`DELETE` to the datasource exactly as it forwards `GET`, and Grafana has no
+method-level control over it. So the same credential that runs a PromQL query can
+also reach Loki's ingestion endpoint:
+
+    POST /api/datasources/proxy/uid/<loki>/loki/api/v1/push
+
+Verified: that request reaches Loki's push handler (it answers with Loki's own
+validation error, not a Grafana block).
+
+**Writes to Loki, Thanos and Tempo are durable.** All three are backed by S3
+object storage, so anything written survives pod restarts and full cluster
+rebuilds. Deleting pods does not undo it — the objects have to be removed from the
+bucket. Log lines in particular carry no provenance: Loki records what was pushed,
+not who pushed it, so an injected line is indistinguishable from an ingested one.
+Ingestion limits bound this (`reject_old_samples_max_age: 168h`, rate limits,
+`max_streams_per_user`) but do not prevent it.
+
+**No audit trail.** Grafana OSS does not log datasource-proxy requests per user,
+so there is no record of who queried or wrote what.
+
+**This is not privilege escalation.** Anyone holding a toolbox token already has
+ArgoCD and OpenBao access; they are trusted operators. The point is that the
+datasource write path is a side effect of enabling CLI authentication, not
+something anyone deliberately granted — so treat these tools as read-only by
+convention, because nothing enforces it.
+
+If that convention is ever not enough, the enforcement point is the edge: a
+Traefik router rule matching `Method(`GET`)` on `/api/datasources/proxy/` would
+make the read-only intent real. It is deliberately not done today.
+
 ## Known issues
 
 **An ArgoCD permission error can look like a login problem.** The `argocd` CLI
@@ -226,3 +297,9 @@ silently put amd64 binaries in an arm64 image when built natively on a Mac.
 The platform must have a public Dex client matching `TOOLBOX_CLIENT_ID`, that
 audience accepted by oauth2-proxy (`oidc_extra_audiences`) and by ArgoCD
 (`allowedAudiences`), and jwt-type roles in OpenBao bound to it.
+
+For `promtool`/`logcli`/`tempo-cli`, Grafana additionally needs `[auth.jwt]`
+enabled with `header_name = X-JWT-Assertion` and the same audience in
+`expect_claims` (GlueOps/k8s-monitoring-helm). Without it those three fail with a
+302 to the login page; `argocd` and `bao` are unaffected, so an older cluster
+degrades rather than breaking.

bin/grafana-ds

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Resolve a Grafana datasource UID by type, e.g. `grafana-ds loki`.
+#
+# The proxy path embeds the datasource UID, and UIDs are per-cluster (Grafana
+# generates them unless the chart pins one), so the wrappers look them up at run
+# time rather than hard-coding. Cached for the life of the container.
+set -euo pipefail
+type="${1:?usage: grafana-ds <prometheus|loki|tempo>}"
+cache="${TMPDIR:-/tmp}/toolbox-ds-${type}"
+if [ -s "$cache" ]; then cat "$cache"; exit 0; fi
+TOKEN="$(toolbox-token --no-login)"
+uid="$(curl -fsSk \
+  -H "Authorization: Bearer ${TOKEN}" \
+  -H "X-JWT-Assertion: ${TOKEN}" \
+  "https://grafana.${TOOLBOX_CAPTAIN_DOMAIN}/api/datasources" \
+  | python3 -c "
+import json,sys
+t='${type}'
+d=[x for x in json.load(sys.stdin) if x['type']==t]
+if not d:
+    sys.stderr.write(f'toolbox: no {t} datasource on this cluster\n'); sys.exit(1)
+print(d[0]['uid'])
+")"
+printf '%s' "$uid" | tee "$cache"

bin/logcli

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+# logcli, pointed at Loki through Grafana's datasource proxy.
+#
+# --bearer-token carries the edge credential (logcli puts it in Authorization);
+# --header carries the same token for Grafana's [auth.jwt].
+set -euo pipefail
+if ! TOKEN="$(toolbox-token --no-login)"; then
+    echo "toolbox: not authenticated. Run 'toolbox-login' first." >&2; exit 1
+fi
+UID_LOKI="$(grafana-ds loki)"
+exec /usr/local/bin/logcli.real \
+    --addr="https://grafana.${TOOLBOX_CAPTAIN_DOMAIN}/api/datasources/proxy/uid/${UID_LOKI}" \
+    --bearer-token="${TOKEN}" \
+    --header="X-JWT-Assertion: ${TOKEN}" \
+    "$@"

bin/promtool

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# promtool, pointed at Thanos through Grafana's datasource proxy.
+#
+# Two headers, because each side reads only its own: Authorization is consumed by
+# oauth2-proxy at the edge, X-JWT-Assertion by Grafana's [auth.jwt]. Sending only
+# one gets a 302 to the login page.
+#
+# `query`/`debug` subcommands take a <server> argument; everything else (check,
+# push, test) is local and passed straight through.
+set -euo pipefail
+case "${1:-}" in
+  query|debug)
+    if ! TOKEN="$(toolbox-token --no-login)"; then
+        echo "toolbox: not authenticated. Run 'toolbox-login' first." >&2; exit 1
+    fi
+    sub="$1"; shift
+    exec /usr/local/bin/promtool.real "$sub" "$@" \
+        --header "Authorization: Bearer ${TOKEN}" \
+        --header "X-JWT-Assertion: ${TOKEN}"
+    ;;
+  *)
+    exec /usr/local/bin/promtool.real "$@"
+    ;;
+esac

bin/tempo-cli

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# tempo-cli, pointed at Tempo through Grafana's datasource proxy.
+#
+# Three things differ from promtool/logcli and are handled here so callers do not
+# have to know them:
+#
+#   1. Headers use `Key=Value`, not `Key: Value`. Parsed with strings.Cut on the
+#      first '=', so base64 padding in a JWT is safe.
+#   2. `search`/`search-tags`/`search-tag-values`/`metrics` take a bare host and
+#      build the URL from --path-prefix; `trace-id` has no --path-prefix and takes
+#      a full URL instead. Both forms are constructed below.
+#   3. --use-grpc must NOT be used: headers then travel as gRPC metadata and never
+#      reach oauth2-proxy, so the edge rejects the request.
+set -euo pipefail
+if ! TOKEN="$(toolbox-token --no-login)"; then
+    echo "toolbox: not authenticated. Run 'toolbox-login' first." >&2; exit 1
+fi
+for a in "$@"; do
+    if [ "$a" = "--use-grpc" ]; then
+        echo "toolbox: --use-grpc cannot work through the edge (headers become gRPC metadata)." >&2
+        exit 1
+    fi
+done
+HOST="grafana.${TOOLBOX_CAPTAIN_DOMAIN}"
+UID_TEMPO="$(grafana-ds tempo)"
+PREFIX="/api/datasources/proxy/uid/${UID_TEMPO}"
+HDRS=(--header "Authorization=Bearer ${TOKEN}" --header "X-JWT-Assertion=${TOKEN}")
+
+# `query api <sub> ...` is the only subtree that talks to a server.
+if [ "${1:-}" = "query" ] && [ "${2:-}" = "api" ]; then
+    sub="${3:-}"; shift 3 || true
+    case "$sub" in
+      trace-id)
+        exec /usr/local/bin/tempo-cli.real query api trace-id "${HDRS[@]}" \
+             "https://${HOST}${PREFIX}" "$@" ;;
+      "")
+        exec /usr/local/bin/tempo-cli.real query api --help ;;
+      *)
+        exec /usr/local/bin/tempo-cli.real query api "$sub" --secure \
+             --path-prefix "${PREFIX}" "${HDRS[@]}" "${HOST}" "$@" ;;
+    esac
+fi
+exec /usr/local/bin/tempo-cli.real "$@"