From 2fc63aea22047aabe264e612d6f4f9e092779249 Mon Sep 17 00:00:00 2001 From: Venkat Date: Sat, 5 Sep 2026 14:58:45 +0000 Subject: [PATCH] feat: add promtool, logcli and tempo-cli for querying the observability stack toolbox covered argocd and bao but nothing for metrics, logs, traces or alerts, so anything observability-shaped meant opening a browser. Add the three native CLIs, each pointed at Grafana's datasource proxy so there is one edge host and one credential for everything. All three accept arbitrary headers, so each wrapper simply sends two: Authorization: Bearer read by oauth2-proxy at the edge X-JWT-Assertion: read by Grafana's [auth.jwt] Two are needed because the edge consumes Authorization for itself and Traefik's forwardauth deletes it before Grafana sees it. X-JWT-Assertion is in no authResponseHeaders list and passes through untouched, which is why none of this needs a loopback proxy the way bao does, or a bearer-preserving middleware the way argocd does. alerts come free: promtool reaches /api/v1/alerts and /api/v1/rules on the same datasource proxy, so alert state and rule listings need no extra tool. grafana-ds resolves datasource UIDs at run time, since Grafana generates them per cluster. The tempo-cli wrapper hides three sharp edges: headers are Key=Value not Key: Value; `search` takes a bare host plus --path-prefix while `trace-id` takes a full URL; and --use-grpc is refused outright, because headers would then travel as gRPC metadata and never reach the edge. Requires Grafana [auth.jwt] (GlueOps/k8s-monitoring-helm#353). Without it these three get a 302 to the login page while argocd and bao keep working, so an older cluster degrades rather than breaking - no version gate needed. README documents the accepted risk: Grafana's datasource proxy forwards non-GET methods, so these credentials can write to Loki/Thanos/Tempo, and those writes are durable in object storage. Agents are told explicitly to treat everything as read-only. Verified against nonprod.jupiter.onglueops.rocks from the built image: all three query successfully, grafana-ds resolves all three UIDs, and the --use-grpc guard fires. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01TrpnuCduw2KivngVRQmnG7 --- Dockerfile | 36 +++++++++++++++++++++++ README.md | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++ bin/grafana-ds | 24 ++++++++++++++++ bin/logcli | 15 ++++++++++ bin/promtool | 24 ++++++++++++++++ bin/tempo-cli | 43 ++++++++++++++++++++++++++++ 6 files changed, 219 insertions(+) create mode 100755 bin/grafana-ds create mode 100755 bin/logcli create mode 100755 bin/promtool create mode 100755 bin/tempo-cli diff --git a/Dockerfile b/Dockerfile index 63d8446..8412445 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,9 @@ FROM debian:12-slim ARG ARGOCD_VERSION=v3.3.12 ARG OPENBAO_VERSION=2.4.4 +ARG PROMETHEUS_VERSION=3.14.0 +ARG LOKI_VERSION=3.7.7 +ARG TEMPO_VERSION=3.0.3 # Supplied automatically by BuildKit for the platform being built. Deliberately # left without a default: a default would silently produce an arm64 image full of @@ -37,6 +40,37 @@ RUN set -eux; \ rm -f /tmp/bao.tar.gz; \ /usr/local/bin/bao version >/dev/null +# promtool (metrics, alert state and rule listings) - queries Thanos through Grafana's +# datasource proxy. Only promtool is kept; the tarball also ships the server binaries. +RUN set -eux; \ + curl -fsSL -o /tmp/prom.tar.gz \ + "https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/prometheus-${PROMETHEUS_VERSION}.linux-${TARGETARCH}.tar.gz"; \ + tar -xzf /tmp/prom.tar.gz --strip-components=1 -C /usr/local/bin --wildcards '*/promtool'; \ + mv /usr/local/bin/promtool /usr/local/bin/promtool.real; \ + chmod +x /usr/local/bin/promtool.real; \ + rm -f /tmp/prom.tar.gz; \ + /usr/local/bin/promtool.real --version >/dev/null + +# logcli (logs). Ships as a zip of a single arch-suffixed binary. +RUN set -eux; \ + curl -fsSL -o /tmp/logcli.zip \ + "https://github.com/grafana/loki/releases/download/v${LOKI_VERSION}/logcli-linux-${TARGETARCH}.zip"; \ + (cd /tmp && jar xf logcli.zip 2>/dev/null || python3 -c "import zipfile;zipfile.ZipFile('/tmp/logcli.zip').extractall('/tmp')"); \ + mv "/tmp/logcli-linux-${TARGETARCH}" /usr/local/bin/logcli.real; \ + chmod +x /usr/local/bin/logcli.real; \ + rm -f /tmp/logcli.zip; \ + /usr/local/bin/logcli.real --version >/dev/null + +# tempo-cli (traces). `query api` is a TraceQL client; the rest of the binary is +# backend tooling we do not use. +RUN set -eux; \ + curl -fsSL -o /tmp/tempo.tar.gz \ + "https://github.com/grafana/tempo/releases/download/v${TEMPO_VERSION}/tempo_${TEMPO_VERSION}_linux_${TARGETARCH}.tar.gz"; \ + tar -xzf /tmp/tempo.tar.gz -C /usr/local/bin tempo-cli; \ + mv /usr/local/bin/tempo-cli /usr/local/bin/tempo-cli.real; \ + chmod +x /usr/local/bin/tempo-cli.real; \ + rm -f /tmp/tempo.tar.gz + COPY lib/ /opt/toolbox/lib/ COPY bin/ /usr/local/bin/ COPY entrypoint.sh /usr/local/bin/entrypoint.sh @@ -46,6 +80,8 @@ COPY bin/toolbox-env /etc/toolbox-env.sh RUN printf '. /etc/toolbox-env.sh\n' > /etc/profile.d/toolbox.sh RUN chmod +x /usr/local/bin/toolbox-token /usr/local/bin/toolbox-proxy \ /usr/local/bin/toolbox-login /usr/local/bin/argocd \ + /usr/local/bin/promtool /usr/local/bin/logcli \ + /usr/local/bin/tempo-cli /usr/local/bin/grafana-ds \ /usr/local/bin/entrypoint.sh # Unprivileged, with the token-cache directory created up front and owned by the diff --git a/README.md b/README.md index b7818f7..c393253 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,12 @@ > If `wait` says `still waiting` (exit 2), run it again. Every later command is > `./toolbox `. `up` handles dockerd, proxies and CAs itself and prints > what it decided — don't investigate the environment first. +> +> **Do not delete or modify data.** These credentials carry write access well +> beyond what the commands suggest — see [Known risks](#known-risks). Treat +> everything here as read-only: query, inspect and report. Do not run `bao kv +> delete`/`destroy`, `argocd app delete`, Grafana `DELETE` calls, or push to any +> datasource. If a task seems to need a destructive action, stop and ask. The platform CLIs in one container, already wired up to authenticate. Developers don't install `argocd`, `bao`, or anything else locally — and they don't need @@ -96,6 +102,10 @@ container variable below is passed through if set. | `toolbox-login --begin` / `--wait` | The same login in two halves: print the URL and return (idempotent), then wait for approval — about 90 s per call, exit 2 means call again. For callers that can't sit on a blocking command. | | `toolbox-login --force` | Re-authenticate, e.g. to switch accounts. | | `toolbox-token` | Print the raw token, for scripting. | +| `promtool …` | Prometheus CLI, pointed at Thanos. Metrics, plus alert state and rule listings. | +| `logcli …` | Loki CLI. Log queries and `--tail`. | +| `tempo-cli query api …` | Tempo CLI. TraceQL search and trace lookup. | +| `grafana-ds ` | Print a datasource UID (`prometheus`/`loki`/`tempo`); used by the wrappers. | ## Configuration @@ -168,6 +178,67 @@ grant. The proxy binds to `127.0.0.1` only — it attaches your credential to whatever it forwards, so it must never be exposed. +**Grafana, Loki, Thanos and Tempo** need no proxy. `promtool`, `logcli` and +`tempo-cli` all accept arbitrary headers, so each wrapper simply sends two: + +| header | read by | +|---|---| +| `Authorization: Bearer ` | oauth2-proxy at the edge | +| `X-JWT-Assertion: ` | Grafana's `[auth.jwt]` | + +Two headers are needed because the edge consumes `Authorization` for itself and +Traefik's forwardauth deletes it before Grafana sees it. `X-JWT-Assertion` is in +no `authResponseHeaders` list, so it passes through untouched — which is why this +needs no bearer-preserving middleware, unlike `argocd`. + +Queries go through Grafana's datasource proxy rather than to Loki/Thanos/Tempo +directly, so there is one edge host and one credential for everything. The +wrappers resolve the datasource UID at run time (`grafana-ds`), since Grafana +generates UIDs per cluster. + +`tempo-cli` differs in three ways the wrapper hides: headers are `Key=Value` not +`Key: Value`; `search` takes a bare host plus `--path-prefix` while `trace-id` +takes a full URL; and `--use-grpc` is refused, because headers would then travel +as gRPC metadata and never reach the edge. + +## Known risks + +Everything below is **known and accepted**, not a bug report. It is written down +because the capabilities are wider than the commands imply, and nothing in the +platform currently constrains them. + +**Your token can write to the observability datasources, not just read them.** +Grafana's datasource proxy is a full pass-through: it forwards `POST`, `PUT` and +`DELETE` to the datasource exactly as it forwards `GET`, and Grafana has no +method-level control over it. So the same credential that runs a PromQL query can +also reach Loki's ingestion endpoint: + + POST /api/datasources/proxy/uid//loki/api/v1/push + +Verified: that request reaches Loki's push handler (it answers with Loki's own +validation error, not a Grafana block). + +**Writes to Loki, Thanos and Tempo are durable.** All three are backed by S3 +object storage, so anything written survives pod restarts and full cluster +rebuilds. Deleting pods does not undo it — the objects have to be removed from the +bucket. Log lines in particular carry no provenance: Loki records what was pushed, +not who pushed it, so an injected line is indistinguishable from an ingested one. +Ingestion limits bound this (`reject_old_samples_max_age: 168h`, rate limits, +`max_streams_per_user`) but do not prevent it. + +**No audit trail.** Grafana OSS does not log datasource-proxy requests per user, +so there is no record of who queried or wrote what. + +**This is not privilege escalation.** Anyone holding a toolbox token already has +ArgoCD and OpenBao access; they are trusted operators. The point is that the +datasource write path is a side effect of enabling CLI authentication, not +something anyone deliberately granted — so treat these tools as read-only by +convention, because nothing enforces it. + +If that convention is ever not enough, the enforcement point is the edge: a +Traefik router rule matching `Method(`GET`)` on `/api/datasources/proxy/` would +make the read-only intent real. It is deliberately not done today. + ## Known issues **An ArgoCD permission error can look like a login problem.** The `argocd` CLI @@ -226,3 +297,9 @@ silently put amd64 binaries in an arm64 image when built natively on a Mac. The platform must have a public Dex client matching `TOOLBOX_CLIENT_ID`, that audience accepted by oauth2-proxy (`oidc_extra_audiences`) and by ArgoCD (`allowedAudiences`), and jwt-type roles in OpenBao bound to it. + +For `promtool`/`logcli`/`tempo-cli`, Grafana additionally needs `[auth.jwt]` +enabled with `header_name = X-JWT-Assertion` and the same audience in +`expect_claims` (GlueOps/k8s-monitoring-helm). Without it those three fail with a +302 to the login page; `argocd` and `bao` are unaffected, so an older cluster +degrades rather than breaking. diff --git a/bin/grafana-ds b/bin/grafana-ds new file mode 100755 index 0000000..0eb82e7 --- /dev/null +++ b/bin/grafana-ds @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Resolve a Grafana datasource UID by type, e.g. `grafana-ds loki`. +# +# The proxy path embeds the datasource UID, and UIDs are per-cluster (Grafana +# generates them unless the chart pins one), so the wrappers look them up at run +# time rather than hard-coding. Cached for the life of the container. +set -euo pipefail +type="${1:?usage: grafana-ds }" +cache="${TMPDIR:-/tmp}/toolbox-ds-${type}" +if [ -s "$cache" ]; then cat "$cache"; exit 0; fi +TOKEN="$(toolbox-token --no-login)" +uid="$(curl -fsSk \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "X-JWT-Assertion: ${TOKEN}" \ + "https://grafana.${TOOLBOX_CAPTAIN_DOMAIN}/api/datasources" \ + | python3 -c " +import json,sys +t='${type}' +d=[x for x in json.load(sys.stdin) if x['type']==t] +if not d: + sys.stderr.write(f'toolbox: no {t} datasource on this cluster\n'); sys.exit(1) +print(d[0]['uid']) +")" +printf '%s' "$uid" | tee "$cache" diff --git a/bin/logcli b/bin/logcli new file mode 100755 index 0000000..57a5b16 --- /dev/null +++ b/bin/logcli @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# logcli, pointed at Loki through Grafana's datasource proxy. +# +# --bearer-token carries the edge credential (logcli puts it in Authorization); +# --header carries the same token for Grafana's [auth.jwt]. +set -euo pipefail +if ! TOKEN="$(toolbox-token --no-login)"; then + echo "toolbox: not authenticated. Run 'toolbox-login' first." >&2; exit 1 +fi +UID_LOKI="$(grafana-ds loki)" +exec /usr/local/bin/logcli.real \ + --addr="https://grafana.${TOOLBOX_CAPTAIN_DOMAIN}/api/datasources/proxy/uid/${UID_LOKI}" \ + --bearer-token="${TOKEN}" \ + --header="X-JWT-Assertion: ${TOKEN}" \ + "$@" diff --git a/bin/promtool b/bin/promtool new file mode 100755 index 0000000..4518699 --- /dev/null +++ b/bin/promtool @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# promtool, pointed at Thanos through Grafana's datasource proxy. +# +# Two headers, because each side reads only its own: Authorization is consumed by +# oauth2-proxy at the edge, X-JWT-Assertion by Grafana's [auth.jwt]. Sending only +# one gets a 302 to the login page. +# +# `query`/`debug` subcommands take a argument; everything else (check, +# push, test) is local and passed straight through. +set -euo pipefail +case "${1:-}" in + query|debug) + if ! TOKEN="$(toolbox-token --no-login)"; then + echo "toolbox: not authenticated. Run 'toolbox-login' first." >&2; exit 1 + fi + sub="$1"; shift + exec /usr/local/bin/promtool.real "$sub" "$@" \ + --header "Authorization: Bearer ${TOKEN}" \ + --header "X-JWT-Assertion: ${TOKEN}" + ;; + *) + exec /usr/local/bin/promtool.real "$@" + ;; +esac diff --git a/bin/tempo-cli b/bin/tempo-cli new file mode 100755 index 0000000..fd305ec --- /dev/null +++ b/bin/tempo-cli @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# tempo-cli, pointed at Tempo through Grafana's datasource proxy. +# +# Three things differ from promtool/logcli and are handled here so callers do not +# have to know them: +# +# 1. Headers use `Key=Value`, not `Key: Value`. Parsed with strings.Cut on the +# first '=', so base64 padding in a JWT is safe. +# 2. `search`/`search-tags`/`search-tag-values`/`metrics` take a bare host and +# build the URL from --path-prefix; `trace-id` has no --path-prefix and takes +# a full URL instead. Both forms are constructed below. +# 3. --use-grpc must NOT be used: headers then travel as gRPC metadata and never +# reach oauth2-proxy, so the edge rejects the request. +set -euo pipefail +if ! TOKEN="$(toolbox-token --no-login)"; then + echo "toolbox: not authenticated. Run 'toolbox-login' first." >&2; exit 1 +fi +for a in "$@"; do + if [ "$a" = "--use-grpc" ]; then + echo "toolbox: --use-grpc cannot work through the edge (headers become gRPC metadata)." >&2 + exit 1 + fi +done +HOST="grafana.${TOOLBOX_CAPTAIN_DOMAIN}" +UID_TEMPO="$(grafana-ds tempo)" +PREFIX="/api/datasources/proxy/uid/${UID_TEMPO}" +HDRS=(--header "Authorization=Bearer ${TOKEN}" --header "X-JWT-Assertion=${TOKEN}") + +# `query api ...` is the only subtree that talks to a server. +if [ "${1:-}" = "query" ] && [ "${2:-}" = "api" ]; then + sub="${3:-}"; shift 3 || true + case "$sub" in + trace-id) + exec /usr/local/bin/tempo-cli.real query api trace-id "${HDRS[@]}" \ + "https://${HOST}${PREFIX}" "$@" ;; + "") + exec /usr/local/bin/tempo-cli.real query api --help ;; + *) + exec /usr/local/bin/tempo-cli.real query api "$sub" --secure \ + --path-prefix "${PREFIX}" "${HDRS[@]}" "${HOST}" "$@" ;; + esac +fi +exec /usr/local/bin/tempo-cli.real "$@"