diff --git a/Dockerfile b/Dockerfile index 63d8446..8412445 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,9 @@ FROM debian:12-slim ARG ARGOCD_VERSION=v3.3.12 ARG OPENBAO_VERSION=2.4.4 +ARG PROMETHEUS_VERSION=3.14.0 +ARG LOKI_VERSION=3.7.7 +ARG TEMPO_VERSION=3.0.3 # Supplied automatically by BuildKit for the platform being built. Deliberately # left without a default: a default would silently produce an arm64 image full of @@ -37,6 +40,37 @@ RUN set -eux; \ rm -f /tmp/bao.tar.gz; \ /usr/local/bin/bao version >/dev/null +# promtool (metrics, alert state and rule listings) - queries Thanos through Grafana's +# datasource proxy. Only promtool is kept; the tarball also ships the server binaries. +RUN set -eux; \ + curl -fsSL -o /tmp/prom.tar.gz \ + "https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/prometheus-${PROMETHEUS_VERSION}.linux-${TARGETARCH}.tar.gz"; \ + tar -xzf /tmp/prom.tar.gz --strip-components=1 -C /usr/local/bin --wildcards '*/promtool'; \ + mv /usr/local/bin/promtool /usr/local/bin/promtool.real; \ + chmod +x /usr/local/bin/promtool.real; \ + rm -f /tmp/prom.tar.gz; \ + /usr/local/bin/promtool.real --version >/dev/null + +# logcli (logs). Ships as a zip of a single arch-suffixed binary. +RUN set -eux; \ + curl -fsSL -o /tmp/logcli.zip \ + "https://github.com/grafana/loki/releases/download/v${LOKI_VERSION}/logcli-linux-${TARGETARCH}.zip"; \ + (cd /tmp && jar xf logcli.zip 2>/dev/null || python3 -c "import zipfile;zipfile.ZipFile('/tmp/logcli.zip').extractall('/tmp')"); \ + mv "/tmp/logcli-linux-${TARGETARCH}" /usr/local/bin/logcli.real; \ + chmod +x /usr/local/bin/logcli.real; \ + rm -f /tmp/logcli.zip; \ + /usr/local/bin/logcli.real --version >/dev/null + +# tempo-cli (traces). `query api` is a TraceQL client; the rest of the binary is +# backend tooling we do not use. +RUN set -eux; \ + curl -fsSL -o /tmp/tempo.tar.gz \ + "https://github.com/grafana/tempo/releases/download/v${TEMPO_VERSION}/tempo_${TEMPO_VERSION}_linux_${TARGETARCH}.tar.gz"; \ + tar -xzf /tmp/tempo.tar.gz -C /usr/local/bin tempo-cli; \ + mv /usr/local/bin/tempo-cli /usr/local/bin/tempo-cli.real; \ + chmod +x /usr/local/bin/tempo-cli.real; \ + rm -f /tmp/tempo.tar.gz + COPY lib/ /opt/toolbox/lib/ COPY bin/ /usr/local/bin/ COPY entrypoint.sh /usr/local/bin/entrypoint.sh @@ -46,6 +80,8 @@ COPY bin/toolbox-env /etc/toolbox-env.sh RUN printf '. /etc/toolbox-env.sh\n' > /etc/profile.d/toolbox.sh RUN chmod +x /usr/local/bin/toolbox-token /usr/local/bin/toolbox-proxy \ /usr/local/bin/toolbox-login /usr/local/bin/argocd \ + /usr/local/bin/promtool /usr/local/bin/logcli \ + /usr/local/bin/tempo-cli /usr/local/bin/grafana-ds \ /usr/local/bin/entrypoint.sh # Unprivileged, with the token-cache directory created up front and owned by the diff --git a/README.md b/README.md index b7818f7..c393253 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,12 @@ > If `wait` says `still waiting` (exit 2), run it again. Every later command is > `./toolbox `. `up` handles dockerd, proxies and CAs itself and prints > what it decided — don't investigate the environment first. +> +> **Do not delete or modify data.** These credentials carry write access well +> beyond what the commands suggest — see [Known risks](#known-risks). Treat +> everything here as read-only: query, inspect and report. Do not run `bao kv +> delete`/`destroy`, `argocd app delete`, Grafana `DELETE` calls, or push to any +> datasource. If a task seems to need a destructive action, stop and ask. The platform CLIs in one container, already wired up to authenticate. Developers don't install `argocd`, `bao`, or anything else locally — and they don't need @@ -96,6 +102,10 @@ container variable below is passed through if set. | `toolbox-login --begin` / `--wait` | The same login in two halves: print the URL and return (idempotent), then wait for approval — about 90 s per call, exit 2 means call again. For callers that can't sit on a blocking command. | | `toolbox-login --force` | Re-authenticate, e.g. to switch accounts. | | `toolbox-token` | Print the raw token, for scripting. | +| `promtool …` | Prometheus CLI, pointed at Thanos. Metrics, plus alert state and rule listings. | +| `logcli …` | Loki CLI. Log queries and `--tail`. | +| `tempo-cli query api …` | Tempo CLI. TraceQL search and trace lookup. | +| `grafana-ds ` | Print a datasource UID (`prometheus`/`loki`/`tempo`); used by the wrappers. | ## Configuration @@ -168,6 +178,67 @@ grant. The proxy binds to `127.0.0.1` only — it attaches your credential to whatever it forwards, so it must never be exposed. +**Grafana, Loki, Thanos and Tempo** need no proxy. `promtool`, `logcli` and +`tempo-cli` all accept arbitrary headers, so each wrapper simply sends two: + +| header | read by | +|---|---| +| `Authorization: Bearer ` | oauth2-proxy at the edge | +| `X-JWT-Assertion: ` | Grafana's `[auth.jwt]` | + +Two headers are needed because the edge consumes `Authorization` for itself and +Traefik's forwardauth deletes it before Grafana sees it. `X-JWT-Assertion` is in +no `authResponseHeaders` list, so it passes through untouched — which is why this +needs no bearer-preserving middleware, unlike `argocd`. + +Queries go through Grafana's datasource proxy rather than to Loki/Thanos/Tempo +directly, so there is one edge host and one credential for everything. The +wrappers resolve the datasource UID at run time (`grafana-ds`), since Grafana +generates UIDs per cluster. + +`tempo-cli` differs in three ways the wrapper hides: headers are `Key=Value` not +`Key: Value`; `search` takes a bare host plus `--path-prefix` while `trace-id` +takes a full URL; and `--use-grpc` is refused, because headers would then travel +as gRPC metadata and never reach the edge. + +## Known risks + +Everything below is **known and accepted**, not a bug report. It is written down +because the capabilities are wider than the commands imply, and nothing in the +platform currently constrains them. + +**Your token can write to the observability datasources, not just read them.** +Grafana's datasource proxy is a full pass-through: it forwards `POST`, `PUT` and +`DELETE` to the datasource exactly as it forwards `GET`, and Grafana has no +method-level control over it. So the same credential that runs a PromQL query can +also reach Loki's ingestion endpoint: + + POST /api/datasources/proxy/uid//loki/api/v1/push + +Verified: that request reaches Loki's push handler (it answers with Loki's own +validation error, not a Grafana block). + +**Writes to Loki, Thanos and Tempo are durable.** All three are backed by S3 +object storage, so anything written survives pod restarts and full cluster +rebuilds. Deleting pods does not undo it — the objects have to be removed from the +bucket. Log lines in particular carry no provenance: Loki records what was pushed, +not who pushed it, so an injected line is indistinguishable from an ingested one. +Ingestion limits bound this (`reject_old_samples_max_age: 168h`, rate limits, +`max_streams_per_user`) but do not prevent it. + +**No audit trail.** Grafana OSS does not log datasource-proxy requests per user, +so there is no record of who queried or wrote what. + +**This is not privilege escalation.** Anyone holding a toolbox token already has +ArgoCD and OpenBao access; they are trusted operators. The point is that the +datasource write path is a side effect of enabling CLI authentication, not +something anyone deliberately granted — so treat these tools as read-only by +convention, because nothing enforces it. + +If that convention is ever not enough, the enforcement point is the edge: a +Traefik router rule matching `Method(`GET`)` on `/api/datasources/proxy/` would +make the read-only intent real. It is deliberately not done today. + ## Known issues **An ArgoCD permission error can look like a login problem.** The `argocd` CLI @@ -226,3 +297,9 @@ silently put amd64 binaries in an arm64 image when built natively on a Mac. The platform must have a public Dex client matching `TOOLBOX_CLIENT_ID`, that audience accepted by oauth2-proxy (`oidc_extra_audiences`) and by ArgoCD (`allowedAudiences`), and jwt-type roles in OpenBao bound to it. + +For `promtool`/`logcli`/`tempo-cli`, Grafana additionally needs `[auth.jwt]` +enabled with `header_name = X-JWT-Assertion` and the same audience in +`expect_claims` (GlueOps/k8s-monitoring-helm). Without it those three fail with a +302 to the login page; `argocd` and `bao` are unaffected, so an older cluster +degrades rather than breaking. diff --git a/bin/grafana-ds b/bin/grafana-ds new file mode 100755 index 0000000..0eb82e7 --- /dev/null +++ b/bin/grafana-ds @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Resolve a Grafana datasource UID by type, e.g. `grafana-ds loki`. +# +# The proxy path embeds the datasource UID, and UIDs are per-cluster (Grafana +# generates them unless the chart pins one), so the wrappers look them up at run +# time rather than hard-coding. Cached for the life of the container. +set -euo pipefail +type="${1:?usage: grafana-ds }" +cache="${TMPDIR:-/tmp}/toolbox-ds-${type}" +if [ -s "$cache" ]; then cat "$cache"; exit 0; fi +TOKEN="$(toolbox-token --no-login)" +uid="$(curl -fsSk \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "X-JWT-Assertion: ${TOKEN}" \ + "https://grafana.${TOOLBOX_CAPTAIN_DOMAIN}/api/datasources" \ + | python3 -c " +import json,sys +t='${type}' +d=[x for x in json.load(sys.stdin) if x['type']==t] +if not d: + sys.stderr.write(f'toolbox: no {t} datasource on this cluster\n'); sys.exit(1) +print(d[0]['uid']) +")" +printf '%s' "$uid" | tee "$cache" diff --git a/bin/logcli b/bin/logcli new file mode 100755 index 0000000..57a5b16 --- /dev/null +++ b/bin/logcli @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# logcli, pointed at Loki through Grafana's datasource proxy. +# +# --bearer-token carries the edge credential (logcli puts it in Authorization); +# --header carries the same token for Grafana's [auth.jwt]. +set -euo pipefail +if ! TOKEN="$(toolbox-token --no-login)"; then + echo "toolbox: not authenticated. Run 'toolbox-login' first." >&2; exit 1 +fi +UID_LOKI="$(grafana-ds loki)" +exec /usr/local/bin/logcli.real \ + --addr="https://grafana.${TOOLBOX_CAPTAIN_DOMAIN}/api/datasources/proxy/uid/${UID_LOKI}" \ + --bearer-token="${TOKEN}" \ + --header="X-JWT-Assertion: ${TOKEN}" \ + "$@" diff --git a/bin/promtool b/bin/promtool new file mode 100755 index 0000000..4518699 --- /dev/null +++ b/bin/promtool @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# promtool, pointed at Thanos through Grafana's datasource proxy. +# +# Two headers, because each side reads only its own: Authorization is consumed by +# oauth2-proxy at the edge, X-JWT-Assertion by Grafana's [auth.jwt]. Sending only +# one gets a 302 to the login page. +# +# `query`/`debug` subcommands take a argument; everything else (check, +# push, test) is local and passed straight through. +set -euo pipefail +case "${1:-}" in + query|debug) + if ! TOKEN="$(toolbox-token --no-login)"; then + echo "toolbox: not authenticated. Run 'toolbox-login' first." >&2; exit 1 + fi + sub="$1"; shift + exec /usr/local/bin/promtool.real "$sub" "$@" \ + --header "Authorization: Bearer ${TOKEN}" \ + --header "X-JWT-Assertion: ${TOKEN}" + ;; + *) + exec /usr/local/bin/promtool.real "$@" + ;; +esac diff --git a/bin/tempo-cli b/bin/tempo-cli new file mode 100755 index 0000000..fd305ec --- /dev/null +++ b/bin/tempo-cli @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# tempo-cli, pointed at Tempo through Grafana's datasource proxy. +# +# Three things differ from promtool/logcli and are handled here so callers do not +# have to know them: +# +# 1. Headers use `Key=Value`, not `Key: Value`. Parsed with strings.Cut on the +# first '=', so base64 padding in a JWT is safe. +# 2. `search`/`search-tags`/`search-tag-values`/`metrics` take a bare host and +# build the URL from --path-prefix; `trace-id` has no --path-prefix and takes +# a full URL instead. Both forms are constructed below. +# 3. --use-grpc must NOT be used: headers then travel as gRPC metadata and never +# reach oauth2-proxy, so the edge rejects the request. +set -euo pipefail +if ! TOKEN="$(toolbox-token --no-login)"; then + echo "toolbox: not authenticated. Run 'toolbox-login' first." >&2; exit 1 +fi +for a in "$@"; do + if [ "$a" = "--use-grpc" ]; then + echo "toolbox: --use-grpc cannot work through the edge (headers become gRPC metadata)." >&2 + exit 1 + fi +done +HOST="grafana.${TOOLBOX_CAPTAIN_DOMAIN}" +UID_TEMPO="$(grafana-ds tempo)" +PREFIX="/api/datasources/proxy/uid/${UID_TEMPO}" +HDRS=(--header "Authorization=Bearer ${TOKEN}" --header "X-JWT-Assertion=${TOKEN}") + +# `query api ...` is the only subtree that talks to a server. +if [ "${1:-}" = "query" ] && [ "${2:-}" = "api" ]; then + sub="${3:-}"; shift 3 || true + case "$sub" in + trace-id) + exec /usr/local/bin/tempo-cli.real query api trace-id "${HDRS[@]}" \ + "https://${HOST}${PREFIX}" "$@" ;; + "") + exec /usr/local/bin/tempo-cli.real query api --help ;; + *) + exec /usr/local/bin/tempo-cli.real query api "$sub" --secure \ + --path-prefix "${PREFIX}" "${HDRS[@]}" "${HOST}" "$@" ;; + esac +fi +exec /usr/local/bin/tempo-cli.real "$@"