From a7aecc8b20d217ebc4d9318ba850d6565c8d0980 Mon Sep 17 00:00:00 2001 From: Jodson Graves Date: Mon, 13 Jul 2026 13:16:58 -0400 Subject: [PATCH] ops(spire): durable x509pop node attestation (TODO 36) join_token is single-use: an SVID expiry + agent restart crash-loops on "token already used", dropping the orchestrator into SPIFFE-degraded mode. Recurred 2026-07-09 and twice 2026-07-13, each needing manual token rotation + workload-entry re-registration. x509pop derives the agent SPIFFE ID from a leaf-cert fingerprint: stable across restarts, no consumable token, so the crash-loop cannot occur. - deploy/spire/server.conf: add the x509pop NodeAttestor (ca_bundle_path); join_token kept as a break-glass fallback attestor. - deploy/spire/agent.conf: switch the agent to x509pop (leaf cert + key). - docker-compose.yml: mount the CA bundle into spire-server and the leaf cert/key into spire-agent (from ${SPIRE_X509POP_DIR:-./deploy/spire/x509pop}); drop the -joinToken command args. - .gitignore: deploy/spire/x509pop/ (host-local trust material, like .env). - deploy/spire/x509pop-migration.md: cert-gen + apply + re-register + rollback runbook. Both configs pass `spire-{server,agent} validate`; the generated CA/agent chain verifies (openssl verify OK); docker compose config parses. The live cutover is a deliberate host-side step per the runbook (certs are not committed); rollback restores join_token. Blast radius of a bad apply is bounded to degraded mode. Signed-off-by: Jodson Graves --- .gitignore | 1 + deploy/spire/agent.conf | 9 ++- deploy/spire/server.conf | 10 ++++ deploy/spire/x509pop-migration.md | 93 +++++++++++++++++++++++++++++++ docker-compose.yml | 9 ++- 5 files changed, 115 insertions(+), 7 deletions(-) create mode 100644 deploy/spire/x509pop-migration.md diff --git a/.gitignore b/.gitignore index fea532d..adeacf8 100644 --- a/.gitignore +++ b/.gitignore @@ -88,3 +88,4 @@ installer/windows/*.wixpdb installer/windows/*.bmp installer/windows/soholink-agent.exe installer/windows/spire-agent.exe +deploy/spire/x509pop/ diff --git a/deploy/spire/agent.conf b/deploy/spire/agent.conf index 09cb58a..7e6279c 100644 --- a/deploy/spire/agent.conf +++ b/deploy/spire/agent.conf @@ -9,8 +9,13 @@ agent { } plugins { - NodeAttestor "join_token" { - plugin_data {} + # Durable cert-based attestation (TODO 36): stable, cert-fingerprint-derived + # SPIFFE ID, no consumable join token. Survives SVID expiry + restart. + NodeAttestor "x509pop" { + plugin_data { + certificate_path = "/etc/spire/agent.crt" + private_key_path = "/etc/spire/agent.key" + } } WorkloadAttestor "unix" { diff --git a/deploy/spire/server.conf b/deploy/spire/server.conf index d55bdde..c7ec123 100644 --- a/deploy/spire/server.conf +++ b/deploy/spire/server.conf @@ -22,6 +22,16 @@ plugins { } } + # Durable cert-based node attestation (TODO 36). The agent's SPIFFE ID is + # derived from its leaf-cert fingerprint and is STABLE across restarts, so + # there is no one-time token to exhaust (the join_token failure mode where + # an SVID expiry + restart crash-loops on "token already used"). join_token + # kept for transition/fallback; the agent now uses x509pop. + NodeAttestor "x509pop" { + plugin_data { + ca_bundle_path = "/etc/spire/x509pop-ca.crt" + } + } NodeAttestor "join_token" { plugin_data {} } diff --git a/deploy/spire/x509pop-migration.md b/deploy/spire/x509pop-migration.md new file mode 100644 index 0000000..f6f10e3 --- /dev/null +++ b/deploy/spire/x509pop-migration.md @@ -0,0 +1,93 @@ +# SPIRE join_token → x509pop migration (TODO 36) + +**Why.** `join_token` is single-use: it is consumed at first attestation and the +SVID is cached. If the SVID expires while the agent is down (a reboot, a +multi-hour outage) the agent restarts, tries to re-attest with the already-used +token, and crash-loops on *"join token does not exist or has already been +used"* — dragging the orchestrator into SPIFFE-degraded mode. This recurred +2026-07-09 and twice on 2026-07-13, each needing a manual token-rotation + +workload-entry re-registration. + +**Fix.** `x509pop` derives the agent's SPIFFE ID from a leaf-certificate +fingerprint. The ID is **stable across restarts** and there is **no consumable +token** — the crash-loop cannot happen. The config + compose in this repo are +already switched to x509pop (server keeps `join_token` as a fallback attestor); +this runbook is the one-time host-side apply. + +Certs are host-local trust material (like `.env`), never committed: +`deploy/spire/x509pop/` is gitignored. `SPIRE_X509POP_DIR` overrides the path. + +## 1 — Generate the CA + agent cert (once, on the host) + +From the repo root, in Git Bash. `MSYS_NO_PATHCONV=1` stops MSYS mangling the +openssl `-subj`. + +```sh +mkdir -p deploy/spire/x509pop && cd deploy/spire/x509pop +export MSYS_NO_PATHCONV=1 +openssl req -x509 -newkey rsa:2048 -nodes -keyout ca.key -out ca.crt -days 3650 \ + -subj "/O=NTARI/CN=SoHoLINK SPIRE x509pop CA" +openssl req -newkey rsa:2048 -nodes -keyout agent.key -out agent.csr \ + -subj "/O=NTARI/CN=soholink-spire-agent-host" +openssl x509 -req -in agent.csr -CA ca.crt -CAkey ca.key -CAcreateserial \ + -out agent.crt -days 1825 +openssl verify -CAfile ca.crt agent.crt # must print: agent.crt: OK +cd - +``` + +Keep `ca.key` safe (it can mint new agent certs). `agent.key` is the agent's +identity — 0600, never leaves the host. + +## 2 — Apply (recreate the SPIRE pair, then re-anchor the orchestrator) + +The agent's SPIFFE ID changes (join_token/ → x509pop/), so +the orchestrator workload entry must be re-registered under the new parent. + +```sh +cd "" +docker compose up -d --force-recreate spire-server spire-agent +# wait for the agent to attest, then read its new stable ID: +MSYS_NO_PATHCONV=1 docker compose exec spire-server \ + /opt/spire/bin/spire-server agent list \ + -socketPath /run/spire-server/private/api.sock +# -> note the SPIFFE ID: spiffe://soholink.org/spire/agent/x509pop/ + +# register the orchestrator entry under the new parent (uid:0 selector): +MSYS_NO_PATHCONV=1 docker compose exec spire-server \ + /opt/spire/bin/spire-server entry create \ + -socketPath /run/spire-server/private/api.sock \ + -parentID "spiffe://soholink.org/spire/agent/x509pop/" \ + -spiffeID "spiffe://soholink.org/orchestrator" \ + -selector unix:uid:0 -ttl 3600 + +docker compose up -d --force-recreate orchestrator +``` + +## 3 — Verify (including the thing join_token failed at) + +```sh +curl -s https://api.soholink.org/health # {"identity":"ready","status":"ok"} +# the actual fix: restart the agent and confirm it re-attests with NO token: +docker compose restart spire-agent +docker compose logs --since 1m spire-agent | grep -i attest # succeeds, no "token" error +curl -s https://api.soholink.org/health # still ready +``` + +`SPIRE_AGENT_JOIN_TOKEN` in `.env` is now unused (the server keeps the +`join_token` attestor only as a break-glass fallback). Old join_token workload +entries can be pruned with `spire-server entry delete -entryID `. + +## Rollback (to the previous join_token model) + +```sh +git checkout -- deploy/spire/server.conf deploy/spire/agent.conf docker-compose.yml +# generate a fresh token, set SPIRE_AGENT_JOIN_TOKEN in .env, then: +docker compose up -d --force-recreate spire-server spire-agent +bash deploy/register-entries.sh +docker compose up -d --force-recreate orchestrator +``` + +Blast radius of a bad apply is bounded: a failed attestation only puts the +orchestrator in degraded mode (public site stays up; SPIFFE-protected node +routes 503) — recover by rollback. Do the apply as a deliberate step, not +inside another change. diff --git a/docker-compose.yml b/docker-compose.yml index 37ac219..788885e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,6 +40,7 @@ services: command: ["-config", "/etc/spire/server.conf"] volumes: - ./deploy/spire/server.conf:/etc/spire/server.conf:ro + - ${SPIRE_X509POP_DIR:-./deploy/spire/x509pop}/ca.crt:/etc/spire/x509pop-ca.crt:ro - spire_data:/opt/spire/data/server - spire_socket:/run/spire-server/private ports: @@ -56,13 +57,11 @@ services: image: ghcr.io/spiffe/spire-agent:1.9.6 restart: unless-stopped pid: "host" - command: - - "-config" - - "/etc/spire/agent.conf" - - "-joinToken" - - "${SPIRE_AGENT_JOIN_TOKEN}" + command: ["-config", "/etc/spire/agent.conf"] volumes: - ./deploy/spire/agent.conf:/etc/spire/agent.conf:ro + - ${SPIRE_X509POP_DIR:-./deploy/spire/x509pop}/agent.crt:/etc/spire/agent.crt:ro + - ${SPIRE_X509POP_DIR:-./deploy/spire/x509pop}/agent.key:/etc/spire/agent.key:ro - spire_agent_socket:/run/spire/sockets - spire_agent_data:/opt/spire/data/agent depends_on: