From e907b72a5d0b16c8668327d12f4e1866e2763b27 Mon Sep 17 00:00:00 2001 From: Bill Minckler Date: Wed, 5 Aug 2026 20:05:09 +0000 Subject: [PATCH] feat(node-auth): self-signed bearer JWTs for Scout, DPU-agent and fmds (#355) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nodes authenticate to the API with short-lived ES256 JWTs signed by the private key of their existing mTLS client certificate, carrying the cert chain in the token's x5c header. The API verifies that chain against the same root CA its TLS listener already trusts and maps the leaf's SPIFFE SAN through the existing SpiffeContext, so the machine principal and RBAC are unchanged. No new key material, no server-side signing key, no issuance or refresh RPCs. On a DPU only the dpu-agent holds the machine key. It serves AgentLocal/GetNodeToken over a unix socket, so co-located services get tokens rather than the key: token-mode fmds pods mount that socket plus a trust-anchor directory the agent publishes, and never reference the credentials volume. The socket's directory must be dedicated to it — created 0700, or refused if it holds anything else — since the directory is what closes the window between bind and the socket's own chmod. fmds token mode follows [node_auth] enabled, so with node-auth off the chart renders as before. Configuration is [node_auth]: enabled (accept bearer JWTs, requires a TLS listener), mtls_enabled (machine client-cert authn, disableable once the fleet presents tokens), audience and max_token_ttl_sec. Both switches off is rejected at startup. The audience must agree between the API and each node, and is validated on the node whether it arrives by flag or by config file. Bearer tokens never travel in the clear. The API refuses to accept them on a non-TLS listener and keeps its previous TLS acceptor if a rebuild fails, rather than falling back to plaintext while the authenticator stays armed. Clients enforce server-certificate validation whenever a token provider is attached, and refuse a non-HTTPS endpoint outright. Rotation is covered on both sides: the validator reloads its trust anchors on the listener's client-CA refresh, and the minter checks its key against the certified public key before signing, so neither CA rotation nor certificate renewal can lock a node out. The machine key is written 0600. Design doc: docs/design/machine-identity/node-auth-jwt.md Co-Authored-By: Claude Fable 5 Co-Authored-By: Claude Opus 5 (1M context) --- Cargo.lock | 6 + .../nico-dpu-agent/templates/daemonset.yaml | 6 + .../tests/node_auth_audience_test.yaml | 46 ++ bluefield/charts/nico-dpu-agent/values.yaml | 9 + .../charts/nico-fmds/templates/daemonset.yaml | 55 +- .../nico-fmds/tests/node_tokens_test.yaml | 156 ++++++ bluefield/charts/nico-fmds/values.yaml | 12 + crates/agent/Cargo.toml | 1 + crates/agent/example_agent_config.toml | 3 + crates/agent/src/command_line.rs | 21 + crates/agent/src/lib.rs | 88 +++- crates/agent/src/local_api.rs | 338 ++++++++++++ crates/agent/src/tests/common/mod.rs | 1 + crates/api-core/src/api.rs | 4 + crates/api-core/src/cfg/README.md | 14 + crates/api-core/src/cfg/file.rs | 116 +++++ crates/api-core/src/dpf_services.rs | 44 +- crates/api-core/src/lib.rs | 1 + crates/api-core/src/listener.rs | 120 ++++- crates/api-core/src/node_auth.rs | 480 ++++++++++++++++++ crates/api-core/src/setup.rs | 32 ++ crates/api-core/src/test_support/builder.rs | 1 + .../src/test_support/default_config.rs | 1 + crates/authn/Cargo.toml | 1 + crates/authn/src/middleware.rs | 292 ++++++++++- crates/fmds/src/cfg.rs | 7 + crates/fmds/src/main.rs | 47 +- crates/host-support/src/agent_config.rs | 79 +++ crates/host-support/src/registration.rs | 108 +++- .../test/min_agent_config/output.toml | 2 + crates/rpc/Cargo.toml | 4 + crates/rpc/build.rs | 1 + crates/rpc/proto/agent_local.proto | 50 ++ crates/rpc/src/forge_tls_client.rs | 189 +++++++ crates/rpc/src/lib.rs | 4 +- crates/rpc/src/node_jwt.rs | 479 +++++++++++++++++ crates/rpc/src/node_token_socket.rs | 281 ++++++++++ crates/rpc/src/protos/mod.rs | 6 + crates/scout/src/cfg/command_line.rs | 18 + crates/scout/src/client.rs | 5 +- .../api/config-files/nico-api-config.toml | 18 + docs/design/machine-identity/node-auth-jwt.md | 439 ++++++++++++++++ .../nico-api/files/carbide-api-config.toml | 18 + .../proto/core/gen/v1/agent_local_nico.pb.go | 180 +++++++ .../core/gen/v1/agent_local_nico_grpc.pb.go | 156 ++++++ .../proto/core/src/v1/agent_local_nico.proto | 38 ++ 46 files changed, 3927 insertions(+), 50 deletions(-) create mode 100644 bluefield/charts/nico-dpu-agent/tests/node_auth_audience_test.yaml create mode 100644 bluefield/charts/nico-fmds/tests/node_tokens_test.yaml create mode 100644 crates/agent/src/local_api.rs create mode 100644 crates/api-core/src/node_auth.rs create mode 100644 crates/rpc/proto/agent_local.proto create mode 100644 crates/rpc/src/node_jwt.rs create mode 100644 crates/rpc/src/node_token_socket.rs create mode 100644 docs/design/machine-identity/node-auth-jwt.md create mode 100644 rest-api/proto/core/gen/v1/agent_local_nico.pb.go create mode 100644 rest-api/proto/core/gen/v1/agent_local_nico_grpc.pb.go create mode 100644 rest-api/proto/core/src/v1/agent_local_nico.proto diff --git a/Cargo.lock b/Cargo.lock index c640054f67..204a5a1283 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1099,6 +1099,7 @@ dependencies = [ "prost", "prost-types", "rand 0.10.1", + "rcgen", "regex", "reqwest 0.13.4", "resolv-conf", @@ -1550,6 +1551,7 @@ dependencies = [ "rcgen", "serde", "thiserror 2.0.18", + "tokio", "tonic", "tower", "tracing", @@ -2913,13 +2915,16 @@ dependencies = [ "hyper-util", "ipnetwork", "itertools 0.14.0", + "jsonwebtoken", "log", "mac_address", "nonempty", "once_cell", + "p256 0.14.0", "prettytable-rs", "prost", "prost-types", + "rcgen", "regex", "resolv-conf", "rustls", @@ -2929,6 +2934,7 @@ dependencies = [ "serde_yaml", "sha2 0.11.0", "sqlx", + "tempfile", "thiserror 2.0.18", "tokio", "tokio-stream", diff --git a/bluefield/charts/nico-dpu-agent/templates/daemonset.yaml b/bluefield/charts/nico-dpu-agent/templates/daemonset.yaml index 096ec5741f..1c230282df 100644 --- a/bluefield/charts/nico-dpu-agent/templates/daemonset.yaml +++ b/bluefield/charts/nico-dpu-agent/templates/daemonset.yaml @@ -131,6 +131,12 @@ spec: {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" args: + {{- with .Values.nodeAuth.audience }} + # DPF agents run with no config file, so the API's [node_auth] + # audience can only reach them as a flag. Both ends must agree or + # every token this agent mints is rejected. + - {{ printf "--node-auth-audience=%s" . | quote }} + {{- end }} - run - "--hbn-config-mode=nvue-rest" - "--agent-platform-type=containerized" diff --git a/bluefield/charts/nico-dpu-agent/tests/node_auth_audience_test.yaml b/bluefield/charts/nico-dpu-agent/tests/node_auth_audience_test.yaml new file mode 100644 index 0000000000..c725cf0ee7 --- /dev/null +++ b/bluefield/charts/nico-dpu-agent/tests/node_auth_audience_test.yaml @@ -0,0 +1,46 @@ +suite: node-auth audience +templates: + - daemonset.yaml +tests: + # DPF agents run with no config file, so the flag is the only way the API's + # [node_auth] audience reaches them. A mismatch means every token this agent + # mints -- and every token it brokers to co-located services -- is rejected. + - it: should pass the configured audience to the agent + set: + image: + repository: test + tag: test + nodeAuth: + audience: nico-api-eu + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --node-auth-audience=nico-api-eu + + # Empty means "say nothing and let the agent default apply", so the flag must + # not be rendered with an empty value -- that would fail the agent's + # non-blank parser at startup. + - it: should omit the flag entirely when no audience is set + set: + image: + repository: test + tag: test + asserts: + - notContains: + path: spec.template.spec.containers[0].args + content: --node-auth-audience= + - lengthEqual: + path: spec.template.spec.containers[0].args + count: 5 + + - it: should quote an audience containing shell-significant characters + set: + image: + repository: test + tag: test + nodeAuth: + audience: 'nico"api' + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: '--node-auth-audience=nico"api' diff --git a/bluefield/charts/nico-dpu-agent/values.yaml b/bluefield/charts/nico-dpu-agent/values.yaml index c741fee685..1c1db94983 100644 --- a/bluefield/charts/nico-dpu-agent/values.yaml +++ b/bluefield/charts/nico-dpu-agent/values.yaml @@ -67,3 +67,12 @@ dhcp_server: fmds: # Will be updated in dpf_services.rs service_name: "" + +### Node-auth (issue #355). +nodeAuth: + # `aud` the agent stamps on the bearer JWTs it mints, both for its own API + # calls and for the tokens it brokers to co-located services. Must match the + # API's [node_auth] audience, which is where dpf_services.rs templates this + # from — so a DPF deployment always sets it. Empty means "pass no flag" and + # falls back to the agent's own default. + audience: "" diff --git a/bluefield/charts/nico-fmds/templates/daemonset.yaml b/bluefield/charts/nico-fmds/templates/daemonset.yaml index e8de5fadf0..711012d566 100644 --- a/bluefield/charts/nico-fmds/templates/daemonset.yaml +++ b/bluefield/charts/nico-fmds/templates/daemonset.yaml @@ -53,15 +53,32 @@ spec: - | {{- $dir := .Values.certsDir | default "/opt/nico" }} {{- $rootCa := .Values.rootCaFile | default "nico_root.pem" }} + {{- if .Values.useNodeTokens }} + echo "Waiting for root CA in {{ $dir }}/pub ..." + while [ ! -f {{ $dir }}/pub/{{ $rootCa }} ]; do + sleep 5 + done + echo "Root CA found, starting nico-fmds (node tokens from the dpu-agent)." + {{- else }} echo "Waiting for certificates in {{ $dir }} ..." while [ ! -f {{ $dir }}/{{ $rootCa }} ] || [ ! -f {{ $dir }}/machine_cert.pem ] || [ ! -f {{ $dir }}/machine_cert.key ]; do sleep 5 done echo "Certificates found, starting nico-fmds." + {{- end }} volumeMounts: + {{- if .Values.useNodeTokens }} + # Token mode never needs the credentials directory, not even to + # wait: the agent publishes the trust anchor to pub/, which holds + # nothing else. + - name: nico-certs-pub + mountPath: {{ .Values.certsDir | default "/opt/nico" }}/pub + readOnly: true + {{- else }} - name: nico-certs mountPath: {{ .Values.certsDir | default "/opt/nico" }} readOnly: true + {{- end }} containers: - name: nico-fmds securityContext: @@ -71,9 +88,14 @@ spec: {{- $dir := .Values.certsDir | default "/opt/nico" }} {{- $rootCa := .Values.rootCaFile | default "nico_root.pem" }} - "--grpc-address=$(POD_IP):50052" + {{- if .Values.useNodeTokens }} + - "--root-ca={{ $dir }}/pub/{{ $rootCa }}" + - "--node-token-socket={{ $dir }}/run/agent.sock" + {{- else }} - "--root-ca={{ $dir }}/{{ $rootCa }}" - "--client-cert={{ $dir }}/machine_cert.pem" - "--client-key={{ $dir }}/machine_cert.key" + {{- end }} imagePullPolicy: {{ .Values.image.pullPolicy }} {{- with toYaml .Values.serviceDaemonSet.resources }} resources: @@ -106,14 +128,45 @@ spec: fieldRef: fieldPath: metadata.namespace volumeMounts: + {{- $dir := .Values.certsDir | default "/opt/nico" }} + {{- $rootCa := .Values.rootCaFile | default "nico_root.pem" }} + {{- if .Values.useNodeTokens }} + # Token mode: the bearer JWT is the credential, so this container + # gets the trust anchor and nothing else. pub/ holds only the CA, + # so the machine key is absent from the pod — and because this is a + # directory mount rather than a subPath, the agent's atomic + # republish of the CA reaches a running pod. + - name: nico-certs-pub + mountPath: {{ $dir }}/pub + readOnly: true + # The agent's token socket gets its own writable mount — connect(2) + # needs write access to the socket inode, which a read-only mount + # (correctly) denies. + - name: nico-agent-run + mountPath: {{ $dir }}/run + {{- else }} - name: nico-certs - mountPath: {{ .Values.certsDir | default "/opt/nico" }} + mountPath: {{ $dir }} readOnly: true + {{- end }} volumes: + {{- if .Values.useNodeTokens }} + # No credentials-directory volume at all in token mode: nothing in this + # pod is entitled to the machine key. + - name: nico-certs-pub + hostPath: + path: {{ .Values.certsDir | default "/opt/nico" }}/pub + type: DirectoryOrCreate + - name: nico-agent-run + hostPath: + path: {{ .Values.certsDir | default "/opt/nico" }}/run + type: DirectoryOrCreate + {{- else }} - name: nico-certs hostPath: path: {{ .Values.certsDir | default "/opt/nico" }} type: DirectoryOrCreate + {{- end }} {{- with .Values.tolerations }} tolerations: {{- toYaml . | nindent 8 }} diff --git a/bluefield/charts/nico-fmds/tests/node_tokens_test.yaml b/bluefield/charts/nico-fmds/tests/node_tokens_test.yaml new file mode 100644 index 0000000000..3bec41f802 --- /dev/null +++ b/bluefield/charts/nico-fmds/tests/node_tokens_test.yaml @@ -0,0 +1,156 @@ +suite: node-auth token mode +templates: + - daemonset.yaml +tests: + # The point of token mode is that the machine private key is not reachable + # from this pod at all. The key sits beside the root CA in certsDir, and the + # container runs as UID 0, so a read-only mount of that directory would not + # be enough -- the volume must be absent entirely. + - it: should not mount the credentials directory anywhere in token mode + set: + image: + repository: test + tag: test + useNodeTokens: true + asserts: + - notContains: + path: spec.template.spec.volumes + content: + name: nico-certs + hostPath: + path: /opt/forge + type: DirectoryOrCreate + - notContains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: nico-certs + mountPath: /opt/forge + readOnly: true + - notContains: + path: spec.template.spec.initContainers[0].volumeMounts + content: + name: nico-certs + mountPath: /opt/forge + readOnly: true + + # pub/ is mounted as a directory, not as a subPath of the CA file: the agent + # installs the CA by atomic rename, and a subPath mount would pin the old + # inode so a rotation never reached a running pod. + - it: should mount only the published CA directory and the agent socket + set: + image: + repository: test + tag: test + useNodeTokens: true + asserts: + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: nico-certs-pub + mountPath: /opt/forge/pub + readOnly: true + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: nico-agent-run + mountPath: /opt/forge/run + - lengthEqual: + path: spec.template.spec.containers[0].volumeMounts + count: 2 + - contains: + path: spec.template.spec.volumes + content: + name: nico-certs-pub + hostPath: + path: /opt/forge/pub + type: DirectoryOrCreate + + - it: should read the CA from pub/ and authenticate with a token, not a cert + set: + image: + repository: test + tag: test + useNodeTokens: true + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --root-ca=/opt/forge/pub/forge_root.pem + - contains: + path: spec.template.spec.containers[0].args + content: --node-token-socket=/opt/forge/run/agent.sock + - notContains: + path: spec.template.spec.containers[0].args + content: --client-cert=/opt/forge/machine_cert.pem + - notContains: + path: spec.template.spec.containers[0].args + content: --client-key=/opt/forge/machine_cert.key + + # The init container gates the main one; in token mode it has no business + # waiting on the machine cert/key, which may never arrive in this pod. + - it: should wait only for the published CA in token mode + set: + image: + repository: test + tag: test + useNodeTokens: true + asserts: + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: '/opt/forge/pub/forge_root\.pem' + - notMatchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: 'machine_cert\.key' + - contains: + path: spec.template.spec.initContainers[0].volumeMounts + content: + name: nico-certs-pub + mountPath: /opt/forge/pub + readOnly: true + - lengthEqual: + path: spec.template.spec.initContainers[0].volumeMounts + count: 1 + + - it: should follow certsDir into the published CA path + set: + image: + repository: test + tag: test + useNodeTokens: true + certsDir: /srv/creds + rootCaFile: site_root.pem + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --root-ca=/srv/creds/pub/site_root.pem + - contains: + path: spec.template.spec.volumes + content: + name: nico-certs-pub + hostPath: + path: /srv/creds/pub + type: DirectoryOrCreate + + # mTLS remains the default, and must not acquire the token-mode plumbing. + - it: should keep the credentials mount and no pub volume by default + set: + image: + repository: test + tag: test + asserts: + - contains: + path: spec.template.spec.volumes + content: + name: nico-certs + hostPath: + path: /opt/forge + type: DirectoryOrCreate + - notContains: + path: spec.template.spec.volumes + content: + name: nico-certs-pub + hostPath: + path: /opt/forge/pub + type: DirectoryOrCreate + - notContains: + path: spec.template.spec.containers[0].args + content: --node-token-socket=/opt/forge/run/agent.sock diff --git a/bluefield/charts/nico-fmds/values.yaml b/bluefield/charts/nico-fmds/values.yaml index d88ca3dfd1..410af3a336 100644 --- a/bluefield/charts/nico-fmds/values.yaml +++ b/bluefield/charts/nico-fmds/values.yaml @@ -13,6 +13,18 @@ exposedPorts: certsDir: /opt/forge rootCaFile: forge_root.pem +### Node-auth (issue #355). +# When true, fmds authenticates to nico-api with short-lived bearer JWTs +# fetched from the dpu-agent's local API socket (/run/agent.sock) +# instead of the machine mTLS cert/key. The pod then mounts only +# /pub — where the agent publishes the trust anchor and nothing +# else — plus that socket, so the machine private key is absent from it +# rather than merely read-only (the container runs as UID 0). Mounting pub/ +# as a directory also means the agent's atomic republish of a rotated CA +# reaches a running pod. Requires a dpu-agent that serves the local API and +# publishes the CA, and [node_auth] enabled on the API. +useNodeTokens: false + ### Service specific values ### # Prometheus /metrics listen port (must match nico-otelcol scrape target). metricsPort: 8888 diff --git a/crates/agent/Cargo.toml b/crates/agent/Cargo.toml index d1e2d44d6c..891a318f6a 100644 --- a/crates/agent/Cargo.toml +++ b/crates/agent/Cargo.toml @@ -140,6 +140,7 @@ tonic-prost-build = { workspace = true } carbide-instrument = { path = "../instrument", features = ["test-support"] } carbide-test-support = { path = "../test-support" } ctor = { workspace = true } +rcgen = { workspace = true } prost = { workspace = true } rustls-pki-types = { workspace = true } diff --git a/crates/agent/example_agent_config.toml b/crates/agent/example_agent_config.toml index 28fd03b079..c433f9db65 100644 --- a/crates/agent/example_agent_config.toml +++ b/crates/agent/example_agent_config.toml @@ -18,6 +18,9 @@ api-server = "http://localhost:50051" pxe-server = "http://127.0.0.1:8080" root-ca = "/home/amaslennikov/pcarbide/credentials/localhost.pem" +# Unix socket where the agent serves node tokens to co-located services +# (issue #355). Same default containerized (DPF) and on plain DPU OS. +#local-api-socket = "/opt/forge/run/agent.sock" [machine] interface-id = "3048d658-0b13-4a5e-91cc-f47e45a031f9" diff --git a/crates/agent/src/command_line.rs b/crates/agent/src/command_line.rs index 7a633e10bc..80aea661cc 100644 --- a/crates/agent/src/command_line.rs +++ b/crates/agent/src/command_line.rs @@ -25,6 +25,16 @@ use url::Url; use crate::network_monitor::NetworkPingerType; +/// Rejects an empty or whitespace-only audience at parse time. A blank value +/// would otherwise reach the minter and produce tokens the API cannot match, +/// with nothing in the logs pointing at the flag. +fn non_blank_audience(value: &str) -> Result { + if value.trim().is_empty() { + return Err("node-auth audience must not be empty".to_string()); + } + Ok(value.to_string()) +} + #[derive(Parser)] #[clap(name = "forge-dpu-agent")] pub struct Options { @@ -36,6 +46,17 @@ pub struct Options { #[clap(long)] pub config_path: Option, + /// Overrides `[forge-system] node-auth-audience`. DPF deploys the agent + /// with no config file at all, so the audience can only reach it as a + /// flag; the API templates this from its own `[node_auth] audience` so + /// the two ends cannot drift. + #[clap( + long, + value_parser = non_blank_audience, + help = "Audience claim stamped on node-auth bearer JWTs; must match the API's [node_auth] audience" + )] + pub node_auth_audience: Option, + #[clap(subcommand)] pub cmd: Option, } diff --git a/crates/agent/src/lib.rs b/crates/agent/src/lib.rs index f09214c748..2f30ecae47 100644 --- a/crates/agent/src/lib.rs +++ b/crates/agent/src/lib.rs @@ -65,6 +65,7 @@ mod host_machine_id; mod instance_metadata_endpoint; pub mod instrumentation; pub mod lldp; +mod local_api; mod machine_inventory_updater; mod main_loop; mod managed_files; @@ -92,6 +93,15 @@ pub const FMDS_MINIMUM_HBN_VERSION: &str = "1.5.0-doca2.2.0"; pub const NVUE_MINIMUM_HBN_VERSION: &str = "2.0.0-doca2.5.0"; const BOOTSTRAP_CA_OUTPUT_PATH: &str = "/opt/forge/forge_root.pem"; +/// Second copy of the trust anchor, in a directory that holds nothing else. +/// +/// Co-located services need the root CA but must never see the machine private +/// key that lives beside it (issue #355). A container can only be given one or +/// the other: mounting the credentials directory exposes the key, and mounting +/// the CA file alone by `subPath` bind-mounts its inode, so the atomic rename +/// in `install_bootstrap_ca` would never reach a running consumer. A directory +/// containing only the CA gives both properties — no key, and renames resolve. +const BOOTSTRAP_CA_PUBLIC_PATH: &str = "/opt/forge/pub/forge_root.pem"; const MOUNTED_BOOTSTRAP_CA_PATH: &str = "/var/run/secrets/nico-bootstrap-ca/ca.pem"; const MAX_BOOTSTRAP_CA_BYTES: usize = 1024 * 1024; const BOOTSTRAP_CA_DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(30); @@ -272,7 +282,21 @@ fn install_bootstrap_ca(contents: &[u8], output_path: &Path) -> eyre::Result<()> async fn provision_bootstrap_ca(options: &command_line::InitContainerOptions) -> eyre::Result<()> { let contents = acquire_bootstrap_ca(options.bootstrap_ca_source, &options.bootstrap_ca_url).await?; - install_bootstrap_ca(&contents, Path::new(BOOTSTRAP_CA_OUTPUT_PATH)) + install_bootstrap_ca(&contents, Path::new(BOOTSTRAP_CA_OUTPUT_PATH))?; + publish_bootstrap_ca(&contents) +} + +/// Mirrors the trust anchor into [`BOOTSTRAP_CA_PUBLIC_PATH`] for key-less +/// consumers. Same atomic install, so a consumer with the directory mounted +/// picks up a replacement without restarting. +fn publish_bootstrap_ca(contents: &[u8]) -> eyre::Result<()> { + let public_path = Path::new(BOOTSTRAP_CA_PUBLIC_PATH); + let parent = public_path + .parent() + .ok_or_else(|| eyre::eyre!("public CA path has no parent: {}", public_path.display()))?; + std::fs::create_dir_all(parent) + .wrap_err_with(|| format!("failed to create public CA directory {}", parent.display()))?; + install_bootstrap_ca(contents, public_path) } pub async fn start(cmdline: command_line::Options) -> eyre::Result<()> { @@ -281,7 +305,7 @@ pub async fn start(cmdline: command_line::Options) -> eyre::Result<()> { return Ok(()); } - let (agent, path) = match cmdline.config_path { + let (mut agent, path) = match cmdline.config_path { // normal production case None => (AgentConfig::default(), "default".to_string()), // development overrides @@ -293,6 +317,17 @@ pub async fn start(cmdline: command_line::Options) -> eyre::Result<()> { config_path.display().to_string(), ), }; + // Containerized (DPF) agents run without a config file, so the flag is the + // only way the API's configured audience reaches them. + if let Some(node_auth_audience) = cmdline.node_auth_audience.clone() { + agent.forge_system.node_auth_audience = node_auth_audience; + } + // After the override above, so the flag and the file are held to the same + // rule rather than only the flag validating itself at parse time. + agent + .forge_system + .validate() + .map_err(|e| eyre::eyre!("invalid [forge-system] in agent config: {e}"))?; agent .machine_identity .validate() @@ -303,6 +338,36 @@ pub async fn start(cmdline: command_line::Options) -> eyre::Result<()> { tracing::warn!("Pretending local host is a DPU. Dev only."); } + // Republish on every start, not just from the init container: an operator + // replacing the trust anchor out of band would otherwise leave co-located + // services on the previous one until the next initialization. + match std::fs::read(&agent.forge_system.root_ca) { + Ok(contents) => { + if let Err(error) = publish_bootstrap_ca(&contents) { + tracing::warn!( + target: "node_auth", + %error, + "node-auth: could not publish the root CA for co-located services" + ); + } + } + Err(error) => tracing::debug!( + target: "node_auth", + %error, + path = %agent.forge_system.root_ca, + "node-auth: no root CA to publish for co-located services yet" + ), + } + + // Node-auth (#355): the agent is the only process on the DPU that holds + // the machine key. This minter signs bearer JWTs for the agent's own API + // calls AND backs the local API socket that brokers tokens to co-located + // services (fmds, ...). Ignored by the API unless [node_auth] is enabled. + let node_jwt_minter = ::rpc::node_jwt::NodeJwtMinter::with_audience( + agent.forge_system.client_cert.clone(), + agent.forge_system.client_key.clone(), + agent.forge_system.node_auth_audience.clone(), + ); let forge_client_config = Arc::new( ForgeClientConfig::new( agent.forge_system.root_ca.clone(), @@ -311,6 +376,7 @@ pub async fn start(cmdline: command_line::Options) -> eyre::Result<()> { key_path: agent.forge_system.client_key.clone(), }), ) + .with_token_provider(node_jwt_minter.clone()) .use_mgmt_vrf()?, ); @@ -325,6 +391,24 @@ pub async fn start(cmdline: command_line::Options) -> eyre::Result<()> { tracing::warn!("Upgrades disabled. Dev only"); } + // Broker node tokens to co-located services over the local API + // socket. Not fatal on failure, and retried forever: consumers + // fall back to their own credentials (mTLS client cert) until the + // socket comes up, and on non-DPF DPU OS the agent may start + // before its socket directory is usable. + tokio::spawn({ + let minter = node_jwt_minter.clone(); + let socket_path = agent.forge_system.local_api_socket.clone(); + async move { + loop { + if let Err(error) = local_api::serve(minter.clone(), &socket_path).await { + tracing::warn!(target: "node_auth", %error, "node-auth: agent local API server failed; retrying"); + } + tokio::time::sleep(std::time::Duration::from_secs(10)).await; + } + } + }); + let Registration { machine_id, factory_mac_address, diff --git a/crates/agent/src/local_api.rs b/crates/agent/src/local_api.rs new file mode 100644 index 0000000000..80d8a6ea47 --- /dev/null +++ b/crates/agent/src/local_api.rs @@ -0,0 +1,338 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! The dpu-agent's local API: an `AgentLocal` gRPC service on a unix socket, +//! by default in a dedicated `run/` subdirectory of the shared `/opt/forge` +//! credentials directory (issue #355) — so containerized consumers mount the +//! socket read-write while the credentials stay on a read-only mount. +//! +//! This is the consolidation point for agent <-> co-located-service +//! communication on the DPU. First RPC: `GetNodeToken`, which hands +//! co-located NICo services (fmds, …) a short-lived node-auth bearer JWT so +//! they can authenticate to nico-api without mounting the machine's private +//! key — only the agent ever touches the key. Future local needs should add +//! RPCs here rather than new sockets, ports, or file drops. + +use std::sync::Arc; + +use ::rpc::agent_local::agent_local_server::{AgentLocal, AgentLocalServer}; +use ::rpc::agent_local::{GetNodeTokenRequest, GetNodeTokenResponse}; +use ::rpc::node_jwt::NodeJwtMinter; +use eyre::WrapErr; +use tokio_stream::wrappers::UnixListenerStream; +use tonic::{Request, Response, Status}; + +struct AgentLocalService { + minter: Arc, +} + +#[tonic::async_trait] +impl AgentLocal for AgentLocalService { + async fn get_node_token( + &self, + _request: Request, + ) -> Result, Status> { + // Minting reads the cert/key from disk on cache miss — cheap enough to + // do inline, and it means the endpoint starts working the moment the + // machine certificate lands without any coordination. + match self.minter.current_with_expiry() { + Some((token, expires_at)) => { + Ok(Response::new(GetNodeTokenResponse { token, expires_at })) + } + None => Err(Status::unavailable( + "no node token available yet; machine certificate not present or unreadable", + )), + } + } +} + +/// Prepares the socket's parent directory, which must be dedicated to this +/// socket and nothing else. +/// +/// The directory has to be unreachable to other users *before* `bind`: `bind` +/// creates the socket with umask-derived permissions — typically +/// world-connectable — and the 0600 chmod only lands afterwards. The listener +/// is already bound in that window, so a connection from any local user would +/// queue in the backlog and be served a full machine identity the moment +/// accepting starts. An unreachable parent closes the window; the socket's own +/// 0600 is then defence in depth. +/// +/// But `local-api-socket` is operator-configurable, so applying 0700 to +/// whatever the parent happens to be is its own hazard: `/run/agent.sock` +/// would take `/run` to 0700 and lock every non-root service on the box out of +/// its runtime files. Only a directory this function created, or one holding +/// nothing but this socket, is ours to restrict — anything else is rejected +/// with an actionable message rather than silently modified. +fn prepare_socket_dir(dir: &std::path::Path, socket_path: &str) -> eyre::Result<()> { + use std::os::unix::fs::{DirBuilderExt, PermissionsExt}; + + if !dir.exists() { + // Ancestors keep their normal modes; only the socket directory itself + // is created restricted, and created that way from the first instant + // rather than chmod-ed after the fact. + if let Some(ancestor) = dir.parent() { + std::fs::create_dir_all(ancestor) + .wrap_err(format!("creating socket directory {}", ancestor.display()))?; + } + return std::fs::DirBuilder::new() + .mode(0o700) + .create(dir) + .wrap_err(format!("creating socket directory {}", dir.display())); + } + + let socket_name = std::path::Path::new(socket_path).file_name(); + for entry in + std::fs::read_dir(dir).wrap_err(format!("reading socket directory {}", dir.display()))? + { + let entry = entry.wrap_err(format!("reading socket directory {}", dir.display()))?; + let name = entry.file_name(); + if socket_name != Some(name.as_os_str()) { + eyre::bail!( + "socket directory {} is shared with other files (found {}); \ + point local-api-socket at a directory used for nothing else, \ + so restricting it to 0700 cannot lock other services out", + dir.display(), + name.to_string_lossy() + ); + } + } + + std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o700)).wrap_err(format!( + "restricting socket directory permissions on {}", + dir.display() + )) +} + +/// Binds the local API socket and serves until the process exits. The socket +/// is created mode 0600 (root-only): every legitimate consumer on the DPU +/// runs as root, and nothing else on the shared directory should be able to +/// obtain machine credentials. +/// +/// Deployment-agnostic: containerized (DPF) the socket lands on the mounted +/// `/opt/forge` volume; as a plain service on DPU OS (non-DPF) the parent +/// directory is created if the agent starts before anything else touched it. +pub async fn serve(minter: Arc, socket_path: &str) -> eyre::Result<()> { + use std::os::unix::fs::PermissionsExt; + + if let Some(parent) = std::path::Path::new(socket_path).parent() { + prepare_socket_dir(parent, socket_path)?; + } + // Remove a stale socket from a previous run; bind() fails on an existing + // path even when nothing is listening. + match std::fs::remove_file(socket_path) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e).wrap_err(format!("removing stale socket {socket_path}")), + } + + let listener = tokio::net::UnixListener::bind(socket_path) + .wrap_err(format!("binding agent local API socket {socket_path}"))?; + std::fs::set_permissions(socket_path, std::fs::Permissions::from_mode(0o600)) + .wrap_err(format!("restricting socket permissions on {socket_path}"))?; + tracing::info!(target: "node_auth", socket = %socket_path, "node-auth: serving agent local API (node tokens)"); + + tonic::transport::Server::builder() + .add_service(AgentLocalServer::new(AgentLocalService { minter })) + .serve_with_incoming(UnixListenerStream::new(listener)) + .await + .wrap_err("agent local API server exited") +} + +#[cfg(test)] +mod tests { + use ::rpc::node_jwt::NodeTokenProvider; + use ::rpc::node_token_socket::SocketTokenSource; + + use super::*; + + const SPIFFE_URI: &str = "spiffe://forge.local/forge-system/machine/fm100xtest"; + + fn write_cert_and_key(dir: &tempfile::TempDir) -> (String, String) { + let mut params = rcgen::CertificateParams::default(); + params.subject_alt_names = vec![rcgen::SanType::URI( + rcgen::string::Ia5String::try_from(SPIFFE_URI.to_string()).expect("uri"), + )]; + let key = rcgen::KeyPair::generate().expect("key pair"); + let cert = params.self_signed(&key).expect("certificate"); + let cert_path = dir.path().join("cert.pem"); + let key_path = dir.path().join("cert.key"); + std::fs::write(&cert_path, cert.pem()).expect("write cert"); + std::fs::write(&key_path, key.serialize_pem()).expect("write key"); + ( + cert_path.to_string_lossy().into_owned(), + key_path.to_string_lossy().into_owned(), + ) + } + + /// End-to-end broker flow: agent serves tokens minted from the machine + /// cert; a key-less consumer obtains one through `SocketTokenSource`. + #[tokio::test] + async fn keyless_consumer_gets_token_via_socket() { + let dir = tempfile::tempdir().expect("tempdir"); + let (cert_path, key_path) = write_cert_and_key(&dir); + // Its own subdirectory, as in production (`/run/agent.sock`): + // the socket never shares a directory with the credentials. + let socket = dir.path().join("run").join("agent.sock"); + let socket_str = socket.to_string_lossy().into_owned(); + + let minter = NodeJwtMinter::new(cert_path, key_path); + let expected = minter.current().expect("agent side can mint"); + tokio::spawn({ + let socket_str = socket_str.clone(); + async move { serve(minter, &socket_str).await } + }); + + let source = SocketTokenSource::spawn(socket_str); + let mut got = None; + for _ in 0..100 { + if let Some(token) = source.current() { + got = Some(token); + break; + } + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + assert_eq!(got.as_deref(), Some(expected.as_str())); + } + + #[tokio::test] + async fn socket_is_root_only() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().expect("tempdir"); + let (cert_path, key_path) = write_cert_and_key(&dir); + let socket = dir.path().join("run").join("agent.sock"); + let socket_str = socket.to_string_lossy().into_owned(); + + tokio::spawn({ + let socket_str = socket_str.clone(); + async move { serve(NodeJwtMinter::new(cert_path, key_path), &socket_str).await } + }); + for _ in 0..100 { + if socket.exists() { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + let mode = std::fs::metadata(&socket) + .expect("socket metadata") + .permissions() + .mode() + & 0o777; + assert_eq!(mode, 0o600); + } + + /// The socket's own mode is applied only after `bind`, so the directory is + /// what actually keeps other users out during that window. Anyone who got + /// in would be handed a full machine identity. + #[tokio::test] + async fn socket_directory_is_root_only_before_the_socket_exists() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().expect("tempdir"); + let (cert_path, key_path) = write_cert_and_key(&dir); + // A pre-existing, world-traversable directory: the agent must tighten + // it rather than assume a fresh one. + let run_dir = dir.path().join("run"); + std::fs::create_dir(&run_dir).expect("create run dir"); + std::fs::set_permissions(&run_dir, std::fs::Permissions::from_mode(0o755)) + .expect("loosen run dir"); + let socket = run_dir.join("agent.sock"); + let socket_str = socket.to_string_lossy().into_owned(); + + tokio::spawn({ + let socket_str = socket_str.clone(); + async move { serve(NodeJwtMinter::new(cert_path, key_path), &socket_str).await } + }); + for _ in 0..100 { + if socket.exists() { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + + let mode = std::fs::metadata(&run_dir) + .expect("run dir metadata") + .permissions() + .mode() + & 0o777; + assert_eq!(mode, 0o700, "socket directory must exclude other users"); + } + + /// Tightening the parent is only safe when the parent belongs to us. + /// `local-api-socket = /run/agent.sock` would otherwise take `/run` to + /// 0700 and lock every non-root service on the box out of its runtime + /// files, so a directory holding anything else is refused outright. + #[test] + fn shared_socket_directory_is_refused_rather_than_restricted() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().expect("tempdir"); + let shared = dir.path().join("run"); + std::fs::create_dir(&shared).expect("create shared dir"); + std::fs::write(shared.join("someone-elses.pid"), b"1234").expect("write neighbour"); + std::fs::set_permissions(&shared, std::fs::Permissions::from_mode(0o755)) + .expect("loosen shared dir"); + let socket = shared.join("agent.sock"); + + let err = prepare_socket_dir(&shared, &socket.to_string_lossy()) + .expect_err("a shared directory must not be accepted"); + assert!( + err.to_string().contains("someone-elses.pid"), + "the error should name the file that made it shared, got: {err}" + ); + + let mode = std::fs::metadata(&shared) + .expect("shared dir metadata") + .permissions() + .mode() + & 0o777; + assert_eq!( + mode, 0o755, + "a refused directory must be left exactly as it was found" + ); + } + + /// A directory we create ourselves is restricted from its first instant, + /// with no window at a looser mode for anyone to slip through. + #[test] + fn fresh_socket_directory_is_created_root_only() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().expect("tempdir"); + let run_dir = dir.path().join("nested").join("run"); + let socket = run_dir.join("agent.sock"); + + prepare_socket_dir(&run_dir, &socket.to_string_lossy()).expect("prepare fresh dir"); + + let mode = std::fs::metadata(&run_dir) + .expect("run dir metadata") + .permissions() + .mode() + & 0o777; + assert_eq!(mode, 0o700, "a freshly created socket directory is 0700"); + } + + /// The stale socket from a previous run is ours, so it must not be + /// mistaken for a neighbour and block startup. + #[test] + fn a_stale_socket_does_not_make_the_directory_look_shared() { + let dir = tempfile::tempdir().expect("tempdir"); + let run_dir = dir.path().join("run"); + std::fs::create_dir(&run_dir).expect("create run dir"); + let socket = run_dir.join("agent.sock"); + std::fs::write(&socket, b"").expect("stale socket"); + + prepare_socket_dir(&run_dir, &socket.to_string_lossy()) + .expect("a directory holding only our own socket is still dedicated"); + } +} diff --git a/crates/agent/src/tests/common/mod.rs b/crates/agent/src/tests/common/mod.rs index 2fabebac29..4ff31d7786 100644 --- a/crates/agent/src/tests/common/mod.rs +++ b/crates/agent/src/tests/common/mod.rs @@ -113,6 +113,7 @@ pub fn setup_agent_run_env( let opts = crate::Options { version: false, config_path: Some(acf.path().to_path_buf()), + node_auth_audience: None, cmd: Some(crate::AgentCommand::Run(Box::new(crate::RunOptions { enable_metadata_service: test_metadata_service, override_machine_id: None, diff --git a/crates/api-core/src/api.rs b/crates/api-core/src/api.rs index 4e7c3d6e77..79a4402305 100644 --- a/crates/api-core/src/api.rs +++ b/crates/api-core/src/api.rs @@ -90,6 +90,10 @@ pub struct Api { pub(crate) component_manager: Option, pub(crate) bms_client: OnceLock>, pub(crate) secrets_context: Option, + /// Validator for node-auth bearer JWTs (issue #355). `Some` only when + /// `[node_auth] enabled`; installed into the authn middleware by the + /// listener. + pub(crate) node_jwt_validator: Option>, } pub(crate) type ScoutStreamType = diff --git a/crates/api-core/src/cfg/README.md b/crates/api-core/src/cfg/README.md index 8c5bab2c59..3bd696bb9b 100644 --- a/crates/api-core/src/cfg/README.md +++ b/crates/api-core/src/cfg/README.md @@ -123,6 +123,7 @@ applicable. | `dhcp_lease_expiry_handling` | `bool` | `false` | `networking` | Enables IP cleanup when a DHCP lease expires. | | `certificates` | `CertificatesConfig` | *(default)* | `security` | Certificate vending backend, selected independently of the credential store; the default shares the credential Vault (see [CertificatesConfig](#certificatesconfig)). | | `allow_insecure_discovery` | `bool` | `false` | `machines` | Allows machines to submit discovery without enforcing the request comes from the expected IP address. Needed for *Integration tests only*, should otherwise not be used. | +| `node_auth` | `NodeAuthConfig` | *(default)* | `security` | How Scout and the DPU-agent authenticate: bearer JWTs, machine mTLS client certificates, or both during a migration (see [NodeAuthConfig](#nodeauthconfig)). | --- @@ -276,6 +277,19 @@ available for topology-specific flows. | `identity_keyfile_path` | `String` | `""` | Server identity private key. | | `admin_root_cafile_path` | `String` | `""` | Admin root CA for admin client validation. | +### `NodeAuthConfig` + +Node (Scout / DPU-agent) authentication. Bearer tokens are off by default, so +the default is machine mTLS exactly as before. See +`docs/design/machine-identity/node-auth-jwt.md`. + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `enabled` | `bool` | `false` | Accept `Authorization: Bearer` node JWTs. Nodes self-sign these with their existing mTLS client-certificate key and carry the certificate in the token's `x5c` header; the API verifies it against `[tls] root_cafile_path`. Requires a TLS listener -- the API refuses to accept bearer tokens over plaintext. | +| `mtls_enabled` | `bool` | `true` | Accept machine mTLS client certificates as node identity. Turn off only once the fleet presents bearer tokens; startup fails if this and `enabled` are both false. Scoped to machine certificates -- service and admin-CLI certificates are unaffected. | +| `audience` | `String` | `nico-api` | Required `aud` claim. Nodes must stamp the same value (`--node-auth-audience` for Scout and the agent; the API templates it onto DPF-deployed agents), or every token they mint is rejected. | +| `max_token_ttl_sec` | `u32` | `900` | Longest accepted token lifetime, in seconds. Clients mint 300 s tokens; this caps how far a client may push `exp`. Must be greater than zero and at most 86400. | + ### `AuthConfig` | Field | Type | Default | Description | diff --git a/crates/api-core/src/cfg/file.rs b/crates/api-core/src/cfg/file.rs index e5a349b162..adfd5957bb 100644 --- a/crates/api-core/src/cfg/file.rs +++ b/crates/api-core/src/cfg/file.rs @@ -496,6 +496,12 @@ pub struct CarbideConfig { #[serde(default)] pub machine_identity: MachineIdentityConfig, + /// Node-auth: bearer JWTs that Scout / DPU-agent self-sign with their + /// existing mTLS client-certificate key, accepted alongside (or instead + /// of) mTLS. Section `[node_auth]`. + #[serde(default)] + pub node_auth: NodeAuthConfig, + /// Disables role-based access control enforcement. /// Intended for testing and development only. #[serde(default)] @@ -1969,6 +1975,95 @@ impl Default for MachineIdentityConfig { } } +/// Node-auth (Scout / DPU-agent bearer JWT) configuration. +/// Loaded from `[node_auth]` section in config. +/// +/// There is no server-side signing key: nodes self-sign tokens with their +/// existing mTLS client-certificate key and the API validates the embedded +/// `x5c` chain against the client-cert root CA (see +/// `docs/design/machine-identity/node-auth-jwt.md`). +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct NodeAuthConfig { + /// Master switch. When false, no bearer authenticator is installed and + /// nodes keep authenticating via mTLS client certs only. + #[serde(default = "node_auth_default_enabled")] + pub enabled: bool, + /// `aud` claim required on presented tokens. Must match what nodes mint + /// (`rpc::node_jwt::NODE_JWT_AUDIENCE`). + #[serde(default = "node_auth_default_audience")] + pub audience: String, + /// Maximum accepted token lifetime, in seconds. Nodes mint 5-minute + /// tokens; this bounds how far a (compromised) client can stretch `exp`. + #[serde(default = "node_auth_default_max_token_ttl_sec")] + pub max_token_ttl_sec: u32, + /// Whether machine mTLS client certificates are accepted as node identity. + /// On by default. Scoped to MACHINE certs only — service and admin-CLI + /// client certs on the same listener are unaffected. Disable only once the + /// fleet presents bearer tokens. + #[serde(default = "node_auth_default_mtls_enabled")] + pub mtls_enabled: bool, +} + +/// Upper bound on accepted token lifetime. Node tokens are minted locally on +/// demand, so anything beyond a day is almost certainly a misconfiguration. +pub const NODE_AUTH_MAX_TOKEN_TTL_SEC: u32 = 86_400; + +impl NodeAuthConfig { + /// Validates node-auth settings. Call unconditionally at startup: the + /// lockout check applies even when [`enabled`](Self::enabled) is false. + pub fn validate(&self) -> eyre::Result<()> { + if !self.enabled && !self.mtls_enabled { + return Err(eyre::eyre!( + "[node_auth] enabled = false and mtls_enabled = false would leave nodes with no \ + way to authenticate; enable at least one of bearer tokens or machine mTLS" + )); + } + if !self.enabled { + // Remaining checks only constrain token validation. + return Ok(()); + } + if self.audience.trim().is_empty() { + return Err(eyre::eyre!("[node_auth] audience must not be empty")); + } + if self.max_token_ttl_sec == 0 { + return Err(eyre::eyre!( + "[node_auth] max_token_ttl_sec must be greater than zero" + )); + } + if self.max_token_ttl_sec > NODE_AUTH_MAX_TOKEN_TTL_SEC { + return Err(eyre::eyre!( + "[node_auth] max_token_ttl_sec {} exceeds maximum {NODE_AUTH_MAX_TOKEN_TTL_SEC}", + self.max_token_ttl_sec + )); + } + Ok(()) + } +} + +fn node_auth_default_enabled() -> bool { + false +} +fn node_auth_default_audience() -> String { + "nico-api".to_string() +} +fn node_auth_default_max_token_ttl_sec() -> u32 { + 900 +} +fn node_auth_default_mtls_enabled() -> bool { + true +} + +impl Default for NodeAuthConfig { + fn default() -> Self { + Self { + enabled: node_auth_default_enabled(), + audience: node_auth_default_audience(), + max_token_ttl_sec: node_auth_default_max_token_ttl_sec(), + mtls_enabled: node_auth_default_mtls_enabled(), + } + } +} + impl From for model::tenant::IdentityConfigValidationBounds { fn from(mi: MachineIdentityConfig) -> Self { Self { @@ -3610,6 +3705,27 @@ mod tests { use super::*; use crate::test_support::network_segment::FIXTURE_TENANT_ORG_ID; + /// Disabling both bearer tokens and machine mTLS would lock every node out + /// of the API; validation must refuse the combination, and each mechanism + /// alone must pass. + #[test] + fn node_auth_rejects_all_methods_disabled() { + let both_off = NodeAuthConfig { + enabled: false, + mtls_enabled: false, + ..NodeAuthConfig::default() + }; + assert!(both_off.validate().is_err()); + + assert!(NodeAuthConfig::default().validate().is_ok()); + let jwt_only = NodeAuthConfig { + enabled: true, + mtls_enabled: false, + ..NodeAuthConfig::default() + }; + assert!(jwt_only.validate().is_ok()); + } + const TEST_DATA_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/src/cfg/test_data"); fn vpc_config( diff --git a/crates/api-core/src/dpf_services.rs b/crates/api-core/src/dpf_services.rs index e3342b2af6..a5173714a5 100644 --- a/crates/api-core/src/dpf_services.rs +++ b/crates/api-core/src/dpf_services.rs @@ -32,7 +32,7 @@ use carbide_dpf::{ use crate::cfg::file::{ DpfBootstrapCaObjectKind, DpfDpuAgentBootstrapCa, DpfExtraService, - DpfResolvedMandatoryServicesConfig, DpfServiceConfig, + DpfResolvedMandatoryServicesConfig, DpfServiceConfig, NodeAuthConfig, }; /// Default DOCA helm registry (DPUServiceTemplate source.repoURL). @@ -366,6 +366,7 @@ pub fn dts_service(cfg: &DpfServiceConfig) -> ServiceDefinition { fn dpu_agent_helm_values( cfg: &DpfServiceConfig, bootstrap_ca: &DpfDpuAgentBootstrapCa, + node_auth_audience: &str, ) -> serde_json::Value { let mut values = serde_json::json!({ "image": { @@ -376,6 +377,11 @@ fn dpu_agent_helm_values( "nvue_https_address": "nvue", "nvue_credentials_secret_name": "hbn-user-password", "nvue_password_key": "password", + }, + // DPF agents get no config file, so [node_auth] audience has to ride + // in as a flag or they mint tokens the API will reject. + "nodeAuth": { + "audience": node_auth_audience, } }); apply_image_pull_secrets(&mut values, cfg); @@ -417,9 +423,10 @@ fn dpu_agent_helm_values( pub fn dpu_agent_service( cfg: &DpfServiceConfig, bootstrap_ca: &DpfDpuAgentBootstrapCa, + node_auth_audience: &str, ) -> ServiceDefinition { ServiceDefinition { - helm_values: Some(dpu_agent_helm_values(cfg, bootstrap_ca)), + helm_values: Some(dpu_agent_helm_values(cfg, bootstrap_ca, node_auth_audience)), service_daemon_set_annotations: Some(BTreeMap::new()), @@ -479,12 +486,18 @@ pub fn dhcp_server_service(cfg: &DpfServiceConfig) -> ServiceDefinition { } /// Forge FMDS service definition. -pub fn fmds_service(cfg: &DpfServiceConfig) -> ServiceDefinition { +/// +/// `use_node_tokens` follows the API's `[node_auth] enabled` switch: when the +/// API accepts bearer JWTs, fmds is deployed fetching them from the +/// dpu-agent's local API socket instead of mounting the machine cert/key +/// (issue #355). Requires a dpu-agent image that serves the local API. +pub fn fmds_service(cfg: &DpfServiceConfig, use_node_tokens: bool) -> ServiceDefinition { let mut helm_values = serde_json::json!({ "image": { "repository": cfg.docker_repo_url, "tag": cfg.docker_image_tag, - } + }, + "useNodeTokens": use_node_tokens, }); apply_image_pull_secrets(&mut helm_values, cfg); ServiceDefinition { @@ -676,16 +689,22 @@ pub fn doca_xplane_service(cfg: &DpfServiceConfig) -> ServiceDefinition { } /// Build the full list of resolved mandatory DPU services. +/// +/// `node_auth` mirrors the API's `[node_auth]` section: `enabled` switches +/// fmds to bearer tokens from the dpu-agent's local API, and `audience` is +/// templated onto the agent so both ends stamp/expect the same `aud` +/// (issue #355). pub fn mandatory_services( resolved: &DpfResolvedMandatoryServicesConfig, bootstrap_ca: &DpfDpuAgentBootstrapCa, + node_auth: &NodeAuthConfig, ) -> Vec { let mut service_vec = vec![ dts_service(&resolved.base.dts), doca_hbn_service(&resolved.base.doca_hbn), dhcp_server_service(&resolved.base.dhcp_server), - dpu_agent_service(&resolved.base.dpu_agent, bootstrap_ca), - fmds_service(&resolved.base.fmds), + dpu_agent_service(&resolved.base.dpu_agent, bootstrap_ca, &node_auth.audience), + fmds_service(&resolved.base.fmds, node_auth.enabled), otelcol_service(&resolved.base.otel), ]; @@ -720,7 +739,11 @@ mod tests { fn dpu_agent_bootstrap_ca_helm_values_follow_site_policy() { value_scenarios!( run = |policy| { - dpu_agent_helm_values(&default_dpu_agent_service(), &policy) + dpu_agent_helm_values( + &default_dpu_agent_service(), + &policy, + ::rpc::node_jwt::NODE_JWT_AUDIENCE, + ) .get("bootstrapCa") .cloned() }; @@ -859,6 +882,7 @@ mod tests { let dpu_agent = dpu_agent_service( &default_dpu_agent_service(), &DpfDpuAgentBootstrapCa::default(), + ::rpc::node_jwt::NODE_JWT_AUDIENCE, ); assert!( dpu_agent @@ -872,7 +896,11 @@ mod tests { // When a pull secret is configured, the block is emitted with its name. let mut cfg = default_dpu_agent_service(); cfg.docker_image_pull_secret = Some("nico-pull-secret".to_string()); - let agent = dpu_agent_service(&cfg, &DpfDpuAgentBootstrapCa::default()); + let agent = dpu_agent_service( + &cfg, + &DpfDpuAgentBootstrapCa::default(), + ::rpc::node_jwt::NODE_JWT_AUDIENCE, + ); assert_eq!( agent.helm_values.unwrap()["imagePullSecrets"], serde_json::json!([{ "name": "nico-pull-secret" }]) diff --git a/crates/api-core/src/lib.rs b/crates/api-core/src/lib.rs index be5d1d9c29..bebfda4c16 100644 --- a/crates/api-core/src/lib.rs +++ b/crates/api-core/src/lib.rs @@ -68,6 +68,7 @@ mod machine_validation; mod measured_boot; mod mqtt_state_change_hook; mod network_segment; +mod node_auth; mod scout_stream; pub mod secrets; mod setup; diff --git a/crates/api-core/src/listener.rs b/crates/api-core/src/listener.rs index a2024a6550..73e12829da 100644 --- a/crates/api-core/src/listener.rs +++ b/crates/api-core/src/listener.rs @@ -335,8 +335,36 @@ pub async fn start( ), ))?; - let cert_description_layer: CertDescriptionMiddleware = - CertDescriptionMiddleware::new(extra_cli_certs, spiffe_context); + let cert_description_layer: CertDescriptionMiddleware = { + let machine_certs_enabled = api_service.runtime_config.node_auth.mtls_enabled; + if !machine_certs_enabled { + tracing::warn!( + target: "node_auth", + "node-auth: mtls_enabled = false: machine client certificates will NOT be \ + accepted as node identity; nodes must present bearer tokens" + ); + } + let layer = CertDescriptionMiddleware::new(extra_cli_certs, spiffe_context) + .with_machine_certs_enabled(machine_certs_enabled); + // When node-auth is enabled, accept bearer JWTs in addition to mTLS + // client certs (dual-support during the mTLS→JWT migration). Bearer + // tokens must only be accepted over TLS — never plaintext — so guard the + // authenticator on the listener actually being TLS-terminated. + match (&api_service.node_jwt_validator, tls_config.is_some()) { + (Some(node_jwt_validator), true) => { + tracing::info!(target: "node_auth", "node-auth: bearer token authentication enabled"); + layer.with_bearer_authenticator(node_jwt_validator.clone()) + } + (Some(_), false) => { + tracing::warn!( + target: "node_auth", + "node-auth: enabled but listener is not TLS; refusing to accept bearer tokens over plaintext" + ); + layer + } + (None, _) => layer, + } + }; let casbin_layer = if let Some(auth_config) = auth_config { if let Some(casbin_policy_file) = &auth_config.casbin_policy_file { let casbin_authorizer = Arc::new( @@ -406,6 +434,9 @@ pub async fn start( let mut tls_acceptor_created = Instant::now(); let mut initialize_tls_acceptor = true; + // Refreshed alongside the TLS acceptor below; both read the same client-CA + // bundle, so they must not drift apart. + let node_jwt_validator = api_service.node_jwt_validator.clone(); join_set .build_task() @@ -442,18 +473,79 @@ pub async fn start( initialize_tls_acceptor = false; tls_acceptor_created = Instant::now(); - tls_acceptor = tokio::task::Builder::new() - .name("get_tls_acceptor refresh") - .spawn_blocking({ - let tls_config = tls_config.clone(); - move || get_tls_acceptor(&tls_config) - }) - // Safety: spawn_blocking only returns Error if run outside the tokio runtime - .expect("Failed to spawn blocking task") - .await - // Safety: Awaiting a JoinHandle only fails if the task panicked, and we want to - // propagate panics - .expect("task panicked"); + // Node-auth JWTs chain to the same client-CA bundle, so the + // validator's trust anchors have to rotate on the same tick. + // Left stale it would reject tokens issued under the new CA, + // which locks nodes out entirely once mtls_enabled = false. + // + // Reload it BEFORE swapping the acceptor so the two move + // together: if the bundle is caught mid-write, neither is + // replaced and the whole pair stays on the previous anchors + // until the next tick. Swapping the acceptor first would + // leave the listener trusting a CA the token path does not. + let jwt_roots_reloaded = match node_jwt_validator.as_ref() { + None => true, + Some(node_jwt_validator) => { + let validator = node_jwt_validator.clone(); + let refreshed = tokio::task::Builder::new() + .name("node_jwt_validator refresh") + .spawn_blocking(move || validator.refresh_roots()) + .expect("Failed to spawn blocking task") + .await + .expect("task panicked"); + match refreshed { + Ok(()) => true, + Err(error) => { + tracing::warn!( + target: "node_auth", + %error, + "node-auth: could not reload JWT trust anchors; \ + keeping the previous TLS and token trust anchors" + ); + false + } + } + } + }; + + if jwt_roots_reloaded { + let refreshed_acceptor = tokio::task::Builder::new() + .name("get_tls_acceptor refresh") + .spawn_blocking({ + let tls_config = tls_config.clone(); + move || get_tls_acceptor(&tls_config) + }) + // Safety: spawn_blocking only returns Error if run outside the tokio runtime + .expect("Failed to spawn blocking task") + .await + // Safety: Awaiting a JoinHandle only fails if the task panicked, and we want to + // propagate panics + .expect("task panicked"); + + // `get_tls_acceptor` yields `None` for any failure — + // an identity PEM caught mid-write by cert-manager, an + // unreadable key, a CA bundle with nothing parsable in + // it. Assigning that straight through would drop the + // listener onto the plaintext branch below while the + // bearer authenticator, installed once at startup on + // the premise that this listener terminates TLS, keeps + // accepting node JWTs — putting them on the wire in the + // clear. Keep the working acceptor and retry instead: a + // stale-but-valid one beats no TLS, and rotation leaves + // ample overlap to pick the new material up. + match refreshed_acceptor { + Some(acceptor) => tls_acceptor = Some(acceptor), + None => { + // Retry on the next connection rather than + // waiting out another five-minute window. + initialize_tls_acceptor = true; + tracing::error!( + "could not rebuild the TLS acceptor; \ + keeping the previous one and retrying" + ); + } + } + } } let tls_acceptor = tls_acceptor.clone(); diff --git a/crates/api-core/src/node_auth.rs b/crates/api-core/src/node_auth.rs new file mode 100644 index 0000000000..5731cba36b --- /dev/null +++ b/crates/api-core/src/node_auth.rs @@ -0,0 +1,480 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Node-auth: validation of self-signed bearer JWTs from Scout / DPU-agent +//! (issue NVIDIA/infra-controller#355, simple variant). +//! +//! Nodes sign short-lived ES256 JWTs with the private key of their EXISTING +//! mTLS client certificate and carry the certificate chain in the token's +//! `x5c` header. [`NodeJwtValidator`] verifies, in order: +//! +//! 1. the `x5c` chain against the same root CAs the TLS listener trusts for +//! client certificates (chain of trust, validity window, client-auth EKU); +//! 2. the JWT signature against the verified leaf's public key (algorithm +//! pinned to ES256 — the only key type Vault PKI issues to machines); +//! 3. the registered claims: `exp` (with a bounded lifetime), `aud`; +//! 4. the SPIFFE constraints on the leaf and that the token's `sub` matches +//! the leaf's SPIFFE URI SAN — identity always derives from the verified +//! certificate, never from an attacker-controlled claim. +//! +//! The resulting SPIFFE URI is mapped by the authn middleware through the +//! SAME `SpiffeContext` as mTLS client certs, so a JWT and a cert for the +//! same machine yield an identical principal and reuse the existing RBAC +//! unchanged. There is no server-side key material and no issuance path: +//! "public key exchange" is the existing certificate PKI. + +use std::sync::{Arc, RwLock}; + +use carbide_authn::middleware::BearerTokenAuthenticator; +use jsonwebtoken::{Algorithm, DecodingKey, Validation, decode, decode_header}; +use rustls::RootCertStore; +use rustls::server::WebPkiClientVerifier; +use rustls::server::danger::ClientCertVerifier; +use rustls_pki_types::{CertificateDer, UnixTime}; +use serde::Deserialize; +use x509_parser::prelude::{FromDer, X509Certificate}; + +use crate::cfg::file::NodeAuthConfig; + +#[derive(Debug, thiserror::Error)] +pub enum NodeAuthError { + #[error("could not read root CA file {path}: {error}")] + RootCaRead { path: String, error: std::io::Error }, + #[error("root CA file {path} contains no usable trust anchors")] + NoTrustAnchors { path: String }, + #[error("could not build certificate verifier: {0}")] + Verifier(String), +} + +/// Why a presented bearer token was rejected. Only ever logged at debug — +/// rejection simply means the request proceeds without a bearer principal. +#[derive(Debug, thiserror::Error)] +enum RejectReason { + #[error("malformed JWT: {0}")] + Malformed(jsonwebtoken::errors::Error), + #[error("unexpected algorithm {0:?}; only ES256 is accepted")] + Algorithm(Algorithm), + #[error("no x5c certificate chain in the JWT header")] + NoChain, + #[error("x5c chain did not verify against the trusted roots: {0}")] + Chain(rustls::Error), + #[error("leaf certificate is not an EC (P-256) certificate")] + NotEcCertificate, + #[error("signature/claims validation failed: {0}")] + Claims(jsonwebtoken::errors::Error), + #[error("token lifetime exceeds the allowed maximum")] + Lifetime, + #[error("leaf certificate fails SPIFFE validation: {0}")] + Spiffe(String), + #[error("token `sub` does not match the certificate's SPIFFE URI")] + SubjectMismatch, + #[error("system clock is before the UNIX epoch")] + Clock, +} + +/// Registered claims checked on node tokens. `iat` is required so the bounded +/// lifetime check (`exp - iat`) cannot be dodged by omitting it. +#[derive(Debug, Deserialize)] +struct NodeClaims { + sub: String, + iat: u64, + exp: u64, +} + +/// Validates node-auth JWTs against the client-certificate PKI. +pub struct NodeJwtValidator { + /// Kept so the trust anchors can be re-read when the bundle rotates. + root_cafile_path: String, + /// Swapped in place by [`NodeJwtValidator::refresh_roots`]. The validator + /// is shared (the authn layer holds one `Arc` for the process lifetime), + /// so the refresh has to be interior, not a rebuild of the whole struct. + /// Held behind a lock rather than an `ArcSwap` because `arc-swap` cannot + /// store an unsized `dyn ClientCertVerifier`; the guard is released before + /// the (comparatively expensive) chain verification runs. + cert_verifier: RwLock>, + validation: Validation, + max_token_ttl_sec: u64, +} + +impl NodeJwtValidator { + /// Builds a validator trusting the given root CA bundle — the same file + /// the TLS listener uses to verify mTLS client certificates. + pub fn from_root_ca_file( + root_cafile_path: &str, + cfg: &NodeAuthConfig, + ) -> Result { + let cert_verifier = Self::build_verifier(root_cafile_path)?; + + let mut validation = Validation::new(Algorithm::ES256); + validation.set_audience(&[&cfg.audience]); + validation.set_required_spec_claims(&["exp", "sub", "aud", "iat"]); + + Ok(Self { + root_cafile_path: root_cafile_path.to_string(), + cert_verifier: RwLock::new(cert_verifier), + validation, + max_token_ttl_sec: u64::from(cfg.max_token_ttl_sec), + }) + } + + /// Re-reads the root CA bundle from disk and swaps in a verifier built + /// from it. + /// + /// The TLS listener reloads the same file every five minutes to pick up + /// cert-manager rotations; without this the validator would keep its + /// startup snapshot and start rejecting tokens whose `x5c` chains to the + /// new CA — which, with `mtls_enabled = false`, locks nodes out until the + /// API restarts. On error the existing verifier is left in place: a + /// half-written or briefly unreadable bundle should not disarm node auth. + pub fn refresh_roots(&self) -> Result<(), NodeAuthError> { + let cert_verifier = Self::build_verifier(&self.root_cafile_path)?; + *self + .cert_verifier + .write() + .unwrap_or_else(|poisoned| poisoned.into_inner()) = cert_verifier; + Ok(()) + } + + fn build_verifier( + root_cafile_path: &str, + ) -> Result, NodeAuthError> { + let pem = std::fs::read(root_cafile_path).map_err(|error| NodeAuthError::RootCaRead { + path: root_cafile_path.to_string(), + error, + })?; + let mut roots = RootCertStore::empty(); + let certs = rustls_pemfile::certs(&mut std::io::Cursor::new(&pem[..])) + .collect::, _>>() + .map_err(|e| NodeAuthError::Verifier(format!("root CA parse error: {e}")))?; + let (added, _ignored) = roots.add_parsable_certificates(certs); + if added == 0 { + return Err(NodeAuthError::NoTrustAnchors { + path: root_cafile_path.to_string(), + }); + } + + WebPkiClientVerifier::builder_with_provider( + Arc::new(roots), + Arc::new(rustls::crypto::aws_lc_rs::default_provider()), + ) + .allow_unknown_revocation_status() + .build() + .map_err(|e| NodeAuthError::Verifier(e.to_string())) + } + + fn validate(&self, token: &str) -> Result { + let header = decode_header(token).map_err(RejectReason::Malformed)?; + if header.alg != Algorithm::ES256 { + return Err(RejectReason::Algorithm(header.alg)); + } + + // 1. The certificate chain must verify against the trusted roots. + let chain = header + .x5c_der() + .map_err(RejectReason::Malformed)? + .filter(|chain| !chain.is_empty()) + .ok_or(RejectReason::NoChain)?; + let leaf = CertificateDer::from(chain[0].clone()); + let intermediates: Vec = chain[1..] + .iter() + .map(|der| CertificateDer::from(der.clone())) + .collect(); + let cert_verifier = self + .cert_verifier + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone(); + cert_verifier + .verify_client_cert(&leaf, &intermediates, UnixTime::now()) + .map_err(RejectReason::Chain)?; + + // 2. The token must be signed by the verified leaf's key. + let (_, x509) = + X509Certificate::from_der(leaf.as_ref()).map_err(|_| RejectReason::NotEcCertificate)?; + let decoding_key = DecodingKey::from_ec_der(&x509.public_key().subject_public_key.data); + let claims = decode::(token, &decoding_key, &self.validation) + .map_err(RejectReason::Claims)? + .claims; + + // 3. Bounded lifetime: the client controls `exp`, so cap how far in + // the future it may reach. `jsonwebtoken` already rejected expired + // tokens above. + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(|_| RejectReason::Clock)? + .as_secs(); + if claims.exp.saturating_sub(claims.iat) > self.max_token_ttl_sec + || claims.exp > now + self.max_token_ttl_sec + { + return Err(RejectReason::Lifetime); + } + + // 4. Identity comes from the verified certificate, with `sub` only + // cross-checked against it. + let spiffe_id = carbide_authn::validate_x509_certificate(leaf.as_ref()) + .map_err(|e| RejectReason::Spiffe(e.to_string()))?; + let spiffe_uri = spiffe_id.to_string(); + if claims.sub != spiffe_uri { + return Err(RejectReason::SubjectMismatch); + } + Ok(spiffe_uri) + } +} + +impl BearerTokenAuthenticator for NodeJwtValidator { + fn spiffe_id_from_bearer(&self, token: &str) -> Option { + match self.validate(token) { + Ok(spiffe_uri) => Some(spiffe_uri), + Err(reason) => { + tracing::debug!(target: "node_auth", %reason, "node-auth: rejected bearer token"); + None + } + } + } +} + +#[cfg(test)] +mod tests { + use rpc::node_jwt::NodeJwtMinter; + + use super::*; + + const TRUST_DOMAIN: &str = "forge.local"; + const MACHINE_PATH: &str = "/forge-system/machine/fm100xtest"; + + struct TestPki { + ca_pem: String, + cert_pem: String, + key_pem: String, + } + + /// A CA plus a leaf it issued carrying the machine SPIFFE URI SAN — + /// stand-ins for the Vault PKI root and a node's client certificate. + fn test_pki(spiffe_path: &str) -> TestPki { + let mut ca_params = rcgen::CertificateParams::default(); + ca_params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + ca_params + .distinguished_name + .push(rcgen::DnType::CommonName, "test root"); + let ca_key = rcgen::KeyPair::generate().expect("ca key"); + let ca_cert = ca_params.clone().self_signed(&ca_key).expect("ca cert"); + let issuer = rcgen::Issuer::new(ca_params, ca_key); + + let mut leaf_params = rcgen::CertificateParams::default(); + leaf_params.subject_alt_names = vec![rcgen::SanType::URI( + rcgen::string::Ia5String::try_from(format!("spiffe://{TRUST_DOMAIN}{spiffe_path}")) + .expect("uri"), + )]; + leaf_params.use_authority_key_identifier_extension = true; + leaf_params + .extended_key_usages + .push(rcgen::ExtendedKeyUsagePurpose::ClientAuth); + let leaf_key = rcgen::KeyPair::generate().expect("leaf key"); + let leaf_cert = leaf_params + .signed_by(&leaf_key, &issuer) + .expect("leaf cert"); + + TestPki { + ca_pem: ca_cert.pem(), + cert_pem: leaf_cert.pem(), + key_pem: leaf_key.serialize_pem(), + } + } + + fn write_temp(dir: &tempfile::TempDir, name: &str, contents: &str) -> String { + let path = dir.path().join(name); + std::fs::write(&path, contents).expect("write"); + path.to_string_lossy().into_owned() + } + + fn validator_for(dir: &tempfile::TempDir, ca_pem: &str) -> NodeJwtValidator { + let ca_path = write_temp(dir, "ca.pem", ca_pem); + NodeJwtValidator::from_root_ca_file(&ca_path, &NodeAuthConfig::default()) + .expect("validator builds") + } + + fn mint_with(dir: &tempfile::TempDir, pki: &TestPki) -> String { + let minter = NodeJwtMinter::new( + write_temp(dir, "cert.pem", &pki.cert_pem), + write_temp(dir, "key.pem", &pki.key_pem), + ); + minter.current().expect("token minted") + } + + /// `[node_auth] audience` is configurable, so a site that changes it must + /// still work end to end: the minter has to stamp the configured value and + /// the validator has to accept it (and nothing else). + #[test] + fn a_configured_audience_round_trips_and_excludes_the_default() { + let dir = tempfile::tempdir().expect("tempdir"); + let pki = test_pki(MACHINE_PATH); + let ca_path = write_temp(&dir, "ca.pem", &pki.ca_pem); + let cfg = NodeAuthConfig { + audience: "carbide-api-eu".to_string(), + ..NodeAuthConfig::default() + }; + let validator = + NodeJwtValidator::from_root_ca_file(&ca_path, &cfg).expect("validator builds"); + + let cert_path = write_temp(&dir, "cert.pem", &pki.cert_pem); + let key_path = write_temp(&dir, "key.pem", &pki.key_pem); + + let matching = NodeJwtMinter::with_audience( + cert_path.clone(), + key_path.clone(), + "carbide-api-eu".to_string(), + ) + .current() + .expect("token minted"); + assert_eq!( + validator.spiffe_id_from_bearer(&matching).as_deref(), + Some(format!("spiffe://{TRUST_DOMAIN}{MACHINE_PATH}").as_str()), + "a token minted for the configured audience must be accepted" + ); + + let default_aud = NodeJwtMinter::new(cert_path, key_path) + .current() + .expect("token minted"); + assert_eq!( + validator.spiffe_id_from_bearer(&default_aud), + None, + "the default audience must not be accepted once one is configured" + ); + } + + /// A rotated client-CA bundle has to be picked up without a restart: the + /// TLS listener re-reads the same file every five minutes, and tokens + /// chaining to the new CA must start verifying once it does. + #[test] + fn rotated_root_ca_is_honored_after_refresh() { + let dir = tempfile::tempdir().expect("tempdir"); + let old_pki = test_pki(MACHINE_PATH); + let ca_path = write_temp(&dir, "ca.pem", &old_pki.ca_pem); + let validator = NodeJwtValidator::from_root_ca_file(&ca_path, &NodeAuthConfig::default()) + .expect("validator builds"); + + // A second CA, as though cert-manager had rotated the bundle. + let new_pki = test_pki(MACHINE_PATH); + let new_dir = tempfile::tempdir().expect("tempdir"); + let token = mint_with(&new_dir, &new_pki); + assert_eq!( + validator.spiffe_id_from_bearer(&token), + None, + "a token from the not-yet-trusted CA must be rejected" + ); + + std::fs::write(&ca_path, &new_pki.ca_pem).expect("rotate the bundle on disk"); + validator.refresh_roots().expect("roots reload"); + + assert_eq!( + validator.spiffe_id_from_bearer(&token).as_deref(), + Some(format!("spiffe://{TRUST_DOMAIN}{MACHINE_PATH}").as_str()), + "after the refresh the rotated CA must be trusted" + ); + } + + /// A bundle that cannot be read or parsed must not disarm bearer auth. + #[test] + fn a_broken_bundle_leaves_the_previous_roots_in_place() { + let dir = tempfile::tempdir().expect("tempdir"); + let pki = test_pki(MACHINE_PATH); + let ca_path = write_temp(&dir, "ca.pem", &pki.ca_pem); + let validator = NodeJwtValidator::from_root_ca_file(&ca_path, &NodeAuthConfig::default()) + .expect("validator builds"); + let token = mint_with(&dir, &pki); + + std::fs::write(&ca_path, "not a certificate").expect("truncate the bundle"); + validator + .refresh_roots() + .expect_err("a bundle with no trust anchors must not be accepted"); + + assert_eq!( + validator.spiffe_id_from_bearer(&token).as_deref(), + Some(format!("spiffe://{TRUST_DOMAIN}{MACHINE_PATH}").as_str()), + "the previous roots must still verify tokens" + ); + } + + #[test] + fn client_minted_token_round_trips_to_the_cert_spiffe_uri() { + let dir = tempfile::tempdir().expect("tempdir"); + let pki = test_pki(MACHINE_PATH); + let validator = validator_for(&dir, &pki.ca_pem); + + let token = mint_with(&dir, &pki); + assert_eq!( + validator.spiffe_id_from_bearer(&token).as_deref(), + Some(format!("spiffe://{TRUST_DOMAIN}{MACHINE_PATH}").as_str()) + ); + } + + #[test] + fn token_from_an_untrusted_ca_is_rejected() { + let dir = tempfile::tempdir().expect("tempdir"); + let pki = test_pki(MACHINE_PATH); + let other_pki = test_pki(MACHINE_PATH); + // Validator trusts a DIFFERENT root than the one that issued the cert. + let validator = validator_for(&dir, &other_pki.ca_pem); + + let token = mint_with(&dir, &pki); + assert!(validator.spiffe_id_from_bearer(&token).is_none()); + } + + #[test] + fn garbage_and_missing_chain_tokens_are_rejected() { + let dir = tempfile::tempdir().expect("tempdir"); + let pki = test_pki(MACHINE_PATH); + let validator = validator_for(&dir, &pki.ca_pem); + + assert!(validator.spiffe_id_from_bearer("not.a.jwt").is_none()); + + // Structurally valid ES256 JWT without an x5c header. + let key = rcgen::KeyPair::generate().expect("key"); + let encoding_key = jsonwebtoken::EncodingKey::from_ec_pem(key.serialize_pem().as_bytes()) + .expect("encoding key"); + let claims = serde_json::json!({ + "sub": format!("spiffe://{TRUST_DOMAIN}{MACHINE_PATH}"), + "aud": "nico-api", "iat": 0u64, "exp": u64::MAX / 2, + }); + let no_chain = jsonwebtoken::encode( + &jsonwebtoken::Header::new(Algorithm::ES256), + &claims, + &encoding_key, + ) + .expect("token"); + assert!(validator.spiffe_id_from_bearer(&no_chain).is_none()); + } + + #[test] + fn overlong_lifetime_is_rejected() { + let dir = tempfile::tempdir().expect("tempdir"); + let pki = test_pki(MACHINE_PATH); + let ca_path = write_temp(&dir, "ca.pem", &pki.ca_pem); + // A validator whose lifetime cap is below what the client mints. + let strict = NodeJwtValidator::from_root_ca_file( + &ca_path, + &NodeAuthConfig { + max_token_ttl_sec: 1, + ..NodeAuthConfig::default() + }, + ) + .expect("validator builds"); + + let token = mint_with(&dir, &pki); + assert!(strict.spiffe_id_from_bearer(&token).is_none()); + } +} diff --git a/crates/api-core/src/setup.rs b/crates/api-core/src/setup.rs index 96f88f2418..0b9aeb1631 100644 --- a/crates/api-core/src/setup.rs +++ b/crates/api-core/src/setup.rs @@ -406,6 +406,36 @@ pub(crate) async fn start_runtime( .map_err(|e| eyre::eyre!("failed to build NMX-C client pool: {e}"))?; let shared_nmxc_pool: Arc = Arc::new(nmxc_client_pool); + // Node-auth (Scout / DPU-agent bearer JWT, #355) preflight. Run before any + // DPF resource creation below, so a misconfiguration (the + // enabled=false + mtls_enabled=false lockout, bearer-over-plaintext, or an + // unreadable trust anchor) fails startup before it mutates cluster state. + // Validate unconditionally; when explicitly enabled, missing prerequisites + // fail rather than silently degrading. + carbide_config.node_auth.validate()?; + let node_jwt_validator = if carbide_config.node_auth.enabled { + // Bearer tokens must never be accepted over plaintext, and the + // validator trusts the same roots the TLS listener uses for client + // certificates — so a TLS listener is required on both counts. + if !matches!(carbide_config.listen_mode, ListenMode::Tls) { + return Err(eyre::eyre!( + "[node_auth] is enabled but listen_mode is not \"tls\"; bearer tokens must not be accepted over plaintext" + )); + } + let tls_ref = carbide_config + .tls + .as_ref() + .ok_or_else(|| eyre::eyre!("[node_auth] is enabled but [tls] is unset"))?; + Some(Arc::new( + crate::node_auth::NodeJwtValidator::from_root_ca_file( + &tls_ref.root_cafile_path, + &carbide_config.node_auth, + )?, + )) + } else { + None + }; + let dpf_sdk = initialize_dpf_sdk( &carbide_config, credential_manager.clone(), @@ -465,6 +495,7 @@ pub(crate) async fn start_runtime( certificate_provider, common_pools, credential_manager, + node_jwt_validator, database_connection: db_pool.clone(), dpu_health_log_limiter: LogLimiter::default(), dynamic_settings, @@ -610,6 +641,7 @@ async fn initialize_dpf_sdk( services: crate::dpf_services::mandatory_services( &services, &carbide_config.dpf.dpu_agent_bootstrap_ca, + &carbide_config.node_auth, ), proxy: carbide_config.dpf.proxy.clone(), deployment_type, diff --git a/crates/api-core/src/test_support/builder.rs b/crates/api-core/src/test_support/builder.rs index ab871269c6..a8246a0070 100644 --- a/crates/api-core/src/test_support/builder.rs +++ b/crates/api-core/src/test_support/builder.rs @@ -289,6 +289,7 @@ impl TestApiBuilder { bmc_session_manager, bms_client: std::sync::OnceLock::new(), secrets_context: self.secrets_context, + node_jwt_validator: None, } } } diff --git a/crates/api-core/src/test_support/default_config.rs b/crates/api-core/src/test_support/default_config.rs index 86f38b5bdf..04321483ec 100644 --- a/crates/api-core/src/test_support/default_config.rs +++ b/crates/api-core/src/test_support/default_config.rs @@ -95,6 +95,7 @@ pub fn get() -> CarbideConfig { enable_admin_ui: true, web_ui_sidebar_tools: vec![], log_history: Default::default(), + node_auth: Default::default(), observability: Default::default(), bgp_leaf_session_password: None, rack_validation_config: RackValidationConfig { diff --git a/crates/authn/Cargo.toml b/crates/authn/Cargo.toml index f7784741b7..1e86d09c9a 100644 --- a/crates/authn/Cargo.toml +++ b/crates/authn/Cargo.toml @@ -42,6 +42,7 @@ carbide-instrument = { path = "../instrument", features = ["test-support"] } carbide-rpc = { path = "../rpc" } carbide-test-support = { path = "../test-support" } rcgen = { workspace = true } +tokio = { workspace = true } [lints] workspace = true diff --git a/crates/authn/src/middleware.rs b/crates/authn/src/middleware.rs index 9dd44cb5e6..1216d95810 100644 --- a/crates/authn/src/middleware.rs +++ b/crates/authn/src/middleware.rs @@ -36,10 +36,33 @@ use crate::{SpiffeContext, SpiffeError}; // This middleware is not expected to enforce anything on its own, so anything // that an access control policy might need to do its work should be passed // along in the request extensions. +/// Verifies a bearer token (a node-auth JWT) and, on success, returns the +/// validated SPIFFE ID URI string from the token's `sub` claim. The +/// implementation lives outside this crate (api-core's `node_auth`) so the +/// authn layer stays free of JWT signing/key-management concerns. +/// +/// Returning the raw SPIFFE URI — rather than a `Principal` — lets the +/// middleware map it through the SAME [`SpiffeContext`] used for client certs, +/// so a JWT and an mTLS cert for the same machine yield identical principals. +pub trait BearerTokenAuthenticator: Send + Sync { + /// Returns the validated SPIFFE ID URI if `token` is a valid node-auth JWT, + /// or `None` if it is invalid/expired/unrecognized. + fn spiffe_id_from_bearer(&self, token: &str) -> Option; +} + #[derive(Clone)] pub struct CertDescriptionMiddleware { pub spiffe_context: Arc, pub extra_allowed_certs: Option, + /// Optional bearer-token (JWT) authenticator. When set, requests carrying an + /// `Authorization: Bearer ` header gain principals alongside any from + /// mTLS client certs (dual-support during the mTLS→JWT migration). + pub bearer_authenticator: Option>, + /// Whether machine mTLS client certificates mint machine principals. On by + /// default; disabled via `[node_auth] mtls_enabled = false` once a fleet has + /// migrated to bearer tokens. Scoped to machine certs — service and admin + /// client certs are unaffected. + pub machine_certs_enabled: bool, _authorization: std::marker::PhantomData, } @@ -51,9 +74,29 @@ impl CertDescriptionMiddleware { CertDescriptionMiddleware { spiffe_context: Arc::new(spiffe_context), extra_allowed_certs, + bearer_authenticator: None, + machine_certs_enabled: true, _authorization: std::marker::PhantomData, } } + + /// Enables bearer-token (JWT) authentication using the given authenticator. + #[must_use] + pub fn with_bearer_authenticator( + mut self, + authenticator: Arc, + ) -> Self { + self.bearer_authenticator = Some(authenticator); + self + } + + /// Controls whether machine mTLS client certificates are accepted as node + /// identity (`[node_auth] mtls_enabled`). + #[must_use] + pub fn with_machine_certs_enabled(mut self, enabled: bool) -> Self { + self.machine_certs_enabled = enabled; + self + } } impl Layer for CertDescriptionMiddleware { @@ -218,6 +261,12 @@ impl Principal { } } +/// Extracts the token from an `Authorization: Bearer ` header, if present. +fn bearer_token_from_headers(headers: &hyper::HeaderMap) -> Option<&str> { + let value = headers.get(hyper::header::AUTHORIZATION)?.to_str().ok()?; + value.strip_prefix("Bearer ").map(str::trim) +} + // try_external_cert will return a Pricipal::ExternalUser if this looks like some external cert fn try_external_cert( der_certificate: &[u8], @@ -540,12 +589,43 @@ where } fn call(&mut self, mut request: Request) -> Self::Future { - if let Some(_req_auth_header) = request.headers().get(hyper::header::AUTHORIZATION) { - // If we want to extract additional principals from the request's - // Authorization header, we can do it here. + let mut auth_context = AuthContext::::default(); + + // Bearer-token (node-auth JWT) authentication. Validated tokens map + // through the same SpiffeContext as client certs, so a JWT and an mTLS + // cert for the same machine produce identical principals. This runs in + // addition to (not instead of) cert auth to support the migration. + if let Some(authenticator) = &self.authorization_context.bearer_authenticator + && let Some(token) = bearer_token_from_headers(request.headers()) + && let Some(spiffe_uri) = authenticator.spiffe_id_from_bearer(token) + { + match crate::spiffe_id::SpiffeId::new(&spiffe_uri) { + Ok(spiffe_id) => match self + .authorization_context + .spiffe_context + .extract_service_identifier(&spiffe_id) + { + Ok(crate::SpiffeIdClass::Service(id)) => { + auth_context + .principals + .push(Principal::SpiffeServiceIdentifier(id)); + } + Ok(crate::SpiffeIdClass::Machine(id)) => { + auth_context + .principals + .push(Principal::SpiffeMachineIdentifier(id)); + } + Err(e) => { + tracing::debug!(target: "node_auth", "node-auth: bearer token SPIFFE id not recognized: {e}"); + } + }, + Err(e) => { + tracing::debug!(target: "node_auth", "node-auth: bearer token contained an unparsable SPIFFE URI: {e}"); + } + } } + let extensions = request.extensions_mut(); - let mut auth_context = AuthContext::::default(); if let Some(conn_attrs) = extensions.get::>() { let peer_certs = &conn_attrs.peer_certificates; // rustls presents the end-entity certificate first, intermediates @@ -556,15 +636,34 @@ where // per request, only when the whole chain minted no principal. let mut rejections = Vec::new(); let minted_before = auth_context.principals.len(); - let peer_cert_principals = peer_certs.iter().filter_map(|cert| { - match Principal::try_from_client_certificate(cert, &self.authorization_context) { - Ok(x) => Some(x), - Err(e) => { - rejections.push(e); - None + let peer_cert_principals = peer_certs + .iter() + .filter_map(|cert| { + match Principal::try_from_client_certificate(cert, &self.authorization_context) + { + Ok(x) => Some(x), + Err(e) => { + rejections.push(e); + None + } } - } - }); + }) + .filter(|principal| { + // `[node_auth] mtls_enabled = false`: machine certs no longer + // grant node identity (bearer JWTs are the only node auth + // path); service/admin cert principals pass through. + if !self.authorization_context.machine_certs_enabled + && matches!(principal, Principal::SpiffeMachineIdentifier(_)) + { + tracing::debug!( + target: "node_auth", + "node-auth: machine mTLS authentication disabled; ignoring machine client certificate" + ); + false + } else { + true + } + }); auth_context.principals.extend(peer_cert_principals); if auth_context.principals.len() == minted_before && let Some(leaf_error) = rejections.first() @@ -593,10 +692,13 @@ where #[cfg(test)] mod tests { use std::collections::HashSet; + use std::convert::Infallible; use std::sync::Mutex; use std::task::{Context, Poll}; use carbide_instrument::testing::{MetricsCapture, capture_logs}; + use hyper::header::AUTHORIZATION; + use tower::{Layer, ServiceExt}; use super::*; use crate::spiffe_id::TrustDomain; @@ -661,6 +763,14 @@ mod tests { } } + /// Test authenticator: maps the literal token "good" to a fixed SPIFFE URI. + struct FakeAuth(String); + impl BearerTokenAuthenticator for FakeAuth { + fn spiffe_id_from_bearer(&self, token: &str) -> Option { + (token == "good").then(|| self.0.clone()) + } + } + fn spiffe_context() -> SpiffeContext { SpiffeContext { trust_domain: TrustDomain::new("example.test").expect("trust domain"), @@ -840,4 +950,162 @@ mod tests { "recognition" ); } + + /// Runs a request through the middleware and returns the principals the authn + /// layer attached (captured from the request extensions by an inner service). + /// + /// The request carries `ConnectionAttributes` with no peer certificates — + /// a bearer-token client over plain TLS. Omitting them entirely is the + /// wiring-error path, which logs `authentication_connection_attributes_missing` + /// and would poison that callsite's cached tracing interest for the + /// capture-based test of that warning. + async fn principals_for( + middleware: CertDescriptionMiddleware, + auth_header: Option<&str>, + ) -> Vec { + principals_for_certs(middleware, auth_header, Vec::new()).await + } + + #[tokio::test] + async fn valid_bearer_token_yields_machine_principal() { + let middleware = CertDescriptionMiddleware::::new(None, spiffe_context()) + .with_bearer_authenticator(Arc::new(FakeAuth( + "spiffe://example.test/carbide-system/machine/m1".to_string(), + ))); + let principals = principals_for(middleware, Some("Bearer good")).await; + assert!( + principals.contains(&Principal::SpiffeMachineIdentifier("m1".to_string())), + "expected machine principal, got {principals:?}" + ); + } + + #[tokio::test] + async fn invalid_bearer_token_yields_no_machine_principal() { + let middleware = CertDescriptionMiddleware::::new(None, spiffe_context()) + .with_bearer_authenticator(Arc::new(FakeAuth( + "spiffe://example.test/carbide-system/machine/m1".to_string(), + ))); + let principals = principals_for(middleware, Some("Bearer bogus")).await; + assert!( + !principals + .iter() + .any(|p| matches!(p, Principal::SpiffeMachineIdentifier(_))), + "rejected token must not yield a machine principal, got {principals:?}" + ); + } + + /// Like [`principals_for`], but the request also presents client + /// certificates via [`ConnectionAttributes`]. + async fn principals_for_certs( + middleware: CertDescriptionMiddleware, + auth_header: Option<&str>, + peer_certificates: Vec>, + ) -> Vec { + let inner = tower::service_fn(|req: Request| async move { + let principals = req + .extensions() + .get::>() + .map(|ctx| ctx.principals.clone()) + .unwrap_or_default(); + Ok::<_, Infallible>(principals) + }); + let svc = middleware.layer(inner); + let mut builder = Request::builder(); + if let Some(value) = auth_header { + builder = builder.header(AUTHORIZATION, value); + } + let mut request = builder.body(tonic::body::Body::empty()).unwrap(); + request + .extensions_mut() + .insert(Arc::new(ConnectionAttributes { + peer_address: "192.0.2.9:4433".parse().expect("socket address"), + peer_certificates, + })); + svc.oneshot(request).await.unwrap() + } + + #[tokio::test] + async fn machine_cert_yields_machine_principal_by_default() { + let middleware = CertDescriptionMiddleware::::new(None, spiffe_context()); + let principals = principals_for_certs( + middleware, + None, + vec![spiffe_leaf_certificate("/carbide-system/machine/m1")], + ) + .await; + assert!( + principals.contains(&Principal::SpiffeMachineIdentifier("m1".to_string())), + "expected cert-derived machine principal, got {principals:?}" + ); + } + + #[tokio::test] + async fn machine_cert_ignored_when_machine_certs_disabled() { + // `[node_auth] mtls_enabled = false`: a machine client cert mints no + // machine principal, while a valid bearer token still does — the JWT + // becomes the only node auth path. + let middleware = CertDescriptionMiddleware::::new(None, spiffe_context()) + .with_machine_certs_enabled(false) + .with_bearer_authenticator(Arc::new(FakeAuth( + "spiffe://example.test/carbide-system/machine/m2".to_string(), + ))); + let principals = principals_for_certs( + middleware, + Some("Bearer good"), + vec![spiffe_leaf_certificate("/carbide-system/machine/m1")], + ) + .await; + assert!( + !principals.contains(&Principal::SpiffeMachineIdentifier("m1".to_string())), + "cert-derived machine principal must be dropped, got {principals:?}" + ); + assert!( + principals.contains(&Principal::SpiffeMachineIdentifier("m2".to_string())), + "bearer-derived machine principal must survive, got {principals:?}" + ); + } + + #[tokio::test] + async fn service_cert_unaffected_when_machine_certs_disabled() { + // The gate is scoped to machine certs: service identities on the same + // listener keep authenticating via mTLS. + let middleware = CertDescriptionMiddleware::::new(None, spiffe_context()) + .with_machine_certs_enabled(false); + let principals = principals_for_certs( + middleware, + None, + vec![spiffe_leaf_certificate("/carbide-system/sa/test-service")], + ) + .await; + assert!( + principals + .iter() + .any(|p| matches!(p, Principal::SpiffeServiceIdentifier(_))), + "service cert principal must survive, got {principals:?}" + ); + } + + #[tokio::test] + async fn bearer_ignored_when_no_authenticator_configured() { + // Without an authenticator installed, an Authorization header is ignored + // (nodes keep using mTLS) — no machine principal appears. + let middleware = CertDescriptionMiddleware::::new(None, spiffe_context()); + let principals = principals_for(middleware, Some("Bearer good")).await; + assert!( + !principals + .iter() + .any(|p| matches!(p, Principal::SpiffeMachineIdentifier(_))), + "no authenticator => no bearer principals, got {principals:?}" + ); + } + + #[test] + fn bearer_token_from_headers_parses_scheme() { + let mut headers = hyper::HeaderMap::new(); + assert_eq!(bearer_token_from_headers(&headers), None); + headers.insert(AUTHORIZATION, "Bearer abc.def".parse().unwrap()); + assert_eq!(bearer_token_from_headers(&headers), Some("abc.def")); + headers.insert(AUTHORIZATION, "Basic xyz".parse().unwrap()); + assert_eq!(bearer_token_from_headers(&headers), None); + } } diff --git a/crates/fmds/src/cfg.rs b/crates/fmds/src/cfg.rs index 2bdd5049dd..3ca6e7b93f 100644 --- a/crates/fmds/src/cfg.rs +++ b/crates/fmds/src/cfg.rs @@ -59,6 +59,13 @@ pub struct Options { #[clap(long)] pub client_key: Option, + /// Path of the dpu-agent's local API socket. When set, fmds fetches + /// short-lived node-auth bearer JWTs from the agent and presents them to + /// carbide-api — instead of (or in addition to) the mTLS client cert — + /// so this pod no longer needs the machine private key mounted. + #[clap(long, env = "FMDS_NODE_TOKEN_SOCKET")] + pub node_token_socket: Option, + /// Name of the interface to assign the metadata-service address to. #[clap(long, env = "FMDS_INTERFACE_NAME", default_value = "f_pf0hpf_if")] pub interface_name: String, diff --git a/crates/fmds/src/main.rs b/crates/fmds/src/main.rs index 030f2fed2d..57fe4c2490 100644 --- a/crates/fmds/src/main.rs +++ b/crates/fmds/src/main.rs @@ -26,6 +26,7 @@ use fmds::{http_request_metrics, nic_init}; use forge_tls::client_config::ClientCert; use rpc::fmds::fmds_config_service_server::FmdsConfigServiceServer; use rpc::forge_tls_client::ForgeClientConfig; +use rpc::node_token_socket::SocketTokenSource; use tracing::metadata::LevelFilter; use tracing_subscriber::EnvFilter; use tracing_subscriber::layer::SubscriberExt as _; @@ -79,16 +80,42 @@ async fn main() -> eyre::Result<()> { nic_init::assign_address(&options.interface_name, options.interface_cidr).await?; nic_init::setup_metadata_routing(&options.interface_name, options.interface_cidr).await?; - // Build ForgeClientConfig for phone_home if cert paths are provided - let forge_client_config = match (&options.root_ca, &options.client_cert, &options.client_key) { - (Some(root_ca), Some(client_cert), Some(client_key)) => { - Some(Arc::new(ForgeClientConfig::new( - root_ca.clone(), - Some(ClientCert { - cert_path: client_cert.clone(), - key_path: client_key.clone(), - }), - ))) + // Build ForgeClientConfig for phone_home. Either credential works alone: + // the shared mTLS client cert, and/or node-auth bearer tokens fetched from + // the dpu-agent's local API socket (#355) — the latter lets this pod run + // without the machine private key mounted at all. + let client_cert = match (&options.client_cert, &options.client_key) { + (Some(cert_path), Some(key_path)) => Some(ClientCert { + cert_path: cert_path.clone(), + key_path: key_path.clone(), + }), + (None, None) => None, + // Half a credential is a deployment bug; fail loudly instead of + // silently running without mTLS. + (Some(_), None) => { + eyre::bail!("--client-cert was provided without --client-key") + } + (None, Some(_)) => { + eyre::bail!("--client-key was provided without --client-cert") + } + }; + let forge_client_config = match &options.root_ca { + Some(root_ca) if client_cert.is_some() || options.node_token_socket.is_some() => { + let mut config = ForgeClientConfig::new(root_ca.clone(), client_cert); + if let Some(socket) = &options.node_token_socket { + tracing::info!( + socket = %socket, + "fetching node-auth bearer tokens from the dpu-agent local API" + ); + // Token mode usually runs without a client cert, which would + // otherwise leave the channel on the dummy TLS verifier. The + // bearer token is the client credential; the server still has + // to prove itself against the root CA. + config = config + .require_tls_enforcement() + .with_token_provider(SocketTokenSource::spawn(socket.clone())); + } + Some(Arc::new(config)) } _ => { tracing::warn!( diff --git a/crates/host-support/src/agent_config.rs b/crates/host-support/src/agent_config.rs index 1de547d89a..c292a2113f 100644 --- a/crates/host-support/src/agent_config.rs +++ b/crates/host-support/src/agent_config.rs @@ -118,6 +118,36 @@ pub struct ForgeSystemConfig { pub client_cert: String, #[serde(default = "default_client_key")] pub client_key: String, + /// Unix socket where the agent serves its local API (node tokens for + /// co-located services, issue #355). Works the same containerized (DPF) + /// and as a plain service on DPU OS; override when `/opt/forge` is not + /// the shared credential directory in a deployment. + #[serde(default = "default_local_api_socket")] + pub local_api_socket: String, + /// `aud` stamped on node-auth bearer JWTs (issue #355). Must match the + /// API's `[node_auth] audience`; a site that changes one must change the + /// other, or the API rejects every token this node mints. + #[serde(default = "default_node_auth_audience")] + pub node_auth_audience: String, +} + +impl ForgeSystemConfig { + /// Rejects values that would leave the node unable to authenticate. + /// + /// The `--node-auth-audience` flag validates itself at parse time, but the + /// TOML path had no equivalent, so a blank or whitespace-only audience in + /// a config file reached the minter and produced tokens the API rejects on + /// every request — the silent lockout the audience plumbing exists to + /// prevent. Callers run this after applying any CLI override so both + /// sources are held to the same rule. + pub fn validate(&self) -> Result<(), String> { + if self.node_auth_audience.trim().is_empty() { + return Err( + "forge-system.node-auth-audience: must not be empty or whitespace-only".to_string(), + ); + } + Ok(()) + } } // Called if no `[forge-system]` is provided at all. @@ -129,6 +159,8 @@ impl Default for ForgeSystemConfig { root_ca: default_root_ca(), client_cert: default_client_cert(), client_key: default_client_key(), + local_api_socket: default_local_api_socket(), + node_auth_audience: default_node_auth_audience(), } } } @@ -149,6 +181,14 @@ pub fn default_client_key() -> String { tls_default::default_client_key().to_string() } +pub fn default_local_api_socket() -> String { + ::rpc::node_token_socket::DEFAULT_AGENT_LOCAL_SOCKET.to_string() +} + +pub fn default_node_auth_audience() -> String { + ::rpc::node_jwt::NODE_JWT_AUDIENCE.to_string() +} + #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub struct MachineConfig { @@ -1073,4 +1113,43 @@ interface-id = \"91609f10-c91d-470d-a260-6293ea0c1200\" fs::read_to_string(format!("{TEST_DATA_DIR}/min_agent_config/output.toml")).unwrap(); assert_eq!(observed_output, expected_output); } + + /// The `--node-auth-audience` flag rejects a blank value at parse time; the + /// TOML path has to hold the same line. A blank audience mints tokens the + /// API rejects on every request, which is the silent lockout the whole + /// audience plumbing exists to prevent. + #[test] + fn forge_system_config_rejects_a_blank_node_auth_audience() { + assert!( + ForgeSystemConfig::default().validate().is_ok(), + "the default audience must be valid" + ); + + for blank in ["", " ", "\t"] { + let config = ForgeSystemConfig { + node_auth_audience: blank.to_string(), + ..ForgeSystemConfig::default() + }; + let err = config + .validate() + .expect_err("a blank audience must be rejected"); + assert!( + err.contains("node-auth-audience"), + "the error should name the field, got: {err}" + ); + } + } + + /// A config file that omits the key entirely still gets the default, so + /// validation must not turn an ordinary minimal config into a hard failure. + #[test] + fn a_config_without_an_audience_key_still_validates() { + let config: AgentConfig = toml::from_str( + fs::read_to_string(format!("{TEST_DATA_DIR}/min_agent_config/input.toml")) + .unwrap() + .as_str(), + ) + .unwrap(); + assert!(config.forge_system.validate().is_ok()); + } } diff --git a/crates/host-support/src/registration.rs b/crates/host-support/src/registration.rs index 7c642233eb..e2085137c1 100644 --- a/crates/host-support/src/registration.rs +++ b/crates/host-support/src/registration.rs @@ -312,7 +312,7 @@ pub async fn write_certs( ))?; tracing::info!(%client_cert, "Wrote new machine certificate PEM"); - tokio::fs::write(client_key, machine_certificate.private_key.as_slice()) + write_private_key(client_key, machine_certificate.private_key.as_slice()) .await .wrap_err(format!( "failed to write new machine certificate key to: {client_key}" @@ -323,3 +323,109 @@ pub async fn write_certs( Ok(()) } + +/// Writes the machine private key owner-readable only (0600). The key is +/// created with the restrictive mode from the first byte — not chmod'ed after +/// the fact — so there is no window where it sits world-readable; a pre-existing +/// key file from an older agent (written 0644 via plain `fs::write`) is +/// tightened as well. Everything that legitimately reads the key (dpu-agent, +/// scout, fmds, otelcol) runs as root, so 0600 root-owned loses nobody access. +#[cfg(unix)] +async fn write_private_key(path: &str, key: &[u8]) -> Result<(), std::io::Error> { + use std::os::unix::fs::PermissionsExt; + + use tokio::io::AsyncWriteExt; + + let mut file = tokio::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .mode(0o600) + .open(path) + .await?; + // `mode` only applies at creation; tighten files that already existed. + file.set_permissions(std::fs::Permissions::from_mode(0o600)) + .await?; + file.write_all(key).await?; + file.flush().await +} + +#[cfg(not(unix))] +async fn write_private_key(path: &str, key: &[u8]) -> Result<(), std::io::Error> { + tokio::fs::write(path, key).await +} + +#[cfg(all(test, unix))] +mod tests { + use std::os::unix::fs::PermissionsExt; + + use super::*; + + /// `write_certs` concatenates and writes bytes without parsing them, so + /// these are opaque sentinels rather than PEM. Real-looking `-----BEGIN EC + /// PRIVATE KEY-----` markers would buy nothing here and trip secret + /// scanners, which match the marker and not its contents. + fn test_certificate() -> MachineCertificate { + MachineCertificate { + public_key: b"test-leaf-certificate-bytes".to_vec(), + issuing_ca: b"test-issuing-ca-bytes".to_vec(), + private_key: b"test-machine-private-key-bytes".to_vec(), + } + } + + fn key_mode(path: &str) -> u32 { + std::fs::metadata(path) + .expect("key metadata") + .permissions() + .mode() + & 0o777 + } + + #[tokio::test] + async fn private_key_is_written_owner_only() { + let dir = tempfile::tempdir().expect("tempdir"); + let paths = ClientCert { + cert_path: dir.path().join("cert.pem").to_string_lossy().into_owned(), + key_path: dir.path().join("cert.key").to_string_lossy().into_owned(), + }; + + write_certs(Some(test_certificate()), Some(&paths)) + .await + .expect("write_certs succeeds"); + + assert_eq!(key_mode(&paths.key_path), 0o600, "fresh key must be 0600"); + let cert = std::fs::read_to_string(&paths.cert_path).expect("cert readable"); + assert!( + cert.contains("leaf") && cert.contains("ca"), + "cert file carries leaf + chain" + ); + } + + #[tokio::test] + async fn pre_existing_loose_key_is_tightened_on_rewrite() { + let dir = tempfile::tempdir().expect("tempdir"); + let paths = ClientCert { + cert_path: dir.path().join("cert.pem").to_string_lossy().into_owned(), + key_path: dir.path().join("cert.key").to_string_lossy().into_owned(), + }; + // A key written by an older agent via plain fs::write (umask default). + std::fs::write(&paths.key_path, b"old key").expect("seed old key"); + std::fs::set_permissions(&paths.key_path, std::fs::Permissions::from_mode(0o644)) + .expect("loosen"); + + write_certs(Some(test_certificate()), Some(&paths)) + .await + .expect("write_certs succeeds"); + + assert_eq!( + key_mode(&paths.key_path), + 0o600, + "renewal must tighten an old 0644 key" + ); + let key = std::fs::read_to_string(&paths.key_path).expect("key readable by owner"); + assert!( + key.contains("test-machine-private-key-bytes"), + "key content replaced" + ); + } +} diff --git a/crates/host-support/test/min_agent_config/output.toml b/crates/host-support/test/min_agent_config/output.toml index 3c5db0e597..33a5fafed5 100644 --- a/crates/host-support/test/min_agent_config/output.toml +++ b/crates/host-support/test/min_agent_config/output.toml @@ -3,6 +3,8 @@ api-server = "https://127.0.0.1:8001" root-ca = "/opt/forge/forge_root.pem" client-cert = "/opt/forge/machine_cert.pem" client-key = "/opt/forge/machine_cert.key" +local-api-socket = "/opt/forge/run/agent.sock" +node-auth-audience = "nico-api" [machine] interface-id = "91609f10-c91d-470d-a260-6293ea0c1234" diff --git a/crates/rpc/Cargo.toml b/crates/rpc/Cargo.toml index afd92b72d7..eee3aed98f 100644 --- a/crates/rpc/Cargo.toml +++ b/crates/rpc/Cargo.toml @@ -62,6 +62,8 @@ carbide-uuid = { path = "../uuid" } chrono = { features = ["serde"], workspace = true } const_format = { workspace = true } eyre = { workspace = true } +jsonwebtoken = { workspace = true, features = ["rust_crypto"] } +p256 = { workspace = true } hyper = { workspace = true, features = ["full"] } hyper-rustls = { workspace = true, features = ["http2"] } hyper-util = { features = [ @@ -127,6 +129,8 @@ carbide-api-model = { path = "../api-model", features = ["test-support"] } carbide-libmlx-model = { path = "../libmlx-model", features = ["test-support"] } carbide-test-support = { path = "../test-support" } criterion = { workspace = true } +rcgen = { workspace = true } +tempfile = { workspace = true } [[bench]] name = "machine_convert" diff --git a/crates/rpc/build.rs b/crates/rpc/build.rs index e6afbffd34..ada07c5929 100644 --- a/crates/rpc/build.rs +++ b/crates/rpc/build.rs @@ -1098,6 +1098,7 @@ fn main() -> Result<(), Box> { "proto/site_explorer.proto", "proto/dns.proto", "proto/fmds.proto", + "proto/agent_local.proto", ], &["proto"], ) diff --git a/crates/rpc/proto/agent_local.proto b/crates/rpc/proto/agent_local.proto new file mode 100644 index 0000000000..f3d9822a20 --- /dev/null +++ b/crates/rpc/proto/agent_local.proto @@ -0,0 +1,50 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +package agent_local; + +// Local (on-DPU) API served by forge-dpu-agent over a unix domain socket +// (the agent's `local-api-socket` setting, which defaults to a dedicated run/ +// subdirectory of the shared credentials directory, so consumers can mount the +// socket read-write while the credentials stay read-only). The literal default +// lives in DEFAULT_AGENT_LOCAL_SOCKET and is deliberately not repeated here: +// REST proto normalization rewrites vendor names inside comments, which would +// silently corrupt an inlined filesystem path into one that does not exist. +// This is the consolidation point for +// agent <-> co-located-service communication: services that today mount the +// machine cert/key (or synced files) to talk to nico-api can instead ask the +// agent, and future local needs should add RPCs here rather than new +// sockets, ports, or file drops. +service AgentLocal { + // Returns the current node-auth bearer JWT, minted by the agent from the + // machine's client-certificate key (issue #355). Callers present it as + // `Authorization: Bearer ` to nico-api and must fetch a fresh one + // before `expires_at`. Only the agent ever touches the private key. + rpc GetNodeToken(GetNodeTokenRequest) returns (GetNodeTokenResponse); +} + +message GetNodeTokenRequest { +} + +message GetNodeTokenResponse { + // The signed ES256 JWT. + string token = 1; + // Unix seconds when the token expires. + uint64 expires_at = 2; +} diff --git a/crates/rpc/src/forge_tls_client.rs b/crates/rpc/src/forge_tls_client.rs index fd06f71245..90d5a1de1b 100644 --- a/crates/rpc/src/forge_tls_client.rs +++ b/crates/rpc/src/forge_tls_client.rs @@ -41,6 +41,7 @@ use x509_parser::prelude::{FromDer, X509Certificate}; use crate::forge::VersionRequest; use crate::forge_resolver::resolver::ResolverError; use crate::forge_tls_client::ConfigurationError::CouldNotReadRootCa; +use crate::node_jwt::{BearerAuthService, NodeJwtMinter, NodeTokenProvider}; use crate::protos::forge::forge_client::ForgeClient; use crate::protos::nmx_c::nmx_controller_client::NmxControllerClient; use crate::{forge_resolver, protos}; @@ -96,6 +97,12 @@ pub struct ForgeClientConfig { pub socks_proxy: Option, pub connect_retries_max: Option, pub connect_retries_interval: Option, + /// Optional node-auth token provider (issue #355). When set, each request + /// carries an `Authorization: Bearer ` — either self-signed with the + /// client certificate's own private key ([`NodeJwtMinter`]) or fetched + /// from the dpu-agent's local API (`SocketTokenSource`). Independent of + /// mTLS: the channel may present a client cert, a token, or both. + pub node_token_provider: Option>, } impl ForgeClientConfig { @@ -134,9 +141,66 @@ impl ForgeClientConfig { // MR though, I think. connect_retries_max: Some(3), connect_retries_interval: Some(Duration::from_secs(20)), + node_token_provider: None, } } + /// Restores server-certificate validation on a config built without a + /// client certificate. + /// + /// [`ForgeClientConfig::new`] disables TLS enforcement whenever + /// `client_cert` is `None`, which predates node-auth: back then "no client + /// cert" meant "no credentials at all", and such callers were not expected + /// to reach the API over verified TLS. A node-token client does hold + /// credentials — a bearer JWT brokered by the dpu-agent (issue #355) — and + /// must still authenticate the server against `root_ca_path` rather than + /// fall back to [`DummyTlsVerifier`]. Callers that never opt in keep the + /// previous behavior. + /// + /// `DISABLE_TLS_ENFORCEMENT` continues to win, so local-development + /// overrides work the same as they do for mTLS clients. + #[must_use] + pub fn require_tls_enforcement(mut self) -> Self { + self.enforce_tls = std::env::var("DISABLE_TLS_ENFORCEMENT").is_err(); + self + } + + /// Enables node-auth JWTs: requests built from this config mint and carry + /// short-lived bearer tokens signed with the configured client cert's key. + /// A no-op when no client cert is configured. + /// + /// `audience` must match the API's `[node_auth] audience` — the server + /// rejects any other value outright, so a client pinned to the default + /// would fail every request at a site that changed it. + #[must_use] + pub fn with_node_jwt(mut self, audience: String) -> Self { + self.node_token_provider = self.client_cert.as_ref().map(|client_cert| { + NodeJwtMinter::with_audience( + client_cert.cert_path.clone(), + client_cert.key_path.clone(), + audience, + ) as Arc + }); + self + } + + /// Attaches an explicit node-auth token provider — e.g. a pre-built + /// [`NodeJwtMinter`] the caller also serves through the agent's local + /// API, or a `SocketTokenSource` in a process that holds no key at all. + /// + /// Implies [`require_tls_enforcement`](Self::require_tls_enforcement). + /// A bearer token IS the client's credential, so the server must always + /// be authenticated before one is handed over — a token-only config + /// (no client cert) would otherwise sit on [`DummyTlsVerifier`] and + /// present its token to whatever answered. Enforcing it here rather than + /// asking every caller to remember the pairing keeps the type hard to + /// misuse; `DISABLE_TLS_ENFORCEMENT` still wins for local development. + #[must_use] + pub fn with_token_provider(mut self, provider: Arc) -> Self { + self.node_token_provider = Some(provider); + self.require_tls_enforcement() + } + /// This is required when using `ForgeTlsConfig` on a DPU to communicate with site-controller. /// The mgmt interface exists in the mgmt VRF. `use_mgmt_vrf` sets the /// `SO_BINDTODEVICE` socket option on the client socket used when performing DNS queries @@ -431,6 +495,22 @@ impl<'a> ForgeTlsClient<'a> { error: e, })?; + // `enforce_tls` governs how a TLS handshake is verified, but the + // handshake only happens for an https:// URL — a plaintext one skips + // TLS setup entirely while `BearerAuthService` below still stamps the + // token, putting a live credential on the wire in the clear. Refuse + // that combination outright. `DISABLE_TLS_ENFORCEMENT` is honored for + // parity with the rest of this config so local development still works. + if self.forge_client_config.node_token_provider.is_some() + && uri.scheme() != Some(&tonic::codegen::http::uri::Scheme::HTTPS) + && std::env::var("DISABLE_TLS_ENFORCEMENT").is_err() + { + return Err(ConfigurationError::BearerTokenOverPlaintext { + uri_string: url.as_ref().to_string(), + } + .into()); + } + let connector = self.build_https_client(url.as_ref()).await?; // ping interval + ping timeout should add up to less than tcp_user_timeout, @@ -448,6 +528,13 @@ impl<'a> ForgeTlsClient<'a> { .pool_max_idle_per_host(2) .timer(TokioTimer::new()) .build(connector); + // Stamp a freshly-minted node-auth bearer token onto each request when + // configured (issue #355). A `None` minter is a transparent pass-through, + // so the boxed service type is identical in both modes. + let hyper_client = BearerAuthService::new( + hyper_client, + self.forge_client_config.node_token_provider.clone(), + ); // Inject the issuing span's W3C trace context into every request this client sends // (issue #2438). Wrapping before `boxed_clone` keeps the erased `BoxCloneService` type. let hyper_client = trace_propagation::TraceInjectService::new(hyper_client).boxed_clone(); @@ -744,6 +831,11 @@ pub enum ConfigurationError { }, #[error("could not read root CA cert at {path}: {error}")] CouldNotReadRootCa { path: String, error: io::Error }, + #[error( + "refusing to send node-auth bearer tokens to non-HTTPS URL {uri_string}: \ + the token would travel in cleartext" + )] + BearerTokenOverPlaintext { uri_string: String }, #[error("invalid client cert: {0}")] InvalidClientCert(rustls::Error), #[error("error configuring resolver: {0}")] @@ -769,6 +861,103 @@ mod tests { use super::*; + /// A node-token client presents no client certificate, which by itself + /// drops the channel onto `DummyTlsVerifier`. `require_tls_enforcement` + /// puts it back on root-CA validation. + #[test] + fn require_tls_enforcement_restores_validation_without_a_client_cert() { + if std::env::var("DISABLE_TLS_ENFORCEMENT").is_ok() { + // The override wins by design; the assertions below would not hold. + return; + } + + let config = ForgeClientConfig::new("/etc/carbide/root-ca.pem".to_string(), None); + assert!( + !config.enforce_tls, + "a config without a client cert starts out unenforced" + ); + + let config = config.require_tls_enforcement(); + assert!( + config.enforce_tls, + "opting in must restore server certificate validation" + ); + } + + #[derive(Debug)] + struct FixedToken; + + impl NodeTokenProvider for FixedToken { + fn current(&self) -> Option { + Some("a.b.c".to_string()) + } + } + + /// The bearer token IS the credential, so the server must be authenticated + /// before one is sent. Callers must not have to remember to pair + /// `with_token_provider` with `require_tls_enforcement` — forgetting it + /// would hand tokens to whatever answered the connection. + #[test] + fn with_token_provider_enforces_tls_without_being_asked() { + if std::env::var("DISABLE_TLS_ENFORCEMENT").is_ok() { + // The override wins by design; the assertion below would not hold. + return; + } + + let config = ForgeClientConfig::new("/etc/carbide/root-ca.pem".to_string(), None) + .with_token_provider(Arc::new(FixedToken)); + + assert!( + config.enforce_tls, + "attaching a token provider must restore server validation by itself" + ); + } + + /// TLS setup only runs for an https:// URL, so `enforce_tls` alone does not + /// cover a plaintext one — the token would still be stamped and travel in + /// the clear. Building such a client has to fail. + #[tokio::test] + async fn bearer_tokens_are_refused_over_plaintext() { + if std::env::var("DISABLE_TLS_ENFORCEMENT").is_ok() { + // The override deliberately permits plaintext development. + return; + } + + let config = ForgeClientConfig::new("/etc/carbide/root-ca.pem".to_string(), None) + .with_token_provider(Arc::new(FixedToken)); + + let error = ForgeTlsClient::new(&config) + .build("http://carbide-api.local:8080") + .await + .expect_err("a token client must refuse a plaintext endpoint"); + + assert!( + error.to_string().contains("cleartext"), + "the error should explain the refusal, got: {error}" + ); + } + + /// The same client over https:// must still build — the guard above is + /// scoped to the scheme, not to token clients in general. + #[tokio::test] + async fn bearer_tokens_are_allowed_over_https() { + let config = ForgeClientConfig::new("/etc/carbide/root-ca.pem".to_string(), None) + .with_token_provider(Arc::new(FixedToken)); + + // Reaching a root-CA read means the plaintext guard let it through; + // the file itself need not exist in a unit test. + let result = ForgeTlsClient::new(&config) + .build("https://carbide-api.local:8080") + .await; + + if let Err(error) = result { + assert!( + !error.to_string().contains("cleartext"), + "https must not trip the plaintext guard, got: {error}" + ); + } + } + #[tokio::test] // test_max_retries builds up an instance of hyper client using // the ForgeHttpConnector, which is the same configuration used diff --git a/crates/rpc/src/lib.rs b/crates/rpc/src/lib.rs index 40b3beb4ce..be1140d95f 100644 --- a/crates/rpc/src/lib.rs +++ b/crates/rpc/src/lib.rs @@ -74,13 +74,15 @@ pub use crate::protos::machine_discovery::{ self, BlockDevice, Cpu, DiscoveryInfo, DmiData, NetworkInterface, NvmeDevice, PciDeviceProperties, }; -pub use crate::protos::{fmds, health, scout_firmware_upgrade, site_explorer}; +pub use crate::protos::{agent_local, fmds, health, scout_firmware_upgrade, site_explorer}; pub mod errors; pub mod forge_tls_client; pub mod libmlx; pub mod measured_boot; pub mod network; +pub mod node_jwt; +pub mod node_token_socket; pub mod protos; pub mod secrets; mod site_explorer_report; diff --git a/crates/rpc/src/node_jwt.rs b/crates/rpc/src/node_jwt.rs new file mode 100644 index 0000000000..eb6f5dd9b2 --- /dev/null +++ b/crates/rpc/src/node_jwt.rs @@ -0,0 +1,479 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Client-side node-auth JWT minting (issue NVIDIA/infra-controller#355). +//! +//! The node self-signs short-lived ES256 JWTs with the private key of its +//! EXISTING mTLS client certificate; the certificate chain rides along in the +//! token's `x5c` header (RFC 7515 §4.1.6) so the API can verify it against the +//! same root CA its TLS listener already trusts. No new key material, no key +//! storage, and no refresh RPC: a fresh token is minted locally whenever the +//! cached one nears expiry, and key "rotation" happens for free when the +//! client certificate renews. +//! +//! [`BearerAuthService`] is the tower middleware that stamps the current token +//! onto each outgoing request's `Authorization` header. Minting is best-effort: +//! if the cert/key files are missing or unreadable (e.g. before first +//! registration), requests simply carry no bearer header and the channel's +//! mTLS client cert remains the only credential. + +use std::io::Cursor; +use std::sync::{Arc, RwLock}; +use std::task::{Context, Poll}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use data_encoding::BASE64; +use jsonwebtoken::{Algorithm, EncodingKey, Header}; +use p256::pkcs8::{EncodePrivateKey, LineEnding}; +use serde::Serialize; +use tower::Service; +use x509_parser::prelude::{FromDer, GeneralName, X509Certificate}; + +/// `aud` claim stamped on minted tokens; must match the API's +/// `[node_auth] audience`. +pub const NODE_JWT_AUDIENCE: &str = "nico-api"; + +/// Lifetime of minted tokens. Deliberately short: tokens cost nothing to +/// re-mint locally, so a leaked one ages out in minutes. +pub const NODE_JWT_TTL_SECS: u64 = 300; + +/// A cached token is reused until it has less than this long left, then +/// re-minted. Comfortably above per-request latency, comfortably below TTL. +const REMINT_MARGIN_SECS: u64 = 60; + +#[derive(Debug, thiserror::Error)] +pub enum NodeJwtError { + #[error("could not read client certificate or key: {0}")] + Io(#[from] std::io::Error), + #[error("client certificate file contains no parsable certificate")] + NoCertificate, + #[error("client certificate is not usable for node JWTs: {0}")] + BadCertificate(String), + #[error("client private key is not a usable EC key: {0}")] + BadKey(String), + #[error("client private key does not match the certificate's public key")] + KeyCertMismatch, + #[error("JWT signing failed: {0}")] + Sign(#[from] jsonwebtoken::errors::Error), + #[error("system clock is before the UNIX epoch")] + Clock, +} + +#[derive(Clone)] +struct CachedToken { + token: String, + expires_at: u64, +} + +/// Mints and caches node-auth JWTs from the node's existing mTLS client +/// certificate and private key files. +/// +/// Cert and key are re-read from disk on every mint, so certificate renewal +/// (which rewrites both files in place) is picked up automatically on the +/// next re-mint without any coordination. +pub struct NodeJwtMinter { + cert_path: String, + key_path: String, + /// `aud` stamped on every minted token. Must match the API's + /// `[node_auth] audience`, which is configurable — a minter pinned to the + /// default would be rejected wholesale by a site that changed it. + audience: String, + cached: RwLock>, +} + +/// Manual impl so the cached token (a live credential) never lands in debug +/// output of the client config. +impl std::fmt::Debug for NodeJwtMinter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("NodeJwtMinter") + .field("cert_path", &self.cert_path) + .field("key_path", &self.key_path) + .finish_non_exhaustive() + } +} + +/// Claims carried by a node-auth JWT. `sub` duplicates the certificate's +/// SPIFFE URI SAN; the server derives identity from the *verified certificate* +/// and only cross-checks `sub` against it, so a forged `sub` buys nothing. +#[derive(Debug, Serialize)] +struct NodeClaims<'a> { + sub: &'a str, + aud: &'a str, + iat: u64, + exp: u64, +} + +impl NodeJwtMinter { + /// Mints tokens for the default audience ([`NODE_JWT_AUDIENCE`]). Use + /// [`NodeJwtMinter::with_audience`] when the API's `[node_auth] audience` + /// has been changed from the default. + #[must_use] + pub fn new(cert_path: String, key_path: String) -> Arc { + Self::with_audience(cert_path, key_path, NODE_JWT_AUDIENCE.to_string()) + } + + #[must_use] + pub fn with_audience(cert_path: String, key_path: String, audience: String) -> Arc { + Arc::new(Self { + cert_path, + key_path, + audience, + cached: RwLock::new(None), + }) + } + + /// Returns a currently-valid token, re-minting if the cached one is + /// missing or close to expiry. Returns `None` (and logs at debug) when + /// minting is impossible — e.g. the cert/key files don't exist yet — so + /// callers degrade gracefully to mTLS-only. + pub fn current(&self) -> Option { + self.current_with_expiry().map(|(token, _)| token) + } + + /// Like [`current`](Self::current), but also returns the token's expiry + /// (unix seconds) — used by the agent's local API to tell co-located + /// consumers when to re-fetch. + pub fn current_with_expiry(&self) -> Option<(String, u64)> { + let now = unix_now().ok()?; + if let Ok(guard) = self.cached.read() + && let Some(cached) = guard.as_ref() + && cached.expires_at > now + REMINT_MARGIN_SECS + { + return Some((cached.token.clone(), cached.expires_at)); + } + match self.mint(now) { + Ok(minted) => { + let result = (minted.token.clone(), minted.expires_at); + if let Ok(mut guard) = self.cached.write() { + *guard = Some(minted); + } + Some(result) + } + Err(error) => { + tracing::debug!( + target: "node_auth", + cert_path = %self.cert_path, + %error, + "node-auth: could not mint node JWT; continuing with mTLS only" + ); + None + } + } + } + + fn mint(&self, now: u64) -> Result { + let cert_pem = std::fs::read(&self.cert_path)?; + let key_pem = std::fs::read_to_string(&self.key_path)?; + + let chain = rustls_pemfile::certs(&mut Cursor::new(&cert_pem[..])) + .collect::, _>>() + .map_err(|e| NodeJwtError::BadCertificate(e.to_string()))?; + let leaf = chain.first().ok_or(NodeJwtError::NoCertificate)?; + let sub = spiffe_uri_from_cert(leaf.as_ref())?; + + // Certificate renewal rewrites the two files in sequence, so a mint + // landing in between can pair a new certificate with the old key. + // Signing would still succeed and the API would reject the result, + // and the bad token would sit in the cache for a re-mint margin. Catch + // the mismatch here instead: returning an error means no bearer header + // on this request and a fresh attempt on the next one. + if !key_matches_certificate(&key_pem, leaf.as_ref())? { + return Err(NodeJwtError::KeyCertMismatch); + } + + let mut header = Header::new(Algorithm::ES256); + header.typ = Some("JWT".to_string()); + header.x5c = Some(chain.iter().map(|c| BASE64.encode(c.as_ref())).collect()); + + let expires_at = now + NODE_JWT_TTL_SECS; + let claims = NodeClaims { + sub: &sub, + aud: &self.audience, + iat: now, + exp: expires_at, + }; + let token = jsonwebtoken::encode(&header, &claims, &ec_encoding_key(&key_pem)?)?; + Ok(CachedToken { token, expires_at }) + } +} + +fn unix_now() -> Result { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .map_err(|_| NodeJwtError::Clock) +} + +/// Extracts the single SPIFFE URI SAN from the certificate — the same field +/// the server's authn layer maps to a machine principal for mTLS. +fn spiffe_uri_from_cert(der: &[u8]) -> Result { + let (_, cert) = X509Certificate::from_der(der) + .map_err(|e| NodeJwtError::BadCertificate(format!("X.509 parse error: {e}")))?; + let san = cert + .subject_alternative_name() + .map_err(|e| NodeJwtError::BadCertificate(format!("bad SAN extension: {e}")))? + .ok_or_else(|| NodeJwtError::BadCertificate("no SAN extension".to_string()))?; + san.value + .general_names + .iter() + .find_map(|name| match name { + GeneralName::URI(uri) if uri.starts_with("spiffe://") => Some(uri.to_string()), + _ => None, + }) + .ok_or_else(|| NodeJwtError::BadCertificate("no SPIFFE URI SAN".to_string())) +} + +/// Builds an ES256 signing key from the client key PEM. Vault-issued machine +/// keys are SEC1 (`BEGIN EC PRIVATE KEY`), which `jsonwebtoken` cannot load +/// directly, so those are re-encoded to PKCS#8 first. +/// Whether `key_pem`'s public half is the one certified by `leaf_der`. +/// +/// Both encodings the node may hold are accepted: Vault issues SEC1 ("EC +/// PRIVATE KEY") and renewal may leave PKCS#8. The comparison is on the +/// uncompressed SEC1 point, which is exactly what an EC `SubjectPublicKeyInfo` +/// carries. +fn key_matches_certificate(key_pem: &str, leaf_der: &[u8]) -> Result { + let secret = if key_pem.contains("BEGIN EC PRIVATE KEY") { + p256::SecretKey::from_sec1_pem(key_pem).map_err(|e| NodeJwtError::BadKey(e.to_string()))? + } else { + ::from_pkcs8_pem(key_pem) + .map_err(|e| NodeJwtError::BadKey(e.to_string()))? + }; + + let (_, cert) = X509Certificate::from_der(leaf_der) + .map_err(|e| NodeJwtError::BadCertificate(e.to_string()))?; + let certified = &cert.public_key().subject_public_key.data; + let from_key = { + use p256::elliptic_curve::sec1::ToSec1Point as _; + secret.public_key().to_sec1_point(false) + }; + + Ok(certified.as_ref() == from_key.as_bytes()) +} + +fn ec_encoding_key(key_pem: &str) -> Result { + if key_pem.contains("BEGIN EC PRIVATE KEY") { + let secret = p256::SecretKey::from_sec1_pem(key_pem) + .map_err(|e| NodeJwtError::BadKey(e.to_string()))?; + let pkcs8 = secret + .to_pkcs8_pem(LineEnding::LF) + .map_err(|e| NodeJwtError::BadKey(e.to_string()))?; + EncodingKey::from_ec_pem(pkcs8.as_bytes()).map_err(NodeJwtError::Sign) + } else { + EncodingKey::from_ec_pem(key_pem.as_bytes()).map_err(NodeJwtError::Sign) + } +} + +/// A source of node-auth bearer tokens for outgoing requests. Implemented by +/// [`NodeJwtMinter`] (holds the key, signs locally) and +/// [`SocketTokenSource`](crate::node_token_socket::SocketTokenSource) +/// (fetches from the dpu-agent's local API — the caller never sees the key). +/// +/// `current` runs on the request path, so implementations must be non-blocking: +/// return a cached token or `None`, never wait on I/O. +pub trait NodeTokenProvider: Send + Sync + std::fmt::Debug { + /// Returns a currently-valid token, or `None` if one isn't available + /// (the request then proceeds with whatever else the channel carries). + fn current(&self) -> Option; +} + +impl NodeTokenProvider for NodeJwtMinter { + fn current(&self) -> Option { + NodeJwtMinter::current(self) + } +} + +/// Tower middleware that injects `Authorization: Bearer ` onto each +/// request when a [`NodeTokenProvider`] is configured. A `None` provider is a +/// no-op, so the same client construction path serves both token and +/// mTLS-only modes. +#[derive(Clone)] +pub struct BearerAuthService { + inner: S, + minter: Option>, +} + +impl BearerAuthService { + pub fn new(inner: S, minter: Option>) -> Self { + Self { inner, minter } + } +} + +impl Service> for BearerAuthService +where + S: Service>, +{ + type Response = S::Response; + type Error = S::Error; + type Future = S::Future; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx) + } + + fn call(&mut self, mut req: hyper::Request) -> Self::Future { + if let Some(value) = self + .minter + .as_ref() + .and_then(|minter| minter.current()) + .and_then(|token| hyper::http::HeaderValue::from_str(&format!("Bearer {token}")).ok()) + { + req.headers_mut() + .insert(hyper::header::AUTHORIZATION, value); + } + self.inner.call(req) + } +} + +#[cfg(test)] +mod tests { + use jsonwebtoken::{DecodingKey, Validation}; + use serde::Deserialize; + + use super::*; + + #[derive(Debug, Deserialize)] + struct Claims { + sub: String, + exp: u64, + iat: u64, + } + + const SPIFFE_URI: &str = "spiffe://forge.local/forge-system/machine/fm100xtest"; + + /// Self-signed leaf with a SPIFFE URI SAN plus its PKCS#8 key PEM — + /// stand-ins for the Vault-issued client cert/key pair on a node. + fn cert_and_key() -> (String, String) { + let mut params = rcgen::CertificateParams::default(); + params.subject_alt_names = vec![rcgen::SanType::URI( + rcgen::string::Ia5String::try_from(SPIFFE_URI.to_string()).expect("uri"), + )]; + let key = rcgen::KeyPair::generate().expect("key pair"); + let cert = params.self_signed(&key).expect("certificate"); + (cert.pem(), key.serialize_pem()) + } + + fn write_temp(dir: &tempfile::TempDir, name: &str, contents: &str) -> String { + let path = dir.path().join(name); + std::fs::write(&path, contents).expect("write"); + path.to_string_lossy().into_owned() + } + + fn decode_against_own_cert(token: &str) -> Claims { + // Validate exactly as the server does: pull the leaf from x5c, take + // its SPKI EC point, verify the signature with it. + let header = jsonwebtoken::decode_header(token).expect("header"); + assert_eq!(header.alg, Algorithm::ES256); + let chain = header.x5c_der().expect("x5c decodes").expect("x5c present"); + let (_, cert) = X509Certificate::from_der(&chain[0]).expect("leaf parses"); + let decoding_key = DecodingKey::from_ec_der(&cert.public_key().subject_public_key.data); + let mut validation = Validation::new(Algorithm::ES256); + validation.set_audience(&[NODE_JWT_AUDIENCE]); + jsonwebtoken::decode::(token, &decoding_key, &validation) + .expect("token validates") + .claims + } + + /// Certificate renewal rewrites cert and key in sequence, so a mint can + /// land on a new certificate paired with the previous key. That signs + /// cleanly but the API cannot verify it, so it must never be minted or + /// cached — the next attempt, on a consistent pair, should succeed. + #[test] + fn a_certificate_from_a_different_key_is_never_minted_or_cached() { + let dir = tempfile::tempdir().expect("tempdir"); + let (cert_pem, _) = cert_and_key(); + let (_, other_key_pem) = cert_and_key(); + + let cert_path = write_temp(&dir, "cert.pem", &cert_pem); + let key_path = write_temp(&dir, "key.pem", &other_key_pem); + let minter = NodeJwtMinter::new(cert_path, key_path.clone()); + + assert!( + minter.current().is_none(), + "a token the API could not verify must not be minted" + ); + + // Renewal completes: the matching key lands. + let (matching_cert, matching_key) = cert_and_key(); + std::fs::write(&key_path, &matching_key).expect("write key"); + let cert_path = write_temp(&dir, "cert.pem", &matching_cert); + let minter = NodeJwtMinter::new(cert_path, key_path); + assert!( + minter.current().is_some(), + "a consistent cert/key pair must mint normally" + ); + } + + #[test] + fn mints_a_token_signed_by_the_client_cert_key() { + let (cert_pem, key_pem) = cert_and_key(); + let dir = tempfile::tempdir().expect("tempdir"); + let minter = NodeJwtMinter::new( + write_temp(&dir, "cert.pem", &cert_pem), + write_temp(&dir, "key.pem", &key_pem), + ); + + let token = minter.current().expect("token minted"); + let claims = decode_against_own_cert(&token); + assert_eq!(claims.sub, SPIFFE_URI); + assert_eq!(claims.exp - claims.iat, NODE_JWT_TTL_SECS); + } + + #[test] + fn sec1_key_pem_is_accepted() { + // Vault PKI hands out SEC1-encoded EC keys; re-encode the test key the + // same way and make sure minting still works. + use p256::pkcs8::DecodePrivateKey; + let (cert_pem, key_pem) = cert_and_key(); + let secret = p256::SecretKey::from_pkcs8_pem(&key_pem).expect("pkcs8 parses"); + let sec1_pem = secret + .to_sec1_pem(LineEnding::LF) + .expect("sec1 encodes") + .to_string(); + assert!(sec1_pem.contains("BEGIN EC PRIVATE KEY")); + + let dir = tempfile::tempdir().expect("tempdir"); + let minter = NodeJwtMinter::new( + write_temp(&dir, "cert.pem", &cert_pem), + write_temp(&dir, "key.pem", &sec1_pem), + ); + let token = minter.current().expect("token minted from SEC1 key"); + assert_eq!(decode_against_own_cert(&token).sub, SPIFFE_URI); + } + + #[test] + fn token_is_cached_until_near_expiry() { + let (cert_pem, key_pem) = cert_and_key(); + let dir = tempfile::tempdir().expect("tempdir"); + let minter = NodeJwtMinter::new( + write_temp(&dir, "cert.pem", &cert_pem), + write_temp(&dir, "key.pem", &key_pem), + ); + let first = minter.current().expect("token"); + let second = minter.current().expect("token"); + assert_eq!(first, second, "fresh token must be served from cache"); + } + + #[test] + fn missing_files_yield_none_not_panic() { + let minter = NodeJwtMinter::new( + "/nonexistent/cert.pem".to_string(), + "/nonexistent/key.pem".to_string(), + ); + assert!(minter.current().is_none()); + } +} diff --git a/crates/rpc/src/node_token_socket.rs b/crates/rpc/src/node_token_socket.rs new file mode 100644 index 0000000000..5754d8e5a8 --- /dev/null +++ b/crates/rpc/src/node_token_socket.rs @@ -0,0 +1,281 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Node-auth tokens for co-located services that hold no key (issue #355). +//! +//! On a DPU, only the dpu-agent holds the machine's private key; other NICo +//! pods (fmds, …) obtain bearer tokens from the agent's local API — the +//! `AgentLocal` gRPC service on a unix socket in the shared `/opt/forge` +//! directory — instead of mounting the key to do their own mTLS/minting. +//! +//! [`SocketTokenSource`] keeps a cached token fresh with a background task and +//! serves it synchronously from [`NodeTokenProvider::current`] on the request +//! path. Until the first successful fetch (e.g. the agent hasn't started or +//! registered yet), requests simply go out without a bearer header. + +use std::sync::{Arc, RwLock, Weak}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use hyper_util::rt::TokioIo; +use tokio::net::UnixStream; +use tonic::transport::{Channel, Endpoint, Uri}; +use tower::service_fn; + +use crate::node_jwt::NodeTokenProvider; +use crate::protos::agent_local::GetNodeTokenRequest; +use crate::protos::agent_local::agent_local_client::AgentLocalClient; + +/// Default path of the dpu-agent's local API socket. It lives in a dedicated +/// `run/` subdirectory of the shared `/opt/forge` credentials directory so a +/// containerized consumer can mount just that subdirectory read-write — +/// `connect(2)` needs write access to the socket inode — while the +/// credentials themselves stay on a read-only mount. +pub const DEFAULT_AGENT_LOCAL_SOCKET: &str = "/opt/forge/run/agent.sock"; + +/// The background task re-fetches when less than this long remains on the +/// cached token. The request path keeps serving the cached token down to +/// HALF this margin — the refresher normally replaces it well before that — +/// so a briefly-late refresh doesn't strip requests of their header. +const REFRESH_MARGIN_SECS: u64 = 60; + +/// Delay between fetch attempts while the agent socket is absent or erroring, +/// and the per-attempt deadline for a fetch (connect + RPC) so a stalled +/// handshake can't wedge the refresh loop. +const RETRY_DELAY: Duration = Duration::from_secs(5); + +pub struct SocketTokenSource { + socket_path: String, + cached: RwLock>, +} + +/// Manual impl so the cached bearer token (a live credential) never lands in +/// debug output of the client config, which is itself `Debug`-logged. Mirrors +/// the redaction on [`NodeJwtMinter`](crate::node_jwt::NodeJwtMinter). +impl std::fmt::Debug for SocketTokenSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SocketTokenSource") + .field("socket_path", &self.socket_path) + .finish_non_exhaustive() + } +} + +impl SocketTokenSource { + /// Creates the source and spawns its refresh loop on the current tokio + /// runtime. The loop holds only a weak reference, so dropping the last + /// `Arc` (and the clients built from it) shuts the loop down. + #[must_use] + pub fn spawn(socket_path: String) -> Arc { + let source = Arc::new(Self { + socket_path, + cached: RwLock::new(None), + }); + tokio::spawn(refresh_loop(Arc::downgrade(&source))); + source + } + + async fn fetch(socket_path: &str) -> Result<(String, u64), tonic::Status> { + // Deadline over connect + RPC together: a peer that accepts the + // connection but never answers must fail the attempt, not wedge the + // refresh loop forever. + tokio::time::timeout(RETRY_DELAY, async { + let channel = connect_uds(socket_path) + .await + .map_err(|e| tonic::Status::unavailable(format!("agent local socket: {e}")))?; + let response = AgentLocalClient::new(channel) + .get_node_token(GetNodeTokenRequest {}) + .await? + .into_inner(); + Ok((response.token, response.expires_at)) + }) + .await + .map_err(|_| tonic::Status::deadline_exceeded("agent local API fetch timed out"))? + } +} + +impl NodeTokenProvider for SocketTokenSource { + fn current(&self) -> Option { + let now = unix_now()?; + self.cached + .read() + .ok()? + .as_ref() + .filter(|(_, expires_at)| *expires_at > now + REFRESH_MARGIN_SECS / 2) + .map(|(token, _)| token.clone()) + } +} + +async fn connect_uds(socket_path: &str) -> Result { + let socket_path = socket_path.to_owned(); + // The URI is required by the Endpoint API but unused for UDS connections. + Endpoint::try_from("http://[::]:50051")? + .connect_with_connector(service_fn(move |_: Uri| { + let path = socket_path.clone(); + async move { + let stream = UnixStream::connect(path).await?; + Ok::<_, std::io::Error>(TokioIo::new(stream)) + } + })) + .await +} + +async fn refresh_loop(source: Weak) { + loop { + // Re-upgrade each iteration so the loop exits once every user of the + // source is gone, instead of pinning it alive forever. + let Some(source) = source.upgrade() else { + return; + }; + let socket_path = source.socket_path.clone(); + match SocketTokenSource::fetch(&socket_path).await { + Ok((token, expires_at)) => { + if let Ok(mut guard) = source.cached.write() { + *guard = Some((token, expires_at)); + } + drop(source); + let sleep_secs = unix_now() + .map(|now| { + expires_at + .saturating_sub(now) + .saturating_sub(REFRESH_MARGIN_SECS) + }) + .unwrap_or(0) + .max(RETRY_DELAY.as_secs()); + tokio::time::sleep(Duration::from_secs(sleep_secs)).await; + } + Err(status) => { + tracing::debug!( + target: "node_auth", + socket = %socket_path, + %status, + "node-auth: could not fetch node token from agent local API; will retry" + ); + drop(source); + tokio::time::sleep(RETRY_DELAY).await; + } + } + } +} + +fn unix_now() -> Option { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .ok() +} + +#[cfg(test)] +mod tests { + use tokio_stream::wrappers::UnixListenerStream; + use tonic::{Request, Response, Status}; + + use super::*; + use crate::protos::agent_local::GetNodeTokenResponse; + use crate::protos::agent_local::agent_local_server::{AgentLocal, AgentLocalServer}; + + struct FixedToken(String, u64); + + #[tonic::async_trait] + impl AgentLocal for FixedToken { + async fn get_node_token( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(GetNodeTokenResponse { + token: self.0.clone(), + expires_at: self.1, + })) + } + } + + fn serve(socket: &std::path::Path, token: &str, expires_at: u64) { + let listener = tokio::net::UnixListener::bind(socket).expect("bind uds"); + let service = AgentLocalServer::new(FixedToken(token.to_string(), expires_at)); + tokio::spawn( + tonic::transport::Server::builder() + .add_service(service) + .serve_with_incoming(UnixListenerStream::new(listener)), + ); + } + + async fn wait_for_token(source: &SocketTokenSource) -> Option { + for _ in 0..100 { + if let Some(token) = source.current() { + return Some(token); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + None + } + + #[tokio::test] + async fn fetches_token_from_agent_socket() { + let dir = tempfile::tempdir().expect("tempdir"); + let socket = dir.path().join("agent.sock"); + let expires_at = unix_now().expect("clock") + 300; + serve(&socket, "the.node.token", expires_at); + + let source = SocketTokenSource::spawn(socket.to_string_lossy().into_owned()); + assert_eq!( + wait_for_token(&source).await.as_deref(), + Some("the.node.token") + ); + } + + #[tokio::test] + async fn missing_socket_yields_none_and_no_panic() { + let source = SocketTokenSource::spawn("/nonexistent/agent.sock".to_string()); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!(source.current().is_none()); + } + + #[tokio::test] + async fn debug_output_redacts_the_cached_token() { + let dir = tempfile::tempdir().expect("tempdir"); + let socket = dir.path().join("agent.sock"); + serve( + &socket, + "super.secret.token", + unix_now().expect("clock") + 300, + ); + + let source = SocketTokenSource::spawn(socket.to_string_lossy().into_owned()); + wait_for_token(&source).await.expect("token cached"); + + // ForgeClientConfig is Debug-logged, so a cached token must never + // appear in the provider's debug output. + let rendered = format!("{source:?}"); + assert!( + !rendered.contains("super.secret.token"), + "cached token leaked into Debug: {rendered}" + ); + } + + #[tokio::test] + async fn expired_cached_token_is_not_served() { + let dir = tempfile::tempdir().expect("tempdir"); + let socket = dir.path().join("agent.sock"); + // The agent hands out a token that is already (nearly) expired. + serve(&socket, "stale.token", unix_now().expect("clock")); + + let source = SocketTokenSource::spawn(socket.to_string_lossy().into_owned()); + tokio::time::sleep(Duration::from_millis(200)).await; + assert!( + source.current().is_none(), + "a token inside the refresh margin must not be presented" + ); + } +} diff --git a/crates/rpc/src/protos/mod.rs b/crates/rpc/src/protos/mod.rs index c187ab59bb..6c266dbfc2 100644 --- a/crates/rpc/src/protos/mod.rs +++ b/crates/rpc/src/protos/mod.rs @@ -106,6 +106,12 @@ pub mod fmds { include!(concat!(env!("OUT_DIR"), "/fmds.rs")); } +#[allow(non_snake_case, unknown_lints, clippy::all)] +#[rustfmt::skip] +pub mod agent_local { + include!(concat!(env!("OUT_DIR"), "/agent_local.rs")); +} + #[allow(clippy::all, deprecated)] #[rustfmt::skip] pub mod forge_api_client { diff --git a/crates/scout/src/cfg/command_line.rs b/crates/scout/src/cfg/command_line.rs index 686c9524bd..f3739a58d7 100644 --- a/crates/scout/src/cfg/command_line.rs +++ b/crates/scout/src/cfg/command_line.rs @@ -36,6 +36,16 @@ impl std::fmt::Display for Mode { } } } +/// Rejects an empty or whitespace-only audience at parse time. A blank value +/// would otherwise reach the minter and produce tokens the API cannot match, +/// with nothing in the logs pointing at the flag. +fn non_blank_audience(value: &str) -> Result { + if value.trim().is_empty() { + return Err("node-auth audience must not be empty".to_string()); + } + Ok(value.to_string()) +} + #[derive(Clone, Parser)] #[clap(name = env!("CARGO_BIN_NAME"))] pub(crate) struct Options { @@ -84,6 +94,14 @@ pub(crate) struct Options { )] pub client_key: String, + #[clap( + long, + value_parser = non_blank_audience, + help = "Audience claim stamped on node-auth bearer JWTs; must match the API's [node_auth] audience", + default_value_t = ::rpc::node_jwt::NODE_JWT_AUDIENCE.to_string(), + )] + pub node_auth_audience: String, + // Combined with discovery_retries_max, the default of 60 // seconds worth of discovery_retry_secs provides for 1 // week worth of minutely retries. diff --git a/crates/scout/src/client.rs b/crates/scout/src/client.rs index 437dc215ce..833e5f22e7 100644 --- a/crates/scout/src/client.rs +++ b/crates/scout/src/client.rs @@ -30,7 +30,10 @@ pub(crate) async fn create_forge_client( cert_path: config.client_cert.clone(), key_path: config.client_key.clone(), }), - ); + ) + // Node-auth (#355): also present a self-signed bearer JWT minted from the + // client cert's key. Ignored by the API unless [node_auth] is enabled. + .with_node_jwt(config.node_auth_audience.clone()); let api_config = ApiConfig::new(&config.api, &client_config); let client = forge_tls_client::ForgeTlsClient::retry_build(&api_config) diff --git a/deploy/nico-base/api/config-files/nico-api-config.toml b/deploy/nico-base/api/config-files/nico-api-config.toml index 6b096ede25..ae2f92408e 100644 --- a/deploy/nico-base/api/config-files/nico-api-config.toml +++ b/deploy/nico-base/api/config-files/nico-api-config.toml @@ -43,6 +43,24 @@ hardware_health_reports = "MonitorOnly" enabled = true create_machines = true +[node_auth] +# Node (Scout / DPU-agent) authentication to the API. +# See docs/design/machine-identity/node-auth-jwt.md. +# Bearer JWTs: off by default. When enabled, the API accepts short-lived +# `Authorization: Bearer` tokens that nodes self-sign with their existing +# mTLS client-certificate key (the cert rides in the token's x5c header and +# is verified against the same root CA as the TLS listener). +enabled = false +# Machine mTLS client-certificate authentication: on by default. Disable only +# after the fleet presents bearer tokens (requires enabled = true; the API +# refuses to start with both mechanisms off). Service/admin-CLI certs are +# unaffected. +mtls_enabled = true +# `aud` claim required on presented tokens. +#audience = "nico-api" +# Maximum accepted token lifetime in seconds (nodes mint 300 s tokens). +#max_token_ttl_sec = 900 + [firmware_global] autoupdate = true diff --git a/docs/design/machine-identity/node-auth-jwt.md b/docs/design/machine-identity/node-auth-jwt.md new file mode 100644 index 0000000000..1bfbbea860 --- /dev/null +++ b/docs/design/machine-identity/node-auth-jwt.md @@ -0,0 +1,439 @@ +# Node-Auth: Self-Signed Bearer JWTs for Scout and DPU-Agent + +Design for [#355](https://github.com/NVIDIA/infra-controller/issues/355) +(sub-issue of the Vault-elimination epic +[#195](https://github.com/NVIDIA/infra-controller/issues/195)): Scout and the +DPU-agent authenticate to the API with short-lived bearer JWTs alongside — +and eventually instead of — mTLS client certificates. + +Nodes sign their own tokens with the private key of their **existing** mTLS +client certificate. There is no new key material anywhere, no server-side +signing key or key storage, no issuance or refresh RPCs, and no dependency on +the `machine_identity` (tenant JWT-SVID) subsystem. On a DPU only the +dpu-agent ever holds that key; co-located NICo pods get finished tokens from +it over a local unix socket (see [Key distribution on DPF](#key-distribution-on-dpf-the-agent-is-the-token-broker)). +Approaches that were weighed and rejected are recorded under +[Designs not used](#designs-not-used). + +It is distinct from the tenant-facing [SPIFFE JWT-SVID design](spiffe-svid-sdd.md): +that issues identity tokens *to tenant workloads* via IMDS; this design covers +how *NICo's own node agents* authenticate to the NICo API. + +## How it works + +```text +node (scout / dpu-agent) nico-api +------------------------ -------- +has /opt/forge/machine_cert.pem ── x5c ──► 1. verify x5c chain against the +and /opt/forge/machine_cert.key client-cert root CA (same roots + as the TLS listener) +mint ES256 JWT signed with the 2. verify JWT signature with the +cert's own key, 5-min TTL, verified leaf's public key +cert chain in the x5c header 3. enforce exp / iat / aud, bounded + lifetime +attach as Authorization: Bearer 4. SPIFFE-validate the leaf; map its +on every gRPC request URI SAN through the same + SpiffeContext as mTLS certs + ⇒ identical machine principal, RBAC + unchanged +``` + +Key-less co-located services (fmds) never touch the cert or key at all: they +ask the dpu-agent for its current token over a unix socket and attach that +same header. + +```text +fmds pod dpu-agent nico-api +-------- --------- -------- +no key, no cert ──socket──► holds the key, ── same bearer token ──► +GetNodeToken mints as above (identical +(cached, refreshed verification) + in the background) +``` + +## Auth flow: new DPU → first authorized gRPC call + +**Provisioning & bootstrap (no credentials yet)** + +- A new DPU is provisioned (BFB installed via DPF); the `forge-dpu-agent` DPF + service starts on it with only two auth-relevant inputs from its config: the + API endpoint and the root CA bundle (`forge_system.root_ca`). The cert/key + files at `/opt/forge/machine_cert.pem` / `machine_cert.key` don't exist yet. +- The agent opens a TLS connection to `nico-api` that is **server-auth only** + — it verifies the API's cert against the root CA but presents no client + credential (the API listener uses `allow_unauthenticated()`, so the + connection is accepted with an anonymous principal). + +**First credential: the machine certificate** + +- The agent calls `DiscoverMachine` + (`host-support/registration.rs::register_machine`) carrying its hardware + enumeration (`DiscoveryInfo`); this RPC is reachable pre-credential by + design. +- The API registers/matches the machine and returns a `machine_certificate` + in the response: a Vault-PKI-issued EC P-256 leaf whose SAN is the machine's + SPIFFE URI (`spiffe:////machine/`), plus the + issuing CA and private key. (On attestation-enabled sites, hosts get this + via `AttestQuote` after a TPM challenge instead; DPUs take the discovery + path.) +- `write_certs` persists leaf+issuing-CA to `/opt/forge/machine_cert.pem` and + the key to `/opt/forge/machine_cert.key`. **This existing cert key is the + JWT signing key — nothing else is ever created.** + +**Minting the JWT (client side, `rpc::node_jwt`)** + +- The agent's gRPC client was built with + `ForgeClientConfig::new(root_ca, ClientCert{...}).with_node_jwt()`, so a + `NodeJwtMinter` watches those two file paths. +- On the next outgoing RPC, the minter reads cert+key from disk (re-encoding + Vault's SEC1 key PEM to PKCS#8) and signs an **ES256 JWT**: header + `{alg: ES256, x5c: [leaf, issuing CA]}`, claims + `{sub: , aud: , + iat: now, exp: now+300s}`. The token is cached and re-minted when < 60 s + remain — no refresh RPC, and cert renewal is picked up automatically + because the files are re-read at each mint. +- Renewal writes the certificate and the key in two separate operations, so + the minter compares the key's public half against the one certified by the + leaf before signing. A mismatch (a mint landing between the two writes) + fails the mint rather than caching a token the API will reject. +- `BearerAuthService` stamps `Authorization: Bearer ` onto the request. + (The TLS channel may still present the client cert too — dual-support; each + is sufficient alone.) + +**Validation (server side, requires `[node_auth] enabled = true` + TLS listener)** + +- The authn middleware sees the Bearer header and hands it to + `NodeJwtValidator`, which checks in order: + - header `alg` is exactly ES256 (no algorithm substitution); + - the `x5c` chain verifies against the **same root CA file the TLS listener + uses for client certs** (`[tls] root_cafile_path`) — path building, + validity window, client-auth EKU; + - the JWT signature verifies with the *verified leaf's* public key; + - claims: `exp`/`iat`/`aud` enforced, and lifetime bounded by + `max_token_ttl_sec` (default 900 s) so a client can't stretch `exp`; + - the leaf passes SPIFFE validation (leaf-only, single URI SAN), and `sub` + must equal that SAN — identity comes from the verified cert, never from + the claim. + +**Authorization & the first call** + +- The validated SPIFFE URI is mapped through the same `SpiffeContext` as mTLS + certs → a `SpiffeMachineIdentifier` principal, byte-identical to what the + cert path would have minted. +- Casbin RBAC evaluates that principal exactly as before (machine class → + Agent/Scout role rules) — **no RBAC changes** — and the handler executes. + That is the first authorized gRPC call on bearer auth. + +Note: the very first *authorized* call after discovery could ride either +credential, since the agent holds both from the same moment — the JWT only +becomes load-bearing once `mtls_enabled = false`. + +## Q1 — How does the agent get a private key that is trusted to create the JWT? + +**It already has one.** The node's Vault-issued mTLS client certificate key +(`/opt/forge/machine_cert.key`, EC P-256 — Vault PKI role `key_type=ec, +key_bits=256`) signs the JWT, and the certificate itself rides along in the +token's `x5c` header (RFC 7515 §4.1.6). The key is trusted because the +certificate chains to the root CA the API already trusts for client certs — +the JWT is effectively "mTLS at the application layer". + +Bootstrapping is unchanged: a machine obtains its first certificate through +the existing discovery/attestation flow (`DiscoverMachine` / `AttestQuote` +respond with the machine certificate — see the auth-flow walkthrough above), +and from that moment it can mint tokens. Minting is best-effort: before the +cert exists, requests simply carry no bearer header. + +## Q2 — JWT side by side with mTLS + +The authn middleware (`CertDescriptionMiddleware` in `crates/authn`) mints +principals from **both** sources on every request: the TLS-layer client cert, +and the `Authorization: Bearer` token (validated by `NodeJwtValidator`). Both +paths converge on the same SPIFFE URI → `SpiffeMachineIdentifier` mapping +through the same `SpiffeContext`, so RBAC (Casbin policy, role mapping) is +completely unchanged. A node presenting both credentials gets the same +principal twice — harmless. Clients attach the bearer token unconditionally +(`ForgeClientConfig::with_node_jwt()` in scout and dpu-agent); a server with +node-auth disabled ignores the header, so rollout order doesn't matter. + +## Q3 — JWT off by default, configured in the API config + +```toml +[node_auth] +enabled = false # master switch for accepting bearer JWTs +audience = "nico-api" # `aud` required on presented tokens +max_token_ttl_sec = 900 # upper bound on client-chosen lifetimes (cap 86400) +``` + +When `enabled = true`, startup requires a TLS listener (bearer tokens are +never accepted over plaintext) and a readable `[tls] root_cafile_path`; +missing prerequisites fail startup rather than silently degrading. The whole +preflight runs *before* DPF resource creation, so a misconfiguration can't +mutate cluster state on its way to failing. + +`audience` is the one value that must be set on **both** ends: the API +validates `aud` against it, and each node stamps it. Nodes take it from their +own config — scout via `--node-auth-audience`, the agent via `[forge-system] +node-auth-audience` (or the `--node-auth-audience` flag, which is how DPF +deploys it, since containerized agents run with no config file and the API +templates the value from its own `[node_auth] audience`). A site that changes +one end and not the other has every token rejected. + +## Q4 — mTLS on by default, disableable in the API config + +```toml +[node_auth] +mtls_enabled = true +``` + +When `mtls_enabled = false`, the middleware stops minting machine principals +from client certificates — bearer JWTs become the only node auth path. The +gate is scoped to **machine** certs: service and admin-CLI certs on the same +listener are unaffected. `enabled = false` + `mtls_enabled = false` is +rejected at startup (node lockout). + +Note the trust chain is still the certificate PKI: disabling mTLS here +disables the *transport-layer* cert authentication, not cert issuance. Nodes +must keep renewing certificates because the JWT is signed by the cert key. + +## Q5 — Key regeneration and public-key exchange + +**Regeneration** is the existing client-certificate renewal: when +`ClientCertRenewer` rotates the cert/key files, the minter picks the new pair +up on its next re-mint (it re-reads both files from disk each time). No +coordination, no state. + +**Public-key exchange: the x5c header.** Every token carries the certificate +chain that vouches for its signing key, and the API verifies that chain +against the root CA bundle it already holds (`[tls] root_cafile_path`). There +is no JWKS endpoint, no key registry, and no key distribution problem — CA +rotation is handled wherever the root bundle is handled today. + +**CA rotation** moves both consumers of that bundle together. The TLS +listener already re-reads the file every five minutes for cert-manager +rotations; the validator's trust anchors are held behind a lock and refreshed +on that same tick, so a token chaining to a freshly rotated CA is accepted +without an API restart. A failed reload keeps the previous anchors — a bundle +caught mid-write must not disarm node auth. + +**Compromise response** is likewise the PKI's: a stolen key/cert pair is the +same incident as a stolen mTLS cert today. Tokens age out in minutes +(`exp - iat ≤ max_token_ttl_sec`, client mints 5-minute tokens), and that +expiry — together with chain and EKU validation — is what actually bounds a +stolen key. Revocation does not: the validator calls +`allow_unknown_revocation_status()`, so a holder of the private key keeps +minting acceptable tokens until the certificate itself expires. Cutting a +compromised key off sooner needs revocation checking we do not do yet. + +## Q6 — JWT best-practice checklist + +| Practice | How it's honored | +| --- | --- | +| Asymmetric signing, no `alg` confusion | ES256 only; both the header check and `Validation` pin the algorithm, so `none`/HS256 substitution is rejected. | +| Identity never comes from claims | The principal derives from the **chain-verified certificate's SPIFFE SAN**; `sub` is only cross-checked against it. A forged `sub` buys nothing. | +| Short-lived tokens | Clients mint 300 s tokens; the server enforces `exp - iat` and `exp - now` ≤ `max_token_ttl_sec` (default 900 s, hard cap 86400), so a client cannot stretch `exp`. | +| `exp` / `iat` / `aud` enforced | Required claims; validated by `jsonwebtoken` plus the bounded-lifetime check. | +| Chain validation, not pinning | `x5c` verified with rustls `WebPkiClientVerifier` (path building, validity window, client-auth EKU) against the same roots as the TLS listener. | +| SPIFFE leaf constraints | `carbide_authn::validate_x509_certificate` re-checks leaf-ness, key usage, and the single-URI-SAN rule — same code path as mTLS certs. | +| No bearer tokens over plaintext | Enforced at both ends. Server: startup refuses `enabled = true` on a non-TLS listener, the middleware only installs the validator when the listener is TLS-terminated, and a failed acceptor rebuild keeps the previous acceptor rather than falling back to cleartext. Client: `with_token_provider` implies `require_tls_enforcement`, and building a token client against a non-HTTPS URL is an error. | +| No key material at rest beyond the PKI | The server holds no signing key; the client holds only what it already had, and `write_certs` persists the key file owner-only (0600). Credentials never in logs (both token caches have a redacting `Debug`). | +| Least exposure for the signing key | On a DPU the key stays in the dpu-agent alone; consumers get finished short-lived tokens over a local socket and never mount the credentials directory. | + +## Component map + +| Piece | Where | +| --- | --- | +| Client mint + cache + header injection | `crates/rpc/src/node_jwt.rs` (`NodeJwtMinter`, `BearerAuthService`, `NodeTokenProvider`) | +| Client opt-in | `ForgeClientConfig::with_node_jwt()` / `with_token_provider()` (`crates/rpc/src/forge_tls_client.rs`); called in scout `client.rs` and dpu-agent `lib.rs` | +| Key-less token source | `crates/rpc/src/node_token_socket.rs` (`SocketTokenSource`), consumed by fmds via `--node-token-socket` | +| Broker service | `AgentLocal/GetNodeToken` in `crates/rpc/proto/agent_local.proto`, served by `crates/agent/src/local_api.rs` | +| Server validation | `crates/api-core/src/node_auth.rs` (`NodeJwtValidator`) | +| Config | `NodeAuthConfig` in `crates/api-core/src/cfg/file.rs` (`[node_auth]`); node side in `crates/host-support/src/agent_config.rs` and the two `command_line.rs` | +| Middleware hook | `BearerTokenAuthenticator` trait + machine-cert gate in `crates/authn/src/middleware.rs` | +| Wiring | `crates/api-core/src/setup.rs` (preflight, validator construction), `listener.rs` (middleware install, trust-anchor reload), `dpf_services.rs` (fmds token mode) | +| Charts | `bluefield/charts/nico-fmds` (`useNodeTokens`), `bluefield/charts/nico-dpu-agent` | + +All of it logs under one target: `RUST_LOG=node_auth=debug` turns on the +whole feature's tracing, and every message carries a `node-auth:` prefix. + +## Key distribution on DPF: the agent is the token broker + +The machine cert/key live at `/opt/forge` on the DPU, a hostPath directory +several NICo pods mount. Rather than share the key with each of them, the +dpu-agent is the only holder and hands out finished tokens. + +- **The socket.** The agent serves `AgentLocal/GetNodeToken` on a unix socket + at `/opt/forge/run/agent.sock`, mode 0600 — a dedicated `run/` subdirectory + so a consumer can mount just that path read-write (`connect(2)` needs write + access to the socket inode) while credentials stay elsewhere. `bind` creates + the socket at umask permissions and the 0600 chmod only lands afterwards, so + the directory — not the socket — is what closes that window; the agent + therefore requires the socket's parent to be dedicated, creating it `0700` or + refusing a directory that holds anything else. (Without that rule, + `local-api-socket = /run/agent.sock` would take `/run` to `0700` and lock + every non-root service on the box out of its runtime files.) The path is + configurable via `[forge-system] local-api-socket`; the server retries + forever rather than dying once, since a bare-metal boot can start the agent + before its directory exists. This socket is the consolidation point for + future agent ↔ co-located-service traffic, in place of new sockets, ports, + or file drops. +- **The consumer.** `SocketTokenSource` implements the same + `NodeTokenProvider` trait as the minter, so a client built with + `with_token_provider()` is indistinguishable on the wire from one that + minted its own. A background task keeps the cached token fresh (refresh at + 60 s remaining, request-path cutoff at half that, per-attempt deadline + covering connect + RPC); the request path never blocks, and before the + first successful fetch requests simply carry no bearer header. +- **The trust anchor, without the key.** fmds still needs the root CA, so the + agent mirrors it to `/pub/`, a directory holding nothing else. + Token-mode fmds pods mount `pub/` plus `run/` and **do not reference the + credentials volume at all**, not even in the init container — the machine + key is absent from the pod rather than merely read-only, which matters + because the container runs as UID 0. It is a directory mount, not a + `subPath`, so the atomic-rename CA replacement still propagates into a + running pod. The agent republishes on every start as well as from the init + container, so a CA replaced out of band propagates too. +- **The switch.** fmds helm values are rendered by the API, so `useNodeTokens` + is derived from the one setting that makes tokens meaningful: `[node_auth] + enabled`. With node-auth off, the chart renders exactly as before. Rollout + note: enabling it requires an agent image that serves the socket, so + agent and fmds images must be deployed together. + +Scope: this covers *authentication to nico-api*. otelcol also uses the machine +cert for TLS client auth to its OTLP gateway, which a nico-api bearer token +cannot replace, so the key only fully disappears from other pods once that +ingest path has its own credential story. + +## Design decisions (resolved questions) + +1. **How does the server learn the public key?** → from the token itself + (`x5c`), verified against the existing root CA — the one key-distribution + mechanism the system already operates. +2. **Why not drop mTLS immediately, since the JWT proves the same key?** → + dual-support de-risks rollout and keeps requirement 4 orthogonal; and the + JWT still depends on the cert PKI, so cert issuance must outlive transport + mTLS. +3. **Client-side enable knob?** → none. Nodes always mint when they hold a + cert; a disabled server ignores the header (verified by middleware test). + One switch (`[node_auth] enabled`) controls the feature. The audience is + the sole value that must agree on both ends. +4. **Replay window** → a captured token is replayable for ≤ 5 minutes against + the same API over TLS only. Accepted; `jti`/nonce tracking or DPoP-style + proof-of-possession is the hardening path if needed. +5. **RSA machine certs** → not supported (Vault PKI role is EC P-256 + everywhere); the validator rejects non-EC leaves with a clear debug reason. + +## Known issues + +**Disabling `[node_auth]` after fmds has been in token mode is unsequenced.** +The two halves of the switch take effect on different clocks: + +- The API stops accepting bearer tokens *immediately*, when it restarts with + `enabled = false`. +- fmds returns to cert mode only *eventually*. The re-apply is real — + `create_initialization_objects` upserts the DPUServiceConfiguration by + forced server-side apply, so the new `useNodeTokens: false` does land — but + DPF then has to re-render and roll the DaemonSet across the fleet. + +In between, fmds pods are still running token mode with no client cert and +tokens the API now rejects: `phone_home` and machine-identity signing take +401s until the roll reaches each node. Config serving and instance metadata +are unaffected, and it self-heals once the roll completes. + +Nothing is corrupted by the transition. The agent writes `machine_cert.pem` / +`.key` and the base-path root CA unconditionally, in both modes, so cert mode +works the moment a pod rolls; leftover `pub/` and `run/` directories are +inert. + +There is no way to stage it today, because `useNodeTokens` is derived from +`[node_auth] enabled` alone. The safe order — move fmds off tokens, confirm +the roll landed, *then* stop accepting tokens at the API — would need an +explicit per-service override defaulting to the derived value. The enable +direction has the mirror window but fails safe: the init container blocks +waiting for the CA in `pub/` rather than starting into a broken state. + +## Designs not used + +**Server-issued tokens.** A site-level signing key in the credential store, a +`RefreshNodeToken` RPC, and DPU device identity as the refresh anchor. It +brings back everything this design deletes: a server-side key to store, +rotate, and share across HA replicas; issuance and refresh RPCs to build, +version, and rate-limit; and a bootstrap question of its own (what +credential authorizes the *first* refresh). Reusing the node's existing +client-cert key gets the same authenticated principal with no new secret +anywhere. Worth revisiting only if per-node keys are removed from the +architecture, at which point nothing is left to self-sign with. + +**A JWKS endpoint or key registry.** The obvious way to publish per-node +public keys, and unnecessary: `x5c` carries the certificate with every token +and the API already holds the root CA that vouches for it. A registry would +add a distribution channel that can go stale, be unreachable, or disagree +with the PKI — and CA rotation would have to be handled in two places instead +of one. + +**Reusing the tenant JWT-SVID subsystem.** `machine_identity` issues SPIFFE +JWT-SVIDs to *tenant workloads* via IMDS. Node agents authenticating to the +NICo API is a different problem with a different trust root, and coupling +them would make NICo's own control plane depend on a tenant-facing service +being healthy. + +**Kubernetes Secret for the machine key.** Secret mounts are tmpfs, so the +key stops touching DPU flash, and sharing becomes explicit, per-pod, and +auditable through the K8s API. But in DPF the DPU-cluster control plane +(kamaji-hosted etcd) runs on the x86 management cluster, so the key gains a +durable copy — plus backups — *off the DPU*, requiring etcd encryption at +rest to be guaranteed; the agent needs new K8s API rights to create and +update the Secret; and scout (pre-DPF, live-image) can't use this path at +all, so the file mechanism would survive alongside it. The token broker +addresses the same concern — key exposure surface — without moving the key +anywhere, so this became moot rather than merely deferred. + +**Sharing the key by hostPath, read-only.** What token mode replaced. A +read-only mount of the credentials directory stops nothing: the container +runs as UID 0 and can read `machine_cert.key` regardless. Whether the mount +is read-only is not the control; whether the file is in the pod's namespace +at all is. + +**Mounting just the CA file by `subPath`.** A tempting one-line fix for the +above — mount the single file rather than the directory. `subPath` +bind-mounts an inode, and `install_bootstrap_ca` replaces the CA by atomic +rename, so a running pod would keep the old anchor forever and silently fail +after a CA rotation. Hence the dedicated `pub/` directory. + +**A network endpoint for the broker.** There is precedent for it — the agent +already dials `nico-dhcp-server` over gRPC through a k8s `Service` — but the +token broker is a different kind of service, and a port is the wrong shape +for it: + +- *The socket is the authorization check.* `GetNodeToken` ignores its request + entirely: there is no authentication in the handler, because anyone who can + connect is already on the node with the path mounted. On a TCP port that + same RPC hands a full machine identity to every pod in the DPU cluster, and + to anything that can route to the DPU. +- *TLS can't fix that.* Protecting the endpoint means mTLS with the machine + cert — which the consumer does not have, that being the entire premise. The + transport cannot be secured by the credential it exists to distribute, so a + port needs a second credential system to bootstrap the first. The SPIFFE + Workload API is a unix socket for this reason. +- *Locality.* Agent and consumer are DaemonSets on the same node; the call is + same-node by construction. A `Service` routes it through cluster DNS and a + VIP whose control plane (kamaji-hosted etcd) runs on the x86 management + cluster — making "can this node authenticate at all" depend on remote + infrastructure. A socket is a path. +- *No listening port* to firewall or expose by accident, which matters on a + DPU where `nico-otelcol` runs with `hostNetwork: true`. + +The cost of the choice is real: `connect(2)` needs write access to the socket +inode, which is why the socket sits in its own `run/` subdirectory on a +read-write mount. And it serves co-located consumers only — anything off-DPU +needing a node token is not covered by this design. + +Note what mode 0600 does and does not buy. Everything on the DPU runs as +root, so the UID check separates nobody; the real control is which pods mount +the directory. The gain over sharing the key is the blast radius of a leak — +a five-minute token instead of a long-lived private key — not a hard +boundary. + +**Certificate revocation checking.** Not implemented — see Q5. Token expiry +and the certificate's own lifetime are what bound a compromised key; if the +incident model needs a faster cutoff than that, revocation is the work, and +this design does not do it. diff --git a/helm/charts/nico-api/files/carbide-api-config.toml b/helm/charts/nico-api/files/carbide-api-config.toml index 22391afc63..626aa2ff6d 100644 --- a/helm/charts/nico-api/files/carbide-api-config.toml +++ b/helm/charts/nico-api/files/carbide-api-config.toml @@ -41,6 +41,24 @@ hardware_health_reports = "MonitorOnly" enabled = true create_machines = true +[node_auth] +# Node (Scout / DPU-agent) authentication to the API. +# See docs/design/machine-identity/node-auth-jwt.md. +# Bearer JWTs: off by default. When enabled, the API accepts short-lived +# `Authorization: Bearer` tokens that nodes self-sign with their existing +# mTLS client-certificate key (the cert rides in the token's x5c header and +# is verified against the same root CA as the TLS listener). +enabled = false +# Machine mTLS client-certificate authentication: on by default. Disable only +# after the fleet presents bearer tokens (requires enabled = true; the API +# refuses to start with both mechanisms off). Service/admin-CLI certs are +# unaffected. +mtls_enabled = true +# `aud` claim required on presented tokens. +#audience = "nico-api" +# Maximum accepted token lifetime in seconds (nodes mint 300 s tokens). +#max_token_ttl_sec = 900 + [firmware_global] autoupdate = true diff --git a/rest-api/proto/core/gen/v1/agent_local_nico.pb.go b/rest-api/proto/core/gen/v1/agent_local_nico.pb.go new file mode 100644 index 0000000000..17ed48ed4b --- /dev/null +++ b/rest-api/proto/core/gen/v1/agent_local_nico.pb.go @@ -0,0 +1,180 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: agent_local_nico.proto + +package core + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type GetNodeTokenRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetNodeTokenRequest) Reset() { + *x = GetNodeTokenRequest{} + mi := &file_agent_local_nico_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetNodeTokenRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetNodeTokenRequest) ProtoMessage() {} + +func (x *GetNodeTokenRequest) ProtoReflect() protoreflect.Message { + mi := &file_agent_local_nico_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetNodeTokenRequest.ProtoReflect.Descriptor instead. +func (*GetNodeTokenRequest) Descriptor() ([]byte, []int) { + return file_agent_local_nico_proto_rawDescGZIP(), []int{0} +} + +type GetNodeTokenResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The signed ES256 JWT. + Token string `protobuf:"bytes,1,opt,name=token,proto3" json:"token,omitempty"` + // Unix seconds when the token expires. + ExpiresAt uint64 `protobuf:"varint,2,opt,name=expires_at,json=expiresAt,proto3" json:"expires_at,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetNodeTokenResponse) Reset() { + *x = GetNodeTokenResponse{} + mi := &file_agent_local_nico_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetNodeTokenResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetNodeTokenResponse) ProtoMessage() {} + +func (x *GetNodeTokenResponse) ProtoReflect() protoreflect.Message { + mi := &file_agent_local_nico_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetNodeTokenResponse.ProtoReflect.Descriptor instead. +func (*GetNodeTokenResponse) Descriptor() ([]byte, []int) { + return file_agent_local_nico_proto_rawDescGZIP(), []int{1} +} + +func (x *GetNodeTokenResponse) GetToken() string { + if x != nil { + return x.Token + } + return "" +} + +func (x *GetNodeTokenResponse) GetExpiresAt() uint64 { + if x != nil { + return x.ExpiresAt + } + return 0 +} + +var File_agent_local_nico_proto protoreflect.FileDescriptor + +const file_agent_local_nico_proto_rawDesc = "" + + "\n" + + "\x16agent_local_nico.proto\x12\vagent_local\"\x15\n" + + "\x13GetNodeTokenRequest\"K\n" + + "\x14GetNodeTokenResponse\x12\x14\n" + + "\x05token\x18\x01 \x01(\tR\x05token\x12\x1d\n" + + "\n" + + "expires_at\x18\x02 \x01(\x04R\texpiresAt2a\n" + + "\n" + + "AgentLocal\x12S\n" + + "\fGetNodeToken\x12 .agent_local.GetNodeTokenRequest\x1a!.agent_local.GetNodeTokenResponseB8Z6github.com/NVIDIA/infra-controller/rest-api/proto/coreb\x06proto3" + +var ( + file_agent_local_nico_proto_rawDescOnce sync.Once + file_agent_local_nico_proto_rawDescData []byte +) + +func file_agent_local_nico_proto_rawDescGZIP() []byte { + file_agent_local_nico_proto_rawDescOnce.Do(func() { + file_agent_local_nico_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_agent_local_nico_proto_rawDesc), len(file_agent_local_nico_proto_rawDesc))) + }) + return file_agent_local_nico_proto_rawDescData +} + +var file_agent_local_nico_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_agent_local_nico_proto_goTypes = []any{ + (*GetNodeTokenRequest)(nil), // 0: agent_local.GetNodeTokenRequest + (*GetNodeTokenResponse)(nil), // 1: agent_local.GetNodeTokenResponse +} +var file_agent_local_nico_proto_depIdxs = []int32{ + 0, // 0: agent_local.AgentLocal.GetNodeToken:input_type -> agent_local.GetNodeTokenRequest + 1, // 1: agent_local.AgentLocal.GetNodeToken:output_type -> agent_local.GetNodeTokenResponse + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_agent_local_nico_proto_init() } +func file_agent_local_nico_proto_init() { + if File_agent_local_nico_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_agent_local_nico_proto_rawDesc), len(file_agent_local_nico_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_agent_local_nico_proto_goTypes, + DependencyIndexes: file_agent_local_nico_proto_depIdxs, + MessageInfos: file_agent_local_nico_proto_msgTypes, + }.Build() + File_agent_local_nico_proto = out.File + file_agent_local_nico_proto_goTypes = nil + file_agent_local_nico_proto_depIdxs = nil +} diff --git a/rest-api/proto/core/gen/v1/agent_local_nico_grpc.pb.go b/rest-api/proto/core/gen/v1/agent_local_nico_grpc.pb.go new file mode 100644 index 0000000000..e6e7b43d03 --- /dev/null +++ b/rest-api/proto/core/gen/v1/agent_local_nico_grpc.pb.go @@ -0,0 +1,156 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.1 +// - protoc (unknown) +// source: agent_local_nico.proto + +package core + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + AgentLocal_GetNodeToken_FullMethodName = "/agent_local.AgentLocal/GetNodeToken" +) + +// AgentLocalClient is the client API for AgentLocal service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// Local (on-DPU) API served by nico-dpu-agent over a unix domain socket +// (the agent's `local-api-socket` setting, which defaults to a dedicated run/ +// subdirectory of the shared credentials directory, so consumers can mount the +// socket read-write while the credentials stay read-only). The literal default +// lives in DEFAULT_AGENT_LOCAL_SOCKET and is deliberately not repeated here: +// REST proto normalization rewrites vendor names inside comments, which would +// silently corrupt an inlined filesystem path into one that does not exist. +// This is the consolidation point for +// agent <-> co-located-service communication: services that today mount the +// machine cert/key (or synced files) to talk to nico-api can instead ask the +// agent, and future local needs should add RPCs here rather than new +// sockets, ports, or file drops. +type AgentLocalClient interface { + // Returns the current node-auth bearer JWT, minted by the agent from the + // machine's client-certificate key (issue #355). Callers present it as + // `Authorization: Bearer ` to nico-api and must fetch a fresh one + // before `expires_at`. Only the agent ever touches the private key. + GetNodeToken(ctx context.Context, in *GetNodeTokenRequest, opts ...grpc.CallOption) (*GetNodeTokenResponse, error) +} + +type agentLocalClient struct { + cc grpc.ClientConnInterface +} + +func NewAgentLocalClient(cc grpc.ClientConnInterface) AgentLocalClient { + return &agentLocalClient{cc} +} + +func (c *agentLocalClient) GetNodeToken(ctx context.Context, in *GetNodeTokenRequest, opts ...grpc.CallOption) (*GetNodeTokenResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetNodeTokenResponse) + err := c.cc.Invoke(ctx, AgentLocal_GetNodeToken_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// AgentLocalServer is the server API for AgentLocal service. +// All implementations should embed UnimplementedAgentLocalServer +// for forward compatibility. +// +// Local (on-DPU) API served by nico-dpu-agent over a unix domain socket +// (the agent's `local-api-socket` setting, which defaults to a dedicated run/ +// subdirectory of the shared credentials directory, so consumers can mount the +// socket read-write while the credentials stay read-only). The literal default +// lives in DEFAULT_AGENT_LOCAL_SOCKET and is deliberately not repeated here: +// REST proto normalization rewrites vendor names inside comments, which would +// silently corrupt an inlined filesystem path into one that does not exist. +// This is the consolidation point for +// agent <-> co-located-service communication: services that today mount the +// machine cert/key (or synced files) to talk to nico-api can instead ask the +// agent, and future local needs should add RPCs here rather than new +// sockets, ports, or file drops. +type AgentLocalServer interface { + // Returns the current node-auth bearer JWT, minted by the agent from the + // machine's client-certificate key (issue #355). Callers present it as + // `Authorization: Bearer ` to nico-api and must fetch a fresh one + // before `expires_at`. Only the agent ever touches the private key. + GetNodeToken(context.Context, *GetNodeTokenRequest) (*GetNodeTokenResponse, error) +} + +// UnimplementedAgentLocalServer should be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedAgentLocalServer struct{} + +func (UnimplementedAgentLocalServer) GetNodeToken(context.Context, *GetNodeTokenRequest) (*GetNodeTokenResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetNodeToken not implemented") +} +func (UnimplementedAgentLocalServer) testEmbeddedByValue() {} + +// UnsafeAgentLocalServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to AgentLocalServer will +// result in compilation errors. +type UnsafeAgentLocalServer interface { + mustEmbedUnimplementedAgentLocalServer() +} + +func RegisterAgentLocalServer(s grpc.ServiceRegistrar, srv AgentLocalServer) { + // If the following call panics, it indicates UnimplementedAgentLocalServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&AgentLocal_ServiceDesc, srv) +} + +func _AgentLocal_GetNodeToken_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetNodeTokenRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AgentLocalServer).GetNodeToken(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AgentLocal_GetNodeToken_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AgentLocalServer).GetNodeToken(ctx, req.(*GetNodeTokenRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// AgentLocal_ServiceDesc is the grpc.ServiceDesc for AgentLocal service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var AgentLocal_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "agent_local.AgentLocal", + HandlerType: (*AgentLocalServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetNodeToken", + Handler: _AgentLocal_GetNodeToken_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "agent_local_nico.proto", +} diff --git a/rest-api/proto/core/src/v1/agent_local_nico.proto b/rest-api/proto/core/src/v1/agent_local_nico.proto new file mode 100644 index 0000000000..4959b84471 --- /dev/null +++ b/rest-api/proto/core/src/v1/agent_local_nico.proto @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +syntax = "proto3"; + +package agent_local; + +option go_package = "github.com/NVIDIA/infra-controller/rest-api/proto/core"; + +// Local (on-DPU) API served by nico-dpu-agent over a unix domain socket +// (the agent's `local-api-socket` setting, which defaults to a dedicated run/ +// subdirectory of the shared credentials directory, so consumers can mount the +// socket read-write while the credentials stay read-only). The literal default +// lives in DEFAULT_AGENT_LOCAL_SOCKET and is deliberately not repeated here: +// REST proto normalization rewrites vendor names inside comments, which would +// silently corrupt an inlined filesystem path into one that does not exist. +// This is the consolidation point for +// agent <-> co-located-service communication: services that today mount the +// machine cert/key (or synced files) to talk to nico-api can instead ask the +// agent, and future local needs should add RPCs here rather than new +// sockets, ports, or file drops. +service AgentLocal { + // Returns the current node-auth bearer JWT, minted by the agent from the + // machine's client-certificate key (issue #355). Callers present it as + // `Authorization: Bearer ` to nico-api and must fetch a fresh one + // before `expires_at`. Only the agent ever touches the private key. + rpc GetNodeToken(GetNodeTokenRequest) returns (GetNodeTokenResponse); +} + +message GetNodeTokenRequest { +} + +message GetNodeTokenResponse { + // The signed ES256 JWT. + string token = 1; + // Unix seconds when the token expires. + uint64 expires_at = 2; +}