diff --git a/Cargo.lock b/Cargo.lock index c640054f67..204a5a1283 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1099,6 +1099,7 @@ dependencies = [ "prost", "prost-types", "rand 0.10.1", + "rcgen", "regex", "reqwest 0.13.4", "resolv-conf", @@ -1550,6 +1551,7 @@ dependencies = [ "rcgen", "serde", "thiserror 2.0.18", + "tokio", "tonic", "tower", "tracing", @@ -2913,13 +2915,16 @@ dependencies = [ "hyper-util", "ipnetwork", "itertools 0.14.0", + "jsonwebtoken", "log", "mac_address", "nonempty", "once_cell", + "p256 0.14.0", "prettytable-rs", "prost", "prost-types", + "rcgen", "regex", "resolv-conf", "rustls", @@ -2929,6 +2934,7 @@ dependencies = [ "serde_yaml", "sha2 0.11.0", "sqlx", + "tempfile", "thiserror 2.0.18", "tokio", "tokio-stream", diff --git a/bluefield/charts/nico-dpu-agent/templates/daemonset.yaml b/bluefield/charts/nico-dpu-agent/templates/daemonset.yaml index 096ec5741f..1c230282df 100644 --- a/bluefield/charts/nico-dpu-agent/templates/daemonset.yaml +++ b/bluefield/charts/nico-dpu-agent/templates/daemonset.yaml @@ -131,6 +131,12 @@ spec: {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" args: + {{- with .Values.nodeAuth.audience }} + # DPF agents run with no config file, so the API's [node_auth] + # audience can only reach them as a flag. Both ends must agree or + # every token this agent mints is rejected. + - {{ printf "--node-auth-audience=%s" . | quote }} + {{- end }} - run - "--hbn-config-mode=nvue-rest" - "--agent-platform-type=containerized" diff --git a/bluefield/charts/nico-dpu-agent/tests/node_auth_audience_test.yaml b/bluefield/charts/nico-dpu-agent/tests/node_auth_audience_test.yaml new file mode 100644 index 0000000000..c725cf0ee7 --- /dev/null +++ b/bluefield/charts/nico-dpu-agent/tests/node_auth_audience_test.yaml @@ -0,0 +1,46 @@ +suite: node-auth audience +templates: + - daemonset.yaml +tests: + # DPF agents run with no config file, so the flag is the only way the API's + # [node_auth] audience reaches them. A mismatch means every token this agent + # mints -- and every token it brokers to co-located services -- is rejected. + - it: should pass the configured audience to the agent + set: + image: + repository: test + tag: test + nodeAuth: + audience: nico-api-eu + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --node-auth-audience=nico-api-eu + + # Empty means "say nothing and let the agent default apply", so the flag must + # not be rendered with an empty value -- that would fail the agent's + # non-blank parser at startup. + - it: should omit the flag entirely when no audience is set + set: + image: + repository: test + tag: test + asserts: + - notContains: + path: spec.template.spec.containers[0].args + content: --node-auth-audience= + - lengthEqual: + path: spec.template.spec.containers[0].args + count: 5 + + - it: should quote an audience containing shell-significant characters + set: + image: + repository: test + tag: test + nodeAuth: + audience: 'nico"api' + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: '--node-auth-audience=nico"api' diff --git a/bluefield/charts/nico-dpu-agent/values.yaml b/bluefield/charts/nico-dpu-agent/values.yaml index c741fee685..1c1db94983 100644 --- a/bluefield/charts/nico-dpu-agent/values.yaml +++ b/bluefield/charts/nico-dpu-agent/values.yaml @@ -67,3 +67,12 @@ dhcp_server: fmds: # Will be updated in dpf_services.rs service_name: "" + +### Node-auth (issue #355). +nodeAuth: + # `aud` the agent stamps on the bearer JWTs it mints, both for its own API + # calls and for the tokens it brokers to co-located services. Must match the + # API's [node_auth] audience, which is where dpf_services.rs templates this + # from — so a DPF deployment always sets it. Empty means "pass no flag" and + # falls back to the agent's own default. + audience: "" diff --git a/bluefield/charts/nico-fmds/templates/daemonset.yaml b/bluefield/charts/nico-fmds/templates/daemonset.yaml index e8de5fadf0..711012d566 100644 --- a/bluefield/charts/nico-fmds/templates/daemonset.yaml +++ b/bluefield/charts/nico-fmds/templates/daemonset.yaml @@ -53,15 +53,32 @@ spec: - | {{- $dir := .Values.certsDir | default "/opt/nico" }} {{- $rootCa := .Values.rootCaFile | default "nico_root.pem" }} + {{- if .Values.useNodeTokens }} + echo "Waiting for root CA in {{ $dir }}/pub ..." + while [ ! -f {{ $dir }}/pub/{{ $rootCa }} ]; do + sleep 5 + done + echo "Root CA found, starting nico-fmds (node tokens from the dpu-agent)." + {{- else }} echo "Waiting for certificates in {{ $dir }} ..." while [ ! -f {{ $dir }}/{{ $rootCa }} ] || [ ! -f {{ $dir }}/machine_cert.pem ] || [ ! -f {{ $dir }}/machine_cert.key ]; do sleep 5 done echo "Certificates found, starting nico-fmds." + {{- end }} volumeMounts: + {{- if .Values.useNodeTokens }} + # Token mode never needs the credentials directory, not even to + # wait: the agent publishes the trust anchor to pub/, which holds + # nothing else. + - name: nico-certs-pub + mountPath: {{ .Values.certsDir | default "/opt/nico" }}/pub + readOnly: true + {{- else }} - name: nico-certs mountPath: {{ .Values.certsDir | default "/opt/nico" }} readOnly: true + {{- end }} containers: - name: nico-fmds securityContext: @@ -71,9 +88,14 @@ spec: {{- $dir := .Values.certsDir | default "/opt/nico" }} {{- $rootCa := .Values.rootCaFile | default "nico_root.pem" }} - "--grpc-address=$(POD_IP):50052" + {{- if .Values.useNodeTokens }} + - "--root-ca={{ $dir }}/pub/{{ $rootCa }}" + - "--node-token-socket={{ $dir }}/run/agent.sock" + {{- else }} - "--root-ca={{ $dir }}/{{ $rootCa }}" - "--client-cert={{ $dir }}/machine_cert.pem" - "--client-key={{ $dir }}/machine_cert.key" + {{- end }} imagePullPolicy: {{ .Values.image.pullPolicy }} {{- with toYaml .Values.serviceDaemonSet.resources }} resources: @@ -106,14 +128,45 @@ spec: fieldRef: fieldPath: metadata.namespace volumeMounts: + {{- $dir := .Values.certsDir | default "/opt/nico" }} + {{- $rootCa := .Values.rootCaFile | default "nico_root.pem" }} + {{- if .Values.useNodeTokens }} + # Token mode: the bearer JWT is the credential, so this container + # gets the trust anchor and nothing else. pub/ holds only the CA, + # so the machine key is absent from the pod — and because this is a + # directory mount rather than a subPath, the agent's atomic + # republish of the CA reaches a running pod. + - name: nico-certs-pub + mountPath: {{ $dir }}/pub + readOnly: true + # The agent's token socket gets its own writable mount — connect(2) + # needs write access to the socket inode, which a read-only mount + # (correctly) denies. + - name: nico-agent-run + mountPath: {{ $dir }}/run + {{- else }} - name: nico-certs - mountPath: {{ .Values.certsDir | default "/opt/nico" }} + mountPath: {{ $dir }} readOnly: true + {{- end }} volumes: + {{- if .Values.useNodeTokens }} + # No credentials-directory volume at all in token mode: nothing in this + # pod is entitled to the machine key. + - name: nico-certs-pub + hostPath: + path: {{ .Values.certsDir | default "/opt/nico" }}/pub + type: DirectoryOrCreate + - name: nico-agent-run + hostPath: + path: {{ .Values.certsDir | default "/opt/nico" }}/run + type: DirectoryOrCreate + {{- else }} - name: nico-certs hostPath: path: {{ .Values.certsDir | default "/opt/nico" }} type: DirectoryOrCreate + {{- end }} {{- with .Values.tolerations }} tolerations: {{- toYaml . | nindent 8 }} diff --git a/bluefield/charts/nico-fmds/tests/node_tokens_test.yaml b/bluefield/charts/nico-fmds/tests/node_tokens_test.yaml new file mode 100644 index 0000000000..3bec41f802 --- /dev/null +++ b/bluefield/charts/nico-fmds/tests/node_tokens_test.yaml @@ -0,0 +1,156 @@ +suite: node-auth token mode +templates: + - daemonset.yaml +tests: + # The point of token mode is that the machine private key is not reachable + # from this pod at all. The key sits beside the root CA in certsDir, and the + # container runs as UID 0, so a read-only mount of that directory would not + # be enough -- the volume must be absent entirely. + - it: should not mount the credentials directory anywhere in token mode + set: + image: + repository: test + tag: test + useNodeTokens: true + asserts: + - notContains: + path: spec.template.spec.volumes + content: + name: nico-certs + hostPath: + path: /opt/forge + type: DirectoryOrCreate + - notContains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: nico-certs + mountPath: /opt/forge + readOnly: true + - notContains: + path: spec.template.spec.initContainers[0].volumeMounts + content: + name: nico-certs + mountPath: /opt/forge + readOnly: true + + # pub/ is mounted as a directory, not as a subPath of the CA file: the agent + # installs the CA by atomic rename, and a subPath mount would pin the old + # inode so a rotation never reached a running pod. + - it: should mount only the published CA directory and the agent socket + set: + image: + repository: test + tag: test + useNodeTokens: true + asserts: + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: nico-certs-pub + mountPath: /opt/forge/pub + readOnly: true + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: nico-agent-run + mountPath: /opt/forge/run + - lengthEqual: + path: spec.template.spec.containers[0].volumeMounts + count: 2 + - contains: + path: spec.template.spec.volumes + content: + name: nico-certs-pub + hostPath: + path: /opt/forge/pub + type: DirectoryOrCreate + + - it: should read the CA from pub/ and authenticate with a token, not a cert + set: + image: + repository: test + tag: test + useNodeTokens: true + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --root-ca=/opt/forge/pub/forge_root.pem + - contains: + path: spec.template.spec.containers[0].args + content: --node-token-socket=/opt/forge/run/agent.sock + - notContains: + path: spec.template.spec.containers[0].args + content: --client-cert=/opt/forge/machine_cert.pem + - notContains: + path: spec.template.spec.containers[0].args + content: --client-key=/opt/forge/machine_cert.key + + # The init container gates the main one; in token mode it has no business + # waiting on the machine cert/key, which may never arrive in this pod. + - it: should wait only for the published CA in token mode + set: + image: + repository: test + tag: test + useNodeTokens: true + asserts: + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: '/opt/forge/pub/forge_root\.pem' + - notMatchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: 'machine_cert\.key' + - contains: + path: spec.template.spec.initContainers[0].volumeMounts + content: + name: nico-certs-pub + mountPath: /opt/forge/pub + readOnly: true + - lengthEqual: + path: spec.template.spec.initContainers[0].volumeMounts + count: 1 + + - it: should follow certsDir into the published CA path + set: + image: + repository: test + tag: test + useNodeTokens: true + certsDir: /srv/creds + rootCaFile: site_root.pem + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --root-ca=/srv/creds/pub/site_root.pem + - contains: + path: spec.template.spec.volumes + content: + name: nico-certs-pub + hostPath: + path: /srv/creds/pub + type: DirectoryOrCreate + + # mTLS remains the default, and must not acquire the token-mode plumbing. + - it: should keep the credentials mount and no pub volume by default + set: + image: + repository: test + tag: test + asserts: + - contains: + path: spec.template.spec.volumes + content: + name: nico-certs + hostPath: + path: /opt/forge + type: DirectoryOrCreate + - notContains: + path: spec.template.spec.volumes + content: + name: nico-certs-pub + hostPath: + path: /opt/forge/pub + type: DirectoryOrCreate + - notContains: + path: spec.template.spec.containers[0].args + content: --node-token-socket=/opt/forge/run/agent.sock diff --git a/bluefield/charts/nico-fmds/values.yaml b/bluefield/charts/nico-fmds/values.yaml index d88ca3dfd1..410af3a336 100644 --- a/bluefield/charts/nico-fmds/values.yaml +++ b/bluefield/charts/nico-fmds/values.yaml @@ -13,6 +13,18 @@ exposedPorts: certsDir: /opt/forge rootCaFile: forge_root.pem +### Node-auth (issue #355). +# When true, fmds authenticates to nico-api with short-lived bearer JWTs +# fetched from the dpu-agent's local API socket (/run/agent.sock) +# instead of the machine mTLS cert/key. The pod then mounts only +# /pub — where the agent publishes the trust anchor and nothing +# else — plus that socket, so the machine private key is absent from it +# rather than merely read-only (the container runs as UID 0). Mounting pub/ +# as a directory also means the agent's atomic republish of a rotated CA +# reaches a running pod. Requires a dpu-agent that serves the local API and +# publishes the CA, and [node_auth] enabled on the API. +useNodeTokens: false + ### Service specific values ### # Prometheus /metrics listen port (must match nico-otelcol scrape target). metricsPort: 8888 diff --git a/crates/agent/Cargo.toml b/crates/agent/Cargo.toml index d1e2d44d6c..891a318f6a 100644 --- a/crates/agent/Cargo.toml +++ b/crates/agent/Cargo.toml @@ -140,6 +140,7 @@ tonic-prost-build = { workspace = true } carbide-instrument = { path = "../instrument", features = ["test-support"] } carbide-test-support = { path = "../test-support" } ctor = { workspace = true } +rcgen = { workspace = true } prost = { workspace = true } rustls-pki-types = { workspace = true } diff --git a/crates/agent/example_agent_config.toml b/crates/agent/example_agent_config.toml index 28fd03b079..c433f9db65 100644 --- a/crates/agent/example_agent_config.toml +++ b/crates/agent/example_agent_config.toml @@ -18,6 +18,9 @@ api-server = "http://localhost:50051" pxe-server = "http://127.0.0.1:8080" root-ca = "/home/amaslennikov/pcarbide/credentials/localhost.pem" +# Unix socket where the agent serves node tokens to co-located services +# (issue #355). Same default containerized (DPF) and on plain DPU OS. +#local-api-socket = "/opt/forge/run/agent.sock" [machine] interface-id = "3048d658-0b13-4a5e-91cc-f47e45a031f9" diff --git a/crates/agent/src/command_line.rs b/crates/agent/src/command_line.rs index 7a633e10bc..80aea661cc 100644 --- a/crates/agent/src/command_line.rs +++ b/crates/agent/src/command_line.rs @@ -25,6 +25,16 @@ use url::Url; use crate::network_monitor::NetworkPingerType; +/// Rejects an empty or whitespace-only audience at parse time. A blank value +/// would otherwise reach the minter and produce tokens the API cannot match, +/// with nothing in the logs pointing at the flag. +fn non_blank_audience(value: &str) -> Result { + if value.trim().is_empty() { + return Err("node-auth audience must not be empty".to_string()); + } + Ok(value.to_string()) +} + #[derive(Parser)] #[clap(name = "forge-dpu-agent")] pub struct Options { @@ -36,6 +46,17 @@ pub struct Options { #[clap(long)] pub config_path: Option, + /// Overrides `[forge-system] node-auth-audience`. DPF deploys the agent + /// with no config file at all, so the audience can only reach it as a + /// flag; the API templates this from its own `[node_auth] audience` so + /// the two ends cannot drift. + #[clap( + long, + value_parser = non_blank_audience, + help = "Audience claim stamped on node-auth bearer JWTs; must match the API's [node_auth] audience" + )] + pub node_auth_audience: Option, + #[clap(subcommand)] pub cmd: Option, } diff --git a/crates/agent/src/lib.rs b/crates/agent/src/lib.rs index f09214c748..2f30ecae47 100644 --- a/crates/agent/src/lib.rs +++ b/crates/agent/src/lib.rs @@ -65,6 +65,7 @@ mod host_machine_id; mod instance_metadata_endpoint; pub mod instrumentation; pub mod lldp; +mod local_api; mod machine_inventory_updater; mod main_loop; mod managed_files; @@ -92,6 +93,15 @@ pub const FMDS_MINIMUM_HBN_VERSION: &str = "1.5.0-doca2.2.0"; pub const NVUE_MINIMUM_HBN_VERSION: &str = "2.0.0-doca2.5.0"; const BOOTSTRAP_CA_OUTPUT_PATH: &str = "/opt/forge/forge_root.pem"; +/// Second copy of the trust anchor, in a directory that holds nothing else. +/// +/// Co-located services need the root CA but must never see the machine private +/// key that lives beside it (issue #355). A container can only be given one or +/// the other: mounting the credentials directory exposes the key, and mounting +/// the CA file alone by `subPath` bind-mounts its inode, so the atomic rename +/// in `install_bootstrap_ca` would never reach a running consumer. A directory +/// containing only the CA gives both properties — no key, and renames resolve. +const BOOTSTRAP_CA_PUBLIC_PATH: &str = "/opt/forge/pub/forge_root.pem"; const MOUNTED_BOOTSTRAP_CA_PATH: &str = "/var/run/secrets/nico-bootstrap-ca/ca.pem"; const MAX_BOOTSTRAP_CA_BYTES: usize = 1024 * 1024; const BOOTSTRAP_CA_DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(30); @@ -272,7 +282,21 @@ fn install_bootstrap_ca(contents: &[u8], output_path: &Path) -> eyre::Result<()> async fn provision_bootstrap_ca(options: &command_line::InitContainerOptions) -> eyre::Result<()> { let contents = acquire_bootstrap_ca(options.bootstrap_ca_source, &options.bootstrap_ca_url).await?; - install_bootstrap_ca(&contents, Path::new(BOOTSTRAP_CA_OUTPUT_PATH)) + install_bootstrap_ca(&contents, Path::new(BOOTSTRAP_CA_OUTPUT_PATH))?; + publish_bootstrap_ca(&contents) +} + +/// Mirrors the trust anchor into [`BOOTSTRAP_CA_PUBLIC_PATH`] for key-less +/// consumers. Same atomic install, so a consumer with the directory mounted +/// picks up a replacement without restarting. +fn publish_bootstrap_ca(contents: &[u8]) -> eyre::Result<()> { + let public_path = Path::new(BOOTSTRAP_CA_PUBLIC_PATH); + let parent = public_path + .parent() + .ok_or_else(|| eyre::eyre!("public CA path has no parent: {}", public_path.display()))?; + std::fs::create_dir_all(parent) + .wrap_err_with(|| format!("failed to create public CA directory {}", parent.display()))?; + install_bootstrap_ca(contents, public_path) } pub async fn start(cmdline: command_line::Options) -> eyre::Result<()> { @@ -281,7 +305,7 @@ pub async fn start(cmdline: command_line::Options) -> eyre::Result<()> { return Ok(()); } - let (agent, path) = match cmdline.config_path { + let (mut agent, path) = match cmdline.config_path { // normal production case None => (AgentConfig::default(), "default".to_string()), // development overrides @@ -293,6 +317,17 @@ pub async fn start(cmdline: command_line::Options) -> eyre::Result<()> { config_path.display().to_string(), ), }; + // Containerized (DPF) agents run without a config file, so the flag is the + // only way the API's configured audience reaches them. + if let Some(node_auth_audience) = cmdline.node_auth_audience.clone() { + agent.forge_system.node_auth_audience = node_auth_audience; + } + // After the override above, so the flag and the file are held to the same + // rule rather than only the flag validating itself at parse time. + agent + .forge_system + .validate() + .map_err(|e| eyre::eyre!("invalid [forge-system] in agent config: {e}"))?; agent .machine_identity .validate() @@ -303,6 +338,36 @@ pub async fn start(cmdline: command_line::Options) -> eyre::Result<()> { tracing::warn!("Pretending local host is a DPU. Dev only."); } + // Republish on every start, not just from the init container: an operator + // replacing the trust anchor out of band would otherwise leave co-located + // services on the previous one until the next initialization. + match std::fs::read(&agent.forge_system.root_ca) { + Ok(contents) => { + if let Err(error) = publish_bootstrap_ca(&contents) { + tracing::warn!( + target: "node_auth", + %error, + "node-auth: could not publish the root CA for co-located services" + ); + } + } + Err(error) => tracing::debug!( + target: "node_auth", + %error, + path = %agent.forge_system.root_ca, + "node-auth: no root CA to publish for co-located services yet" + ), + } + + // Node-auth (#355): the agent is the only process on the DPU that holds + // the machine key. This minter signs bearer JWTs for the agent's own API + // calls AND backs the local API socket that brokers tokens to co-located + // services (fmds, ...). Ignored by the API unless [node_auth] is enabled. + let node_jwt_minter = ::rpc::node_jwt::NodeJwtMinter::with_audience( + agent.forge_system.client_cert.clone(), + agent.forge_system.client_key.clone(), + agent.forge_system.node_auth_audience.clone(), + ); let forge_client_config = Arc::new( ForgeClientConfig::new( agent.forge_system.root_ca.clone(), @@ -311,6 +376,7 @@ pub async fn start(cmdline: command_line::Options) -> eyre::Result<()> { key_path: agent.forge_system.client_key.clone(), }), ) + .with_token_provider(node_jwt_minter.clone()) .use_mgmt_vrf()?, ); @@ -325,6 +391,24 @@ pub async fn start(cmdline: command_line::Options) -> eyre::Result<()> { tracing::warn!("Upgrades disabled. Dev only"); } + // Broker node tokens to co-located services over the local API + // socket. Not fatal on failure, and retried forever: consumers + // fall back to their own credentials (mTLS client cert) until the + // socket comes up, and on non-DPF DPU OS the agent may start + // before its socket directory is usable. + tokio::spawn({ + let minter = node_jwt_minter.clone(); + let socket_path = agent.forge_system.local_api_socket.clone(); + async move { + loop { + if let Err(error) = local_api::serve(minter.clone(), &socket_path).await { + tracing::warn!(target: "node_auth", %error, "node-auth: agent local API server failed; retrying"); + } + tokio::time::sleep(std::time::Duration::from_secs(10)).await; + } + } + }); + let Registration { machine_id, factory_mac_address, diff --git a/crates/agent/src/local_api.rs b/crates/agent/src/local_api.rs new file mode 100644 index 0000000000..80d8a6ea47 --- /dev/null +++ b/crates/agent/src/local_api.rs @@ -0,0 +1,338 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! The dpu-agent's local API: an `AgentLocal` gRPC service on a unix socket, +//! by default in a dedicated `run/` subdirectory of the shared `/opt/forge` +//! credentials directory (issue #355) — so containerized consumers mount the +//! socket read-write while the credentials stay on a read-only mount. +//! +//! This is the consolidation point for agent <-> co-located-service +//! communication on the DPU. First RPC: `GetNodeToken`, which hands +//! co-located NICo services (fmds, …) a short-lived node-auth bearer JWT so +//! they can authenticate to nico-api without mounting the machine's private +//! key — only the agent ever touches the key. Future local needs should add +//! RPCs here rather than new sockets, ports, or file drops. + +use std::sync::Arc; + +use ::rpc::agent_local::agent_local_server::{AgentLocal, AgentLocalServer}; +use ::rpc::agent_local::{GetNodeTokenRequest, GetNodeTokenResponse}; +use ::rpc::node_jwt::NodeJwtMinter; +use eyre::WrapErr; +use tokio_stream::wrappers::UnixListenerStream; +use tonic::{Request, Response, Status}; + +struct AgentLocalService { + minter: Arc, +} + +#[tonic::async_trait] +impl AgentLocal for AgentLocalService { + async fn get_node_token( + &self, + _request: Request, + ) -> Result, Status> { + // Minting reads the cert/key from disk on cache miss — cheap enough to + // do inline, and it means the endpoint starts working the moment the + // machine certificate lands without any coordination. + match self.minter.current_with_expiry() { + Some((token, expires_at)) => { + Ok(Response::new(GetNodeTokenResponse { token, expires_at })) + } + None => Err(Status::unavailable( + "no node token available yet; machine certificate not present or unreadable", + )), + } + } +} + +/// Prepares the socket's parent directory, which must be dedicated to this +/// socket and nothing else. +/// +/// The directory has to be unreachable to other users *before* `bind`: `bind` +/// creates the socket with umask-derived permissions — typically +/// world-connectable — and the 0600 chmod only lands afterwards. The listener +/// is already bound in that window, so a connection from any local user would +/// queue in the backlog and be served a full machine identity the moment +/// accepting starts. An unreachable parent closes the window; the socket's own +/// 0600 is then defence in depth. +/// +/// But `local-api-socket` is operator-configurable, so applying 0700 to +/// whatever the parent happens to be is its own hazard: `/run/agent.sock` +/// would take `/run` to 0700 and lock every non-root service on the box out of +/// its runtime files. Only a directory this function created, or one holding +/// nothing but this socket, is ours to restrict — anything else is rejected +/// with an actionable message rather than silently modified. +fn prepare_socket_dir(dir: &std::path::Path, socket_path: &str) -> eyre::Result<()> { + use std::os::unix::fs::{DirBuilderExt, PermissionsExt}; + + if !dir.exists() { + // Ancestors keep their normal modes; only the socket directory itself + // is created restricted, and created that way from the first instant + // rather than chmod-ed after the fact. + if let Some(ancestor) = dir.parent() { + std::fs::create_dir_all(ancestor) + .wrap_err(format!("creating socket directory {}", ancestor.display()))?; + } + return std::fs::DirBuilder::new() + .mode(0o700) + .create(dir) + .wrap_err(format!("creating socket directory {}", dir.display())); + } + + let socket_name = std::path::Path::new(socket_path).file_name(); + for entry in + std::fs::read_dir(dir).wrap_err(format!("reading socket directory {}", dir.display()))? + { + let entry = entry.wrap_err(format!("reading socket directory {}", dir.display()))?; + let name = entry.file_name(); + if socket_name != Some(name.as_os_str()) { + eyre::bail!( + "socket directory {} is shared with other files (found {}); \ + point local-api-socket at a directory used for nothing else, \ + so restricting it to 0700 cannot lock other services out", + dir.display(), + name.to_string_lossy() + ); + } + } + + std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o700)).wrap_err(format!( + "restricting socket directory permissions on {}", + dir.display() + )) +} + +/// Binds the local API socket and serves until the process exits. The socket +/// is created mode 0600 (root-only): every legitimate consumer on the DPU +/// runs as root, and nothing else on the shared directory should be able to +/// obtain machine credentials. +/// +/// Deployment-agnostic: containerized (DPF) the socket lands on the mounted +/// `/opt/forge` volume; as a plain service on DPU OS (non-DPF) the parent +/// directory is created if the agent starts before anything else touched it. +pub async fn serve(minter: Arc, socket_path: &str) -> eyre::Result<()> { + use std::os::unix::fs::PermissionsExt; + + if let Some(parent) = std::path::Path::new(socket_path).parent() { + prepare_socket_dir(parent, socket_path)?; + } + // Remove a stale socket from a previous run; bind() fails on an existing + // path even when nothing is listening. + match std::fs::remove_file(socket_path) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e).wrap_err(format!("removing stale socket {socket_path}")), + } + + let listener = tokio::net::UnixListener::bind(socket_path) + .wrap_err(format!("binding agent local API socket {socket_path}"))?; + std::fs::set_permissions(socket_path, std::fs::Permissions::from_mode(0o600)) + .wrap_err(format!("restricting socket permissions on {socket_path}"))?; + tracing::info!(target: "node_auth", socket = %socket_path, "node-auth: serving agent local API (node tokens)"); + + tonic::transport::Server::builder() + .add_service(AgentLocalServer::new(AgentLocalService { minter })) + .serve_with_incoming(UnixListenerStream::new(listener)) + .await + .wrap_err("agent local API server exited") +} + +#[cfg(test)] +mod tests { + use ::rpc::node_jwt::NodeTokenProvider; + use ::rpc::node_token_socket::SocketTokenSource; + + use super::*; + + const SPIFFE_URI: &str = "spiffe://forge.local/forge-system/machine/fm100xtest"; + + fn write_cert_and_key(dir: &tempfile::TempDir) -> (String, String) { + let mut params = rcgen::CertificateParams::default(); + params.subject_alt_names = vec![rcgen::SanType::URI( + rcgen::string::Ia5String::try_from(SPIFFE_URI.to_string()).expect("uri"), + )]; + let key = rcgen::KeyPair::generate().expect("key pair"); + let cert = params.self_signed(&key).expect("certificate"); + let cert_path = dir.path().join("cert.pem"); + let key_path = dir.path().join("cert.key"); + std::fs::write(&cert_path, cert.pem()).expect("write cert"); + std::fs::write(&key_path, key.serialize_pem()).expect("write key"); + ( + cert_path.to_string_lossy().into_owned(), + key_path.to_string_lossy().into_owned(), + ) + } + + /// End-to-end broker flow: agent serves tokens minted from the machine + /// cert; a key-less consumer obtains one through `SocketTokenSource`. + #[tokio::test] + async fn keyless_consumer_gets_token_via_socket() { + let dir = tempfile::tempdir().expect("tempdir"); + let (cert_path, key_path) = write_cert_and_key(&dir); + // Its own subdirectory, as in production (`/run/agent.sock`): + // the socket never shares a directory with the credentials. + let socket = dir.path().join("run").join("agent.sock"); + let socket_str = socket.to_string_lossy().into_owned(); + + let minter = NodeJwtMinter::new(cert_path, key_path); + let expected = minter.current().expect("agent side can mint"); + tokio::spawn({ + let socket_str = socket_str.clone(); + async move { serve(minter, &socket_str).await } + }); + + let source = SocketTokenSource::spawn(socket_str); + let mut got = None; + for _ in 0..100 { + if let Some(token) = source.current() { + got = Some(token); + break; + } + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + assert_eq!(got.as_deref(), Some(expected.as_str())); + } + + #[tokio::test] + async fn socket_is_root_only() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().expect("tempdir"); + let (cert_path, key_path) = write_cert_and_key(&dir); + let socket = dir.path().join("run").join("agent.sock"); + let socket_str = socket.to_string_lossy().into_owned(); + + tokio::spawn({ + let socket_str = socket_str.clone(); + async move { serve(NodeJwtMinter::new(cert_path, key_path), &socket_str).await } + }); + for _ in 0..100 { + if socket.exists() { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + let mode = std::fs::metadata(&socket) + .expect("socket metadata") + .permissions() + .mode() + & 0o777; + assert_eq!(mode, 0o600); + } + + /// The socket's own mode is applied only after `bind`, so the directory is + /// what actually keeps other users out during that window. Anyone who got + /// in would be handed a full machine identity. + #[tokio::test] + async fn socket_directory_is_root_only_before_the_socket_exists() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().expect("tempdir"); + let (cert_path, key_path) = write_cert_and_key(&dir); + // A pre-existing, world-traversable directory: the agent must tighten + // it rather than assume a fresh one. + let run_dir = dir.path().join("run"); + std::fs::create_dir(&run_dir).expect("create run dir"); + std::fs::set_permissions(&run_dir, std::fs::Permissions::from_mode(0o755)) + .expect("loosen run dir"); + let socket = run_dir.join("agent.sock"); + let socket_str = socket.to_string_lossy().into_owned(); + + tokio::spawn({ + let socket_str = socket_str.clone(); + async move { serve(NodeJwtMinter::new(cert_path, key_path), &socket_str).await } + }); + for _ in 0..100 { + if socket.exists() { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + + let mode = std::fs::metadata(&run_dir) + .expect("run dir metadata") + .permissions() + .mode() + & 0o777; + assert_eq!(mode, 0o700, "socket directory must exclude other users"); + } + + /// Tightening the parent is only safe when the parent belongs to us. + /// `local-api-socket = /run/agent.sock` would otherwise take `/run` to + /// 0700 and lock every non-root service on the box out of its runtime + /// files, so a directory holding anything else is refused outright. + #[test] + fn shared_socket_directory_is_refused_rather_than_restricted() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().expect("tempdir"); + let shared = dir.path().join("run"); + std::fs::create_dir(&shared).expect("create shared dir"); + std::fs::write(shared.join("someone-elses.pid"), b"1234").expect("write neighbour"); + std::fs::set_permissions(&shared, std::fs::Permissions::from_mode(0o755)) + .expect("loosen shared dir"); + let socket = shared.join("agent.sock"); + + let err = prepare_socket_dir(&shared, &socket.to_string_lossy()) + .expect_err("a shared directory must not be accepted"); + assert!( + err.to_string().contains("someone-elses.pid"), + "the error should name the file that made it shared, got: {err}" + ); + + let mode = std::fs::metadata(&shared) + .expect("shared dir metadata") + .permissions() + .mode() + & 0o777; + assert_eq!( + mode, 0o755, + "a refused directory must be left exactly as it was found" + ); + } + + /// A directory we create ourselves is restricted from its first instant, + /// with no window at a looser mode for anyone to slip through. + #[test] + fn fresh_socket_directory_is_created_root_only() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().expect("tempdir"); + let run_dir = dir.path().join("nested").join("run"); + let socket = run_dir.join("agent.sock"); + + prepare_socket_dir(&run_dir, &socket.to_string_lossy()).expect("prepare fresh dir"); + + let mode = std::fs::metadata(&run_dir) + .expect("run dir metadata") + .permissions() + .mode() + & 0o777; + assert_eq!(mode, 0o700, "a freshly created socket directory is 0700"); + } + + /// The stale socket from a previous run is ours, so it must not be + /// mistaken for a neighbour and block startup. + #[test] + fn a_stale_socket_does_not_make_the_directory_look_shared() { + let dir = tempfile::tempdir().expect("tempdir"); + let run_dir = dir.path().join("run"); + std::fs::create_dir(&run_dir).expect("create run dir"); + let socket = run_dir.join("agent.sock"); + std::fs::write(&socket, b"").expect("stale socket"); + + prepare_socket_dir(&run_dir, &socket.to_string_lossy()) + .expect("a directory holding only our own socket is still dedicated"); + } +} diff --git a/crates/agent/src/tests/common/mod.rs b/crates/agent/src/tests/common/mod.rs index 2fabebac29..4ff31d7786 100644 --- a/crates/agent/src/tests/common/mod.rs +++ b/crates/agent/src/tests/common/mod.rs @@ -113,6 +113,7 @@ pub fn setup_agent_run_env( let opts = crate::Options { version: false, config_path: Some(acf.path().to_path_buf()), + node_auth_audience: None, cmd: Some(crate::AgentCommand::Run(Box::new(crate::RunOptions { enable_metadata_service: test_metadata_service, override_machine_id: None, diff --git a/crates/api-core/src/api.rs b/crates/api-core/src/api.rs index 4e7c3d6e77..79a4402305 100644 --- a/crates/api-core/src/api.rs +++ b/crates/api-core/src/api.rs @@ -90,6 +90,10 @@ pub struct Api { pub(crate) component_manager: Option, pub(crate) bms_client: OnceLock>, pub(crate) secrets_context: Option, + /// Validator for node-auth bearer JWTs (issue #355). `Some` only when + /// `[node_auth] enabled`; installed into the authn middleware by the + /// listener. + pub(crate) node_jwt_validator: Option>, } pub(crate) type ScoutStreamType = diff --git a/crates/api-core/src/cfg/README.md b/crates/api-core/src/cfg/README.md index 8c5bab2c59..3bd696bb9b 100644 --- a/crates/api-core/src/cfg/README.md +++ b/crates/api-core/src/cfg/README.md @@ -123,6 +123,7 @@ applicable. | `dhcp_lease_expiry_handling` | `bool` | `false` | `networking` | Enables IP cleanup when a DHCP lease expires. | | `certificates` | `CertificatesConfig` | *(default)* | `security` | Certificate vending backend, selected independently of the credential store; the default shares the credential Vault (see [CertificatesConfig](#certificatesconfig)). | | `allow_insecure_discovery` | `bool` | `false` | `machines` | Allows machines to submit discovery without enforcing the request comes from the expected IP address. Needed for *Integration tests only*, should otherwise not be used. | +| `node_auth` | `NodeAuthConfig` | *(default)* | `security` | How Scout and the DPU-agent authenticate: bearer JWTs, machine mTLS client certificates, or both during a migration (see [NodeAuthConfig](#nodeauthconfig)). | --- @@ -276,6 +277,19 @@ available for topology-specific flows. | `identity_keyfile_path` | `String` | `""` | Server identity private key. | | `admin_root_cafile_path` | `String` | `""` | Admin root CA for admin client validation. | +### `NodeAuthConfig` + +Node (Scout / DPU-agent) authentication. Bearer tokens are off by default, so +the default is machine mTLS exactly as before. See +`docs/design/machine-identity/node-auth-jwt.md`. + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `enabled` | `bool` | `false` | Accept `Authorization: Bearer` node JWTs. Nodes self-sign these with their existing mTLS client-certificate key and carry the certificate in the token's `x5c` header; the API verifies it against `[tls] root_cafile_path`. Requires a TLS listener -- the API refuses to accept bearer tokens over plaintext. | +| `mtls_enabled` | `bool` | `true` | Accept machine mTLS client certificates as node identity. Turn off only once the fleet presents bearer tokens; startup fails if this and `enabled` are both false. Scoped to machine certificates -- service and admin-CLI certificates are unaffected. | +| `audience` | `String` | `nico-api` | Required `aud` claim. Nodes must stamp the same value (`--node-auth-audience` for Scout and the agent; the API templates it onto DPF-deployed agents), or every token they mint is rejected. | +| `max_token_ttl_sec` | `u32` | `900` | Longest accepted token lifetime, in seconds. Clients mint 300 s tokens; this caps how far a client may push `exp`. Must be greater than zero and at most 86400. | + ### `AuthConfig` | Field | Type | Default | Description | diff --git a/crates/api-core/src/cfg/file.rs b/crates/api-core/src/cfg/file.rs index e5a349b162..adfd5957bb 100644 --- a/crates/api-core/src/cfg/file.rs +++ b/crates/api-core/src/cfg/file.rs @@ -496,6 +496,12 @@ pub struct CarbideConfig { #[serde(default)] pub machine_identity: MachineIdentityConfig, + /// Node-auth: bearer JWTs that Scout / DPU-agent self-sign with their + /// existing mTLS client-certificate key, accepted alongside (or instead + /// of) mTLS. Section `[node_auth]`. + #[serde(default)] + pub node_auth: NodeAuthConfig, + /// Disables role-based access control enforcement. /// Intended for testing and development only. #[serde(default)] @@ -1969,6 +1975,95 @@ impl Default for MachineIdentityConfig { } } +/// Node-auth (Scout / DPU-agent bearer JWT) configuration. +/// Loaded from `[node_auth]` section in config. +/// +/// There is no server-side signing key: nodes self-sign tokens with their +/// existing mTLS client-certificate key and the API validates the embedded +/// `x5c` chain against the client-cert root CA (see +/// `docs/design/machine-identity/node-auth-jwt.md`). +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct NodeAuthConfig { + /// Master switch. When false, no bearer authenticator is installed and + /// nodes keep authenticating via mTLS client certs only. + #[serde(default = "node_auth_default_enabled")] + pub enabled: bool, + /// `aud` claim required on presented tokens. Must match what nodes mint + /// (`rpc::node_jwt::NODE_JWT_AUDIENCE`). + #[serde(default = "node_auth_default_audience")] + pub audience: String, + /// Maximum accepted token lifetime, in seconds. Nodes mint 5-minute + /// tokens; this bounds how far a (compromised) client can stretch `exp`. + #[serde(default = "node_auth_default_max_token_ttl_sec")] + pub max_token_ttl_sec: u32, + /// Whether machine mTLS client certificates are accepted as node identity. + /// On by default. Scoped to MACHINE certs only — service and admin-CLI + /// client certs on the same listener are unaffected. Disable only once the + /// fleet presents bearer tokens. + #[serde(default = "node_auth_default_mtls_enabled")] + pub mtls_enabled: bool, +} + +/// Upper bound on accepted token lifetime. Node tokens are minted locally on +/// demand, so anything beyond a day is almost certainly a misconfiguration. +pub const NODE_AUTH_MAX_TOKEN_TTL_SEC: u32 = 86_400; + +impl NodeAuthConfig { + /// Validates node-auth settings. Call unconditionally at startup: the + /// lockout check applies even when [`enabled`](Self::enabled) is false. + pub fn validate(&self) -> eyre::Result<()> { + if !self.enabled && !self.mtls_enabled { + return Err(eyre::eyre!( + "[node_auth] enabled = false and mtls_enabled = false would leave nodes with no \ + way to authenticate; enable at least one of bearer tokens or machine mTLS" + )); + } + if !self.enabled { + // Remaining checks only constrain token validation. + return Ok(()); + } + if self.audience.trim().is_empty() { + return Err(eyre::eyre!("[node_auth] audience must not be empty")); + } + if self.max_token_ttl_sec == 0 { + return Err(eyre::eyre!( + "[node_auth] max_token_ttl_sec must be greater than zero" + )); + } + if self.max_token_ttl_sec > NODE_AUTH_MAX_TOKEN_TTL_SEC { + return Err(eyre::eyre!( + "[node_auth] max_token_ttl_sec {} exceeds maximum {NODE_AUTH_MAX_TOKEN_TTL_SEC}", + self.max_token_ttl_sec + )); + } + Ok(()) + } +} + +fn node_auth_default_enabled() -> bool { + false +} +fn node_auth_default_audience() -> String { + "nico-api".to_string() +} +fn node_auth_default_max_token_ttl_sec() -> u32 { + 900 +} +fn node_auth_default_mtls_enabled() -> bool { + true +} + +impl Default for NodeAuthConfig { + fn default() -> Self { + Self { + enabled: node_auth_default_enabled(), + audience: node_auth_default_audience(), + max_token_ttl_sec: node_auth_default_max_token_ttl_sec(), + mtls_enabled: node_auth_default_mtls_enabled(), + } + } +} + impl From for model::tenant::IdentityConfigValidationBounds { fn from(mi: MachineIdentityConfig) -> Self { Self { @@ -3610,6 +3705,27 @@ mod tests { use super::*; use crate::test_support::network_segment::FIXTURE_TENANT_ORG_ID; + /// Disabling both bearer tokens and machine mTLS would lock every node out + /// of the API; validation must refuse the combination, and each mechanism + /// alone must pass. + #[test] + fn node_auth_rejects_all_methods_disabled() { + let both_off = NodeAuthConfig { + enabled: false, + mtls_enabled: false, + ..NodeAuthConfig::default() + }; + assert!(both_off.validate().is_err()); + + assert!(NodeAuthConfig::default().validate().is_ok()); + let jwt_only = NodeAuthConfig { + enabled: true, + mtls_enabled: false, + ..NodeAuthConfig::default() + }; + assert!(jwt_only.validate().is_ok()); + } + const TEST_DATA_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/src/cfg/test_data"); fn vpc_config( diff --git a/crates/api-core/src/dpf_services.rs b/crates/api-core/src/dpf_services.rs index e3342b2af6..a5173714a5 100644 --- a/crates/api-core/src/dpf_services.rs +++ b/crates/api-core/src/dpf_services.rs @@ -32,7 +32,7 @@ use carbide_dpf::{ use crate::cfg::file::{ DpfBootstrapCaObjectKind, DpfDpuAgentBootstrapCa, DpfExtraService, - DpfResolvedMandatoryServicesConfig, DpfServiceConfig, + DpfResolvedMandatoryServicesConfig, DpfServiceConfig, NodeAuthConfig, }; /// Default DOCA helm registry (DPUServiceTemplate source.repoURL). @@ -366,6 +366,7 @@ pub fn dts_service(cfg: &DpfServiceConfig) -> ServiceDefinition { fn dpu_agent_helm_values( cfg: &DpfServiceConfig, bootstrap_ca: &DpfDpuAgentBootstrapCa, + node_auth_audience: &str, ) -> serde_json::Value { let mut values = serde_json::json!({ "image": { @@ -376,6 +377,11 @@ fn dpu_agent_helm_values( "nvue_https_address": "nvue", "nvue_credentials_secret_name": "hbn-user-password", "nvue_password_key": "password", + }, + // DPF agents get no config file, so [node_auth] audience has to ride + // in as a flag or they mint tokens the API will reject. + "nodeAuth": { + "audience": node_auth_audience, } }); apply_image_pull_secrets(&mut values, cfg); @@ -417,9 +423,10 @@ fn dpu_agent_helm_values( pub fn dpu_agent_service( cfg: &DpfServiceConfig, bootstrap_ca: &DpfDpuAgentBootstrapCa, + node_auth_audience: &str, ) -> ServiceDefinition { ServiceDefinition { - helm_values: Some(dpu_agent_helm_values(cfg, bootstrap_ca)), + helm_values: Some(dpu_agent_helm_values(cfg, bootstrap_ca, node_auth_audience)), service_daemon_set_annotations: Some(BTreeMap::new()), @@ -479,12 +486,18 @@ pub fn dhcp_server_service(cfg: &DpfServiceConfig) -> ServiceDefinition { } /// Forge FMDS service definition. -pub fn fmds_service(cfg: &DpfServiceConfig) -> ServiceDefinition { +/// +/// `use_node_tokens` follows the API's `[node_auth] enabled` switch: when the +/// API accepts bearer JWTs, fmds is deployed fetching them from the +/// dpu-agent's local API socket instead of mounting the machine cert/key +/// (issue #355). Requires a dpu-agent image that serves the local API. +pub fn fmds_service(cfg: &DpfServiceConfig, use_node_tokens: bool) -> ServiceDefinition { let mut helm_values = serde_json::json!({ "image": { "repository": cfg.docker_repo_url, "tag": cfg.docker_image_tag, - } + }, + "useNodeTokens": use_node_tokens, }); apply_image_pull_secrets(&mut helm_values, cfg); ServiceDefinition { @@ -676,16 +689,22 @@ pub fn doca_xplane_service(cfg: &DpfServiceConfig) -> ServiceDefinition { } /// Build the full list of resolved mandatory DPU services. +/// +/// `node_auth` mirrors the API's `[node_auth]` section: `enabled` switches +/// fmds to bearer tokens from the dpu-agent's local API, and `audience` is +/// templated onto the agent so both ends stamp/expect the same `aud` +/// (issue #355). pub fn mandatory_services( resolved: &DpfResolvedMandatoryServicesConfig, bootstrap_ca: &DpfDpuAgentBootstrapCa, + node_auth: &NodeAuthConfig, ) -> Vec { let mut service_vec = vec![ dts_service(&resolved.base.dts), doca_hbn_service(&resolved.base.doca_hbn), dhcp_server_service(&resolved.base.dhcp_server), - dpu_agent_service(&resolved.base.dpu_agent, bootstrap_ca), - fmds_service(&resolved.base.fmds), + dpu_agent_service(&resolved.base.dpu_agent, bootstrap_ca, &node_auth.audience), + fmds_service(&resolved.base.fmds, node_auth.enabled), otelcol_service(&resolved.base.otel), ]; @@ -720,7 +739,11 @@ mod tests { fn dpu_agent_bootstrap_ca_helm_values_follow_site_policy() { value_scenarios!( run = |policy| { - dpu_agent_helm_values(&default_dpu_agent_service(), &policy) + dpu_agent_helm_values( + &default_dpu_agent_service(), + &policy, + ::rpc::node_jwt::NODE_JWT_AUDIENCE, + ) .get("bootstrapCa") .cloned() }; @@ -859,6 +882,7 @@ mod tests { let dpu_agent = dpu_agent_service( &default_dpu_agent_service(), &DpfDpuAgentBootstrapCa::default(), + ::rpc::node_jwt::NODE_JWT_AUDIENCE, ); assert!( dpu_agent @@ -872,7 +896,11 @@ mod tests { // When a pull secret is configured, the block is emitted with its name. let mut cfg = default_dpu_agent_service(); cfg.docker_image_pull_secret = Some("nico-pull-secret".to_string()); - let agent = dpu_agent_service(&cfg, &DpfDpuAgentBootstrapCa::default()); + let agent = dpu_agent_service( + &cfg, + &DpfDpuAgentBootstrapCa::default(), + ::rpc::node_jwt::NODE_JWT_AUDIENCE, + ); assert_eq!( agent.helm_values.unwrap()["imagePullSecrets"], serde_json::json!([{ "name": "nico-pull-secret" }]) diff --git a/crates/api-core/src/lib.rs b/crates/api-core/src/lib.rs index be5d1d9c29..bebfda4c16 100644 --- a/crates/api-core/src/lib.rs +++ b/crates/api-core/src/lib.rs @@ -68,6 +68,7 @@ mod machine_validation; mod measured_boot; mod mqtt_state_change_hook; mod network_segment; +mod node_auth; mod scout_stream; pub mod secrets; mod setup; diff --git a/crates/api-core/src/listener.rs b/crates/api-core/src/listener.rs index a2024a6550..73e12829da 100644 --- a/crates/api-core/src/listener.rs +++ b/crates/api-core/src/listener.rs @@ -335,8 +335,36 @@ pub async fn start( ), ))?; - let cert_description_layer: CertDescriptionMiddleware = - CertDescriptionMiddleware::new(extra_cli_certs, spiffe_context); + let cert_description_layer: CertDescriptionMiddleware = { + let machine_certs_enabled = api_service.runtime_config.node_auth.mtls_enabled; + if !machine_certs_enabled { + tracing::warn!( + target: "node_auth", + "node-auth: mtls_enabled = false: machine client certificates will NOT be \ + accepted as node identity; nodes must present bearer tokens" + ); + } + let layer = CertDescriptionMiddleware::new(extra_cli_certs, spiffe_context) + .with_machine_certs_enabled(machine_certs_enabled); + // When node-auth is enabled, accept bearer JWTs in addition to mTLS + // client certs (dual-support during the mTLS→JWT migration). Bearer + // tokens must only be accepted over TLS — never plaintext — so guard the + // authenticator on the listener actually being TLS-terminated. + match (&api_service.node_jwt_validator, tls_config.is_some()) { + (Some(node_jwt_validator), true) => { + tracing::info!(target: "node_auth", "node-auth: bearer token authentication enabled"); + layer.with_bearer_authenticator(node_jwt_validator.clone()) + } + (Some(_), false) => { + tracing::warn!( + target: "node_auth", + "node-auth: enabled but listener is not TLS; refusing to accept bearer tokens over plaintext" + ); + layer + } + (None, _) => layer, + } + }; let casbin_layer = if let Some(auth_config) = auth_config { if let Some(casbin_policy_file) = &auth_config.casbin_policy_file { let casbin_authorizer = Arc::new( @@ -406,6 +434,9 @@ pub async fn start( let mut tls_acceptor_created = Instant::now(); let mut initialize_tls_acceptor = true; + // Refreshed alongside the TLS acceptor below; both read the same client-CA + // bundle, so they must not drift apart. + let node_jwt_validator = api_service.node_jwt_validator.clone(); join_set .build_task() @@ -442,18 +473,79 @@ pub async fn start( initialize_tls_acceptor = false; tls_acceptor_created = Instant::now(); - tls_acceptor = tokio::task::Builder::new() - .name("get_tls_acceptor refresh") - .spawn_blocking({ - let tls_config = tls_config.clone(); - move || get_tls_acceptor(&tls_config) - }) - // Safety: spawn_blocking only returns Error if run outside the tokio runtime - .expect("Failed to spawn blocking task") - .await - // Safety: Awaiting a JoinHandle only fails if the task panicked, and we want to - // propagate panics - .expect("task panicked"); + // Node-auth JWTs chain to the same client-CA bundle, so the + // validator's trust anchors have to rotate on the same tick. + // Left stale it would reject tokens issued under the new CA, + // which locks nodes out entirely once mtls_enabled = false. + // + // Reload it BEFORE swapping the acceptor so the two move + // together: if the bundle is caught mid-write, neither is + // replaced and the whole pair stays on the previous anchors + // until the next tick. Swapping the acceptor first would + // leave the listener trusting a CA the token path does not. + let jwt_roots_reloaded = match node_jwt_validator.as_ref() { + None => true, + Some(node_jwt_validator) => { + let validator = node_jwt_validator.clone(); + let refreshed = tokio::task::Builder::new() + .name("node_jwt_validator refresh") + .spawn_blocking(move || validator.refresh_roots()) + .expect("Failed to spawn blocking task") + .await + .expect("task panicked"); + match refreshed { + Ok(()) => true, + Err(error) => { + tracing::warn!( + target: "node_auth", + %error, + "node-auth: could not reload JWT trust anchors; \ + keeping the previous TLS and token trust anchors" + ); + false + } + } + } + }; + + if jwt_roots_reloaded { + let refreshed_acceptor = tokio::task::Builder::new() + .name("get_tls_acceptor refresh") + .spawn_blocking({ + let tls_config = tls_config.clone(); + move || get_tls_acceptor(&tls_config) + }) + // Safety: spawn_blocking only returns Error if run outside the tokio runtime + .expect("Failed to spawn blocking task") + .await + // Safety: Awaiting a JoinHandle only fails if the task panicked, and we want to + // propagate panics + .expect("task panicked"); + + // `get_tls_acceptor` yields `None` for any failure — + // an identity PEM caught mid-write by cert-manager, an + // unreadable key, a CA bundle with nothing parsable in + // it. Assigning that straight through would drop the + // listener onto the plaintext branch below while the + // bearer authenticator, installed once at startup on + // the premise that this listener terminates TLS, keeps + // accepting node JWTs — putting them on the wire in the + // clear. Keep the working acceptor and retry instead: a + // stale-but-valid one beats no TLS, and rotation leaves + // ample overlap to pick the new material up. + match refreshed_acceptor { + Some(acceptor) => tls_acceptor = Some(acceptor), + None => { + // Retry on the next connection rather than + // waiting out another five-minute window. + initialize_tls_acceptor = true; + tracing::error!( + "could not rebuild the TLS acceptor; \ + keeping the previous one and retrying" + ); + } + } + } } let tls_acceptor = tls_acceptor.clone(); diff --git a/crates/api-core/src/node_auth.rs b/crates/api-core/src/node_auth.rs new file mode 100644 index 0000000000..5731cba36b --- /dev/null +++ b/crates/api-core/src/node_auth.rs @@ -0,0 +1,480 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Node-auth: validation of self-signed bearer JWTs from Scout / DPU-agent +//! (issue NVIDIA/infra-controller#355, simple variant). +//! +//! Nodes sign short-lived ES256 JWTs with the private key of their EXISTING +//! mTLS client certificate and carry the certificate chain in the token's +//! `x5c` header. [`NodeJwtValidator`] verifies, in order: +//! +//! 1. the `x5c` chain against the same root CAs the TLS listener trusts for +//! client certificates (chain of trust, validity window, client-auth EKU); +//! 2. the JWT signature against the verified leaf's public key (algorithm +//! pinned to ES256 — the only key type Vault PKI issues to machines); +//! 3. the registered claims: `exp` (with a bounded lifetime), `aud`; +//! 4. the SPIFFE constraints on the leaf and that the token's `sub` matches +//! the leaf's SPIFFE URI SAN — identity always derives from the verified +//! certificate, never from an attacker-controlled claim. +//! +//! The resulting SPIFFE URI is mapped by the authn middleware through the +//! SAME `SpiffeContext` as mTLS client certs, so a JWT and a cert for the +//! same machine yield an identical principal and reuse the existing RBAC +//! unchanged. There is no server-side key material and no issuance path: +//! "public key exchange" is the existing certificate PKI. + +use std::sync::{Arc, RwLock}; + +use carbide_authn::middleware::BearerTokenAuthenticator; +use jsonwebtoken::{Algorithm, DecodingKey, Validation, decode, decode_header}; +use rustls::RootCertStore; +use rustls::server::WebPkiClientVerifier; +use rustls::server::danger::ClientCertVerifier; +use rustls_pki_types::{CertificateDer, UnixTime}; +use serde::Deserialize; +use x509_parser::prelude::{FromDer, X509Certificate}; + +use crate::cfg::file::NodeAuthConfig; + +#[derive(Debug, thiserror::Error)] +pub enum NodeAuthError { + #[error("could not read root CA file {path}: {error}")] + RootCaRead { path: String, error: std::io::Error }, + #[error("root CA file {path} contains no usable trust anchors")] + NoTrustAnchors { path: String }, + #[error("could not build certificate verifier: {0}")] + Verifier(String), +} + +/// Why a presented bearer token was rejected. Only ever logged at debug — +/// rejection simply means the request proceeds without a bearer principal. +#[derive(Debug, thiserror::Error)] +enum RejectReason { + #[error("malformed JWT: {0}")] + Malformed(jsonwebtoken::errors::Error), + #[error("unexpected algorithm {0:?}; only ES256 is accepted")] + Algorithm(Algorithm), + #[error("no x5c certificate chain in the JWT header")] + NoChain, + #[error("x5c chain did not verify against the trusted roots: {0}")] + Chain(rustls::Error), + #[error("leaf certificate is not an EC (P-256) certificate")] + NotEcCertificate, + #[error("signature/claims validation failed: {0}")] + Claims(jsonwebtoken::errors::Error), + #[error("token lifetime exceeds the allowed maximum")] + Lifetime, + #[error("leaf certificate fails SPIFFE validation: {0}")] + Spiffe(String), + #[error("token `sub` does not match the certificate's SPIFFE URI")] + SubjectMismatch, + #[error("system clock is before the UNIX epoch")] + Clock, +} + +/// Registered claims checked on node tokens. `iat` is required so the bounded +/// lifetime check (`exp - iat`) cannot be dodged by omitting it. +#[derive(Debug, Deserialize)] +struct NodeClaims { + sub: String, + iat: u64, + exp: u64, +} + +/// Validates node-auth JWTs against the client-certificate PKI. +pub struct NodeJwtValidator { + /// Kept so the trust anchors can be re-read when the bundle rotates. + root_cafile_path: String, + /// Swapped in place by [`NodeJwtValidator::refresh_roots`]. The validator + /// is shared (the authn layer holds one `Arc` for the process lifetime), + /// so the refresh has to be interior, not a rebuild of the whole struct. + /// Held behind a lock rather than an `ArcSwap` because `arc-swap` cannot + /// store an unsized `dyn ClientCertVerifier`; the guard is released before + /// the (comparatively expensive) chain verification runs. + cert_verifier: RwLock>, + validation: Validation, + max_token_ttl_sec: u64, +} + +impl NodeJwtValidator { + /// Builds a validator trusting the given root CA bundle — the same file + /// the TLS listener uses to verify mTLS client certificates. + pub fn from_root_ca_file( + root_cafile_path: &str, + cfg: &NodeAuthConfig, + ) -> Result { + let cert_verifier = Self::build_verifier(root_cafile_path)?; + + let mut validation = Validation::new(Algorithm::ES256); + validation.set_audience(&[&cfg.audience]); + validation.set_required_spec_claims(&["exp", "sub", "aud", "iat"]); + + Ok(Self { + root_cafile_path: root_cafile_path.to_string(), + cert_verifier: RwLock::new(cert_verifier), + validation, + max_token_ttl_sec: u64::from(cfg.max_token_ttl_sec), + }) + } + + /// Re-reads the root CA bundle from disk and swaps in a verifier built + /// from it. + /// + /// The TLS listener reloads the same file every five minutes to pick up + /// cert-manager rotations; without this the validator would keep its + /// startup snapshot and start rejecting tokens whose `x5c` chains to the + /// new CA — which, with `mtls_enabled = false`, locks nodes out until the + /// API restarts. On error the existing verifier is left in place: a + /// half-written or briefly unreadable bundle should not disarm node auth. + pub fn refresh_roots(&self) -> Result<(), NodeAuthError> { + let cert_verifier = Self::build_verifier(&self.root_cafile_path)?; + *self + .cert_verifier + .write() + .unwrap_or_else(|poisoned| poisoned.into_inner()) = cert_verifier; + Ok(()) + } + + fn build_verifier( + root_cafile_path: &str, + ) -> Result, NodeAuthError> { + let pem = std::fs::read(root_cafile_path).map_err(|error| NodeAuthError::RootCaRead { + path: root_cafile_path.to_string(), + error, + })?; + let mut roots = RootCertStore::empty(); + let certs = rustls_pemfile::certs(&mut std::io::Cursor::new(&pem[..])) + .collect::, _>>() + .map_err(|e| NodeAuthError::Verifier(format!("root CA parse error: {e}")))?; + let (added, _ignored) = roots.add_parsable_certificates(certs); + if added == 0 { + return Err(NodeAuthError::NoTrustAnchors { + path: root_cafile_path.to_string(), + }); + } + + WebPkiClientVerifier::builder_with_provider( + Arc::new(roots), + Arc::new(rustls::crypto::aws_lc_rs::default_provider()), + ) + .allow_unknown_revocation_status() + .build() + .map_err(|e| NodeAuthError::Verifier(e.to_string())) + } + + fn validate(&self, token: &str) -> Result { + let header = decode_header(token).map_err(RejectReason::Malformed)?; + if header.alg != Algorithm::ES256 { + return Err(RejectReason::Algorithm(header.alg)); + } + + // 1. The certificate chain must verify against the trusted roots. + let chain = header + .x5c_der() + .map_err(RejectReason::Malformed)? + .filter(|chain| !chain.is_empty()) + .ok_or(RejectReason::NoChain)?; + let leaf = CertificateDer::from(chain[0].clone()); + let intermediates: Vec = chain[1..] + .iter() + .map(|der| CertificateDer::from(der.clone())) + .collect(); + let cert_verifier = self + .cert_verifier + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone(); + cert_verifier + .verify_client_cert(&leaf, &intermediates, UnixTime::now()) + .map_err(RejectReason::Chain)?; + + // 2. The token must be signed by the verified leaf's key. + let (_, x509) = + X509Certificate::from_der(leaf.as_ref()).map_err(|_| RejectReason::NotEcCertificate)?; + let decoding_key = DecodingKey::from_ec_der(&x509.public_key().subject_public_key.data); + let claims = decode::(token, &decoding_key, &self.validation) + .map_err(RejectReason::Claims)? + .claims; + + // 3. Bounded lifetime: the client controls `exp`, so cap how far in + // the future it may reach. `jsonwebtoken` already rejected expired + // tokens above. + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(|_| RejectReason::Clock)? + .as_secs(); + if claims.exp.saturating_sub(claims.iat) > self.max_token_ttl_sec + || claims.exp > now + self.max_token_ttl_sec + { + return Err(RejectReason::Lifetime); + } + + // 4. Identity comes from the verified certificate, with `sub` only + // cross-checked against it. + let spiffe_id = carbide_authn::validate_x509_certificate(leaf.as_ref()) + .map_err(|e| RejectReason::Spiffe(e.to_string()))?; + let spiffe_uri = spiffe_id.to_string(); + if claims.sub != spiffe_uri { + return Err(RejectReason::SubjectMismatch); + } + Ok(spiffe_uri) + } +} + +impl BearerTokenAuthenticator for NodeJwtValidator { + fn spiffe_id_from_bearer(&self, token: &str) -> Option { + match self.validate(token) { + Ok(spiffe_uri) => Some(spiffe_uri), + Err(reason) => { + tracing::debug!(target: "node_auth", %reason, "node-auth: rejected bearer token"); + None + } + } + } +} + +#[cfg(test)] +mod tests { + use rpc::node_jwt::NodeJwtMinter; + + use super::*; + + const TRUST_DOMAIN: &str = "forge.local"; + const MACHINE_PATH: &str = "/forge-system/machine/fm100xtest"; + + struct TestPki { + ca_pem: String, + cert_pem: String, + key_pem: String, + } + + /// A CA plus a leaf it issued carrying the machine SPIFFE URI SAN — + /// stand-ins for the Vault PKI root and a node's client certificate. + fn test_pki(spiffe_path: &str) -> TestPki { + let mut ca_params = rcgen::CertificateParams::default(); + ca_params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + ca_params + .distinguished_name + .push(rcgen::DnType::CommonName, "test root"); + let ca_key = rcgen::KeyPair::generate().expect("ca key"); + let ca_cert = ca_params.clone().self_signed(&ca_key).expect("ca cert"); + let issuer = rcgen::Issuer::new(ca_params, ca_key); + + let mut leaf_params = rcgen::CertificateParams::default(); + leaf_params.subject_alt_names = vec![rcgen::SanType::URI( + rcgen::string::Ia5String::try_from(format!("spiffe://{TRUST_DOMAIN}{spiffe_path}")) + .expect("uri"), + )]; + leaf_params.use_authority_key_identifier_extension = true; + leaf_params + .extended_key_usages + .push(rcgen::ExtendedKeyUsagePurpose::ClientAuth); + let leaf_key = rcgen::KeyPair::generate().expect("leaf key"); + let leaf_cert = leaf_params + .signed_by(&leaf_key, &issuer) + .expect("leaf cert"); + + TestPki { + ca_pem: ca_cert.pem(), + cert_pem: leaf_cert.pem(), + key_pem: leaf_key.serialize_pem(), + } + } + + fn write_temp(dir: &tempfile::TempDir, name: &str, contents: &str) -> String { + let path = dir.path().join(name); + std::fs::write(&path, contents).expect("write"); + path.to_string_lossy().into_owned() + } + + fn validator_for(dir: &tempfile::TempDir, ca_pem: &str) -> NodeJwtValidator { + let ca_path = write_temp(dir, "ca.pem", ca_pem); + NodeJwtValidator::from_root_ca_file(&ca_path, &NodeAuthConfig::default()) + .expect("validator builds") + } + + fn mint_with(dir: &tempfile::TempDir, pki: &TestPki) -> String { + let minter = NodeJwtMinter::new( + write_temp(dir, "cert.pem", &pki.cert_pem), + write_temp(dir, "key.pem", &pki.key_pem), + ); + minter.current().expect("token minted") + } + + /// `[node_auth] audience` is configurable, so a site that changes it must + /// still work end to end: the minter has to stamp the configured value and + /// the validator has to accept it (and nothing else). + #[test] + fn a_configured_audience_round_trips_and_excludes_the_default() { + let dir = tempfile::tempdir().expect("tempdir"); + let pki = test_pki(MACHINE_PATH); + let ca_path = write_temp(&dir, "ca.pem", &pki.ca_pem); + let cfg = NodeAuthConfig { + audience: "carbide-api-eu".to_string(), + ..NodeAuthConfig::default() + }; + let validator = + NodeJwtValidator::from_root_ca_file(&ca_path, &cfg).expect("validator builds"); + + let cert_path = write_temp(&dir, "cert.pem", &pki.cert_pem); + let key_path = write_temp(&dir, "key.pem", &pki.key_pem); + + let matching = NodeJwtMinter::with_audience( + cert_path.clone(), + key_path.clone(), + "carbide-api-eu".to_string(), + ) + .current() + .expect("token minted"); + assert_eq!( + validator.spiffe_id_from_bearer(&matching).as_deref(), + Some(format!("spiffe://{TRUST_DOMAIN}{MACHINE_PATH}").as_str()), + "a token minted for the configured audience must be accepted" + ); + + let default_aud = NodeJwtMinter::new(cert_path, key_path) + .current() + .expect("token minted"); + assert_eq!( + validator.spiffe_id_from_bearer(&default_aud), + None, + "the default audience must not be accepted once one is configured" + ); + } + + /// A rotated client-CA bundle has to be picked up without a restart: the + /// TLS listener re-reads the same file every five minutes, and tokens + /// chaining to the new CA must start verifying once it does. + #[test] + fn rotated_root_ca_is_honored_after_refresh() { + let dir = tempfile::tempdir().expect("tempdir"); + let old_pki = test_pki(MACHINE_PATH); + let ca_path = write_temp(&dir, "ca.pem", &old_pki.ca_pem); + let validator = NodeJwtValidator::from_root_ca_file(&ca_path, &NodeAuthConfig::default()) + .expect("validator builds"); + + // A second CA, as though cert-manager had rotated the bundle. + let new_pki = test_pki(MACHINE_PATH); + let new_dir = tempfile::tempdir().expect("tempdir"); + let token = mint_with(&new_dir, &new_pki); + assert_eq!( + validator.spiffe_id_from_bearer(&token), + None, + "a token from the not-yet-trusted CA must be rejected" + ); + + std::fs::write(&ca_path, &new_pki.ca_pem).expect("rotate the bundle on disk"); + validator.refresh_roots().expect("roots reload"); + + assert_eq!( + validator.spiffe_id_from_bearer(&token).as_deref(), + Some(format!("spiffe://{TRUST_DOMAIN}{MACHINE_PATH}").as_str()), + "after the refresh the rotated CA must be trusted" + ); + } + + /// A bundle that cannot be read or parsed must not disarm bearer auth. + #[test] + fn a_broken_bundle_leaves_the_previous_roots_in_place() { + let dir = tempfile::tempdir().expect("tempdir"); + let pki = test_pki(MACHINE_PATH); + let ca_path = write_temp(&dir, "ca.pem", &pki.ca_pem); + let validator = NodeJwtValidator::from_root_ca_file(&ca_path, &NodeAuthConfig::default()) + .expect("validator builds"); + let token = mint_with(&dir, &pki); + + std::fs::write(&ca_path, "not a certificate").expect("truncate the bundle"); + validator + .refresh_roots() + .expect_err("a bundle with no trust anchors must not be accepted"); + + assert_eq!( + validator.spiffe_id_from_bearer(&token).as_deref(), + Some(format!("spiffe://{TRUST_DOMAIN}{MACHINE_PATH}").as_str()), + "the previous roots must still verify tokens" + ); + } + + #[test] + fn client_minted_token_round_trips_to_the_cert_spiffe_uri() { + let dir = tempfile::tempdir().expect("tempdir"); + let pki = test_pki(MACHINE_PATH); + let validator = validator_for(&dir, &pki.ca_pem); + + let token = mint_with(&dir, &pki); + assert_eq!( + validator.spiffe_id_from_bearer(&token).as_deref(), + Some(format!("spiffe://{TRUST_DOMAIN}{MACHINE_PATH}").as_str()) + ); + } + + #[test] + fn token_from_an_untrusted_ca_is_rejected() { + let dir = tempfile::tempdir().expect("tempdir"); + let pki = test_pki(MACHINE_PATH); + let other_pki = test_pki(MACHINE_PATH); + // Validator trusts a DIFFERENT root than the one that issued the cert. + let validator = validator_for(&dir, &other_pki.ca_pem); + + let token = mint_with(&dir, &pki); + assert!(validator.spiffe_id_from_bearer(&token).is_none()); + } + + #[test] + fn garbage_and_missing_chain_tokens_are_rejected() { + let dir = tempfile::tempdir().expect("tempdir"); + let pki = test_pki(MACHINE_PATH); + let validator = validator_for(&dir, &pki.ca_pem); + + assert!(validator.spiffe_id_from_bearer("not.a.jwt").is_none()); + + // Structurally valid ES256 JWT without an x5c header. + let key = rcgen::KeyPair::generate().expect("key"); + let encoding_key = jsonwebtoken::EncodingKey::from_ec_pem(key.serialize_pem().as_bytes()) + .expect("encoding key"); + let claims = serde_json::json!({ + "sub": format!("spiffe://{TRUST_DOMAIN}{MACHINE_PATH}"), + "aud": "nico-api", "iat": 0u64, "exp": u64::MAX / 2, + }); + let no_chain = jsonwebtoken::encode( + &jsonwebtoken::Header::new(Algorithm::ES256), + &claims, + &encoding_key, + ) + .expect("token"); + assert!(validator.spiffe_id_from_bearer(&no_chain).is_none()); + } + + #[test] + fn overlong_lifetime_is_rejected() { + let dir = tempfile::tempdir().expect("tempdir"); + let pki = test_pki(MACHINE_PATH); + let ca_path = write_temp(&dir, "ca.pem", &pki.ca_pem); + // A validator whose lifetime cap is below what the client mints. + let strict = NodeJwtValidator::from_root_ca_file( + &ca_path, + &NodeAuthConfig { + max_token_ttl_sec: 1, + ..NodeAuthConfig::default() + }, + ) + .expect("validator builds"); + + let token = mint_with(&dir, &pki); + assert!(strict.spiffe_id_from_bearer(&token).is_none()); + } +} diff --git a/crates/api-core/src/setup.rs b/crates/api-core/src/setup.rs index 96f88f2418..0b9aeb1631 100644 --- a/crates/api-core/src/setup.rs +++ b/crates/api-core/src/setup.rs @@ -406,6 +406,36 @@ pub(crate) async fn start_runtime( .map_err(|e| eyre::eyre!("failed to build NMX-C client pool: {e}"))?; let shared_nmxc_pool: Arc = Arc::new(nmxc_client_pool); + // Node-auth (Scout / DPU-agent bearer JWT, #355) preflight. Run before any + // DPF resource creation below, so a misconfiguration (the + // enabled=false + mtls_enabled=false lockout, bearer-over-plaintext, or an + // unreadable trust anchor) fails startup before it mutates cluster state. + // Validate unconditionally; when explicitly enabled, missing prerequisites + // fail rather than silently degrading. + carbide_config.node_auth.validate()?; + let node_jwt_validator = if carbide_config.node_auth.enabled { + // Bearer tokens must never be accepted over plaintext, and the + // validator trusts the same roots the TLS listener uses for client + // certificates — so a TLS listener is required on both counts. + if !matches!(carbide_config.listen_mode, ListenMode::Tls) { + return Err(eyre::eyre!( + "[node_auth] is enabled but listen_mode is not \"tls\"; bearer tokens must not be accepted over plaintext" + )); + } + let tls_ref = carbide_config + .tls + .as_ref() + .ok_or_else(|| eyre::eyre!("[node_auth] is enabled but [tls] is unset"))?; + Some(Arc::new( + crate::node_auth::NodeJwtValidator::from_root_ca_file( + &tls_ref.root_cafile_path, + &carbide_config.node_auth, + )?, + )) + } else { + None + }; + let dpf_sdk = initialize_dpf_sdk( &carbide_config, credential_manager.clone(), @@ -465,6 +495,7 @@ pub(crate) async fn start_runtime( certificate_provider, common_pools, credential_manager, + node_jwt_validator, database_connection: db_pool.clone(), dpu_health_log_limiter: LogLimiter::default(), dynamic_settings, @@ -610,6 +641,7 @@ async fn initialize_dpf_sdk( services: crate::dpf_services::mandatory_services( &services, &carbide_config.dpf.dpu_agent_bootstrap_ca, + &carbide_config.node_auth, ), proxy: carbide_config.dpf.proxy.clone(), deployment_type, diff --git a/crates/api-core/src/test_support/builder.rs b/crates/api-core/src/test_support/builder.rs index ab871269c6..a8246a0070 100644 --- a/crates/api-core/src/test_support/builder.rs +++ b/crates/api-core/src/test_support/builder.rs @@ -289,6 +289,7 @@ impl TestApiBuilder { bmc_session_manager, bms_client: std::sync::OnceLock::new(), secrets_context: self.secrets_context, + node_jwt_validator: None, } } } diff --git a/crates/api-core/src/test_support/default_config.rs b/crates/api-core/src/test_support/default_config.rs index 86f38b5bdf..04321483ec 100644 --- a/crates/api-core/src/test_support/default_config.rs +++ b/crates/api-core/src/test_support/default_config.rs @@ -95,6 +95,7 @@ pub fn get() -> CarbideConfig { enable_admin_ui: true, web_ui_sidebar_tools: vec![], log_history: Default::default(), + node_auth: Default::default(), observability: Default::default(), bgp_leaf_session_password: None, rack_validation_config: RackValidationConfig { diff --git a/crates/authn/Cargo.toml b/crates/authn/Cargo.toml index f7784741b7..1e86d09c9a 100644 --- a/crates/authn/Cargo.toml +++ b/crates/authn/Cargo.toml @@ -42,6 +42,7 @@ carbide-instrument = { path = "../instrument", features = ["test-support"] } carbide-rpc = { path = "../rpc" } carbide-test-support = { path = "../test-support" } rcgen = { workspace = true } +tokio = { workspace = true } [lints] workspace = true diff --git a/crates/authn/src/middleware.rs b/crates/authn/src/middleware.rs index 9dd44cb5e6..1216d95810 100644 --- a/crates/authn/src/middleware.rs +++ b/crates/authn/src/middleware.rs @@ -36,10 +36,33 @@ use crate::{SpiffeContext, SpiffeError}; // This middleware is not expected to enforce anything on its own, so anything // that an access control policy might need to do its work should be passed // along in the request extensions. +/// Verifies a bearer token (a node-auth JWT) and, on success, returns the +/// validated SPIFFE ID URI string from the token's `sub` claim. The +/// implementation lives outside this crate (api-core's `node_auth`) so the +/// authn layer stays free of JWT signing/key-management concerns. +/// +/// Returning the raw SPIFFE URI — rather than a `Principal` — lets the +/// middleware map it through the SAME [`SpiffeContext`] used for client certs, +/// so a JWT and an mTLS cert for the same machine yield identical principals. +pub trait BearerTokenAuthenticator: Send + Sync { + /// Returns the validated SPIFFE ID URI if `token` is a valid node-auth JWT, + /// or `None` if it is invalid/expired/unrecognized. + fn spiffe_id_from_bearer(&self, token: &str) -> Option; +} + #[derive(Clone)] pub struct CertDescriptionMiddleware { pub spiffe_context: Arc, pub extra_allowed_certs: Option, + /// Optional bearer-token (JWT) authenticator. When set, requests carrying an + /// `Authorization: Bearer ` header gain principals alongside any from + /// mTLS client certs (dual-support during the mTLS→JWT migration). + pub bearer_authenticator: Option>, + /// Whether machine mTLS client certificates mint machine principals. On by + /// default; disabled via `[node_auth] mtls_enabled = false` once a fleet has + /// migrated to bearer tokens. Scoped to machine certs — service and admin + /// client certs are unaffected. + pub machine_certs_enabled: bool, _authorization: std::marker::PhantomData, } @@ -51,9 +74,29 @@ impl CertDescriptionMiddleware { CertDescriptionMiddleware { spiffe_context: Arc::new(spiffe_context), extra_allowed_certs, + bearer_authenticator: None, + machine_certs_enabled: true, _authorization: std::marker::PhantomData, } } + + /// Enables bearer-token (JWT) authentication using the given authenticator. + #[must_use] + pub fn with_bearer_authenticator( + mut self, + authenticator: Arc, + ) -> Self { + self.bearer_authenticator = Some(authenticator); + self + } + + /// Controls whether machine mTLS client certificates are accepted as node + /// identity (`[node_auth] mtls_enabled`). + #[must_use] + pub fn with_machine_certs_enabled(mut self, enabled: bool) -> Self { + self.machine_certs_enabled = enabled; + self + } } impl Layer for CertDescriptionMiddleware { @@ -218,6 +261,12 @@ impl Principal { } } +/// Extracts the token from an `Authorization: Bearer ` header, if present. +fn bearer_token_from_headers(headers: &hyper::HeaderMap) -> Option<&str> { + let value = headers.get(hyper::header::AUTHORIZATION)?.to_str().ok()?; + value.strip_prefix("Bearer ").map(str::trim) +} + // try_external_cert will return a Pricipal::ExternalUser if this looks like some external cert fn try_external_cert( der_certificate: &[u8], @@ -540,12 +589,43 @@ where } fn call(&mut self, mut request: Request) -> Self::Future { - if let Some(_req_auth_header) = request.headers().get(hyper::header::AUTHORIZATION) { - // If we want to extract additional principals from the request's - // Authorization header, we can do it here. + let mut auth_context = AuthContext::::default(); + + // Bearer-token (node-auth JWT) authentication. Validated tokens map + // through the same SpiffeContext as client certs, so a JWT and an mTLS + // cert for the same machine produce identical principals. This runs in + // addition to (not instead of) cert auth to support the migration. + if let Some(authenticator) = &self.authorization_context.bearer_authenticator + && let Some(token) = bearer_token_from_headers(request.headers()) + && let Some(spiffe_uri) = authenticator.spiffe_id_from_bearer(token) + { + match crate::spiffe_id::SpiffeId::new(&spiffe_uri) { + Ok(spiffe_id) => match self + .authorization_context + .spiffe_context + .extract_service_identifier(&spiffe_id) + { + Ok(crate::SpiffeIdClass::Service(id)) => { + auth_context + .principals + .push(Principal::SpiffeServiceIdentifier(id)); + } + Ok(crate::SpiffeIdClass::Machine(id)) => { + auth_context + .principals + .push(Principal::SpiffeMachineIdentifier(id)); + } + Err(e) => { + tracing::debug!(target: "node_auth", "node-auth: bearer token SPIFFE id not recognized: {e}"); + } + }, + Err(e) => { + tracing::debug!(target: "node_auth", "node-auth: bearer token contained an unparsable SPIFFE URI: {e}"); + } + } } + let extensions = request.extensions_mut(); - let mut auth_context = AuthContext::::default(); if let Some(conn_attrs) = extensions.get::>() { let peer_certs = &conn_attrs.peer_certificates; // rustls presents the end-entity certificate first, intermediates @@ -556,15 +636,34 @@ where // per request, only when the whole chain minted no principal. let mut rejections = Vec::new(); let minted_before = auth_context.principals.len(); - let peer_cert_principals = peer_certs.iter().filter_map(|cert| { - match Principal::try_from_client_certificate(cert, &self.authorization_context) { - Ok(x) => Some(x), - Err(e) => { - rejections.push(e); - None + let peer_cert_principals = peer_certs + .iter() + .filter_map(|cert| { + match Principal::try_from_client_certificate(cert, &self.authorization_context) + { + Ok(x) => Some(x), + Err(e) => { + rejections.push(e); + None + } } - } - }); + }) + .filter(|principal| { + // `[node_auth] mtls_enabled = false`: machine certs no longer + // grant node identity (bearer JWTs are the only node auth + // path); service/admin cert principals pass through. + if !self.authorization_context.machine_certs_enabled + && matches!(principal, Principal::SpiffeMachineIdentifier(_)) + { + tracing::debug!( + target: "node_auth", + "node-auth: machine mTLS authentication disabled; ignoring machine client certificate" + ); + false + } else { + true + } + }); auth_context.principals.extend(peer_cert_principals); if auth_context.principals.len() == minted_before && let Some(leaf_error) = rejections.first() @@ -593,10 +692,13 @@ where #[cfg(test)] mod tests { use std::collections::HashSet; + use std::convert::Infallible; use std::sync::Mutex; use std::task::{Context, Poll}; use carbide_instrument::testing::{MetricsCapture, capture_logs}; + use hyper::header::AUTHORIZATION; + use tower::{Layer, ServiceExt}; use super::*; use crate::spiffe_id::TrustDomain; @@ -661,6 +763,14 @@ mod tests { } } + /// Test authenticator: maps the literal token "good" to a fixed SPIFFE URI. + struct FakeAuth(String); + impl BearerTokenAuthenticator for FakeAuth { + fn spiffe_id_from_bearer(&self, token: &str) -> Option { + (token == "good").then(|| self.0.clone()) + } + } + fn spiffe_context() -> SpiffeContext { SpiffeContext { trust_domain: TrustDomain::new("example.test").expect("trust domain"), @@ -840,4 +950,162 @@ mod tests { "recognition" ); } + + /// Runs a request through the middleware and returns the principals the authn + /// layer attached (captured from the request extensions by an inner service). + /// + /// The request carries `ConnectionAttributes` with no peer certificates — + /// a bearer-token client over plain TLS. Omitting them entirely is the + /// wiring-error path, which logs `authentication_connection_attributes_missing` + /// and would poison that callsite's cached tracing interest for the + /// capture-based test of that warning. + async fn principals_for( + middleware: CertDescriptionMiddleware, + auth_header: Option<&str>, + ) -> Vec { + principals_for_certs(middleware, auth_header, Vec::new()).await + } + + #[tokio::test] + async fn valid_bearer_token_yields_machine_principal() { + let middleware = CertDescriptionMiddleware::::new(None, spiffe_context()) + .with_bearer_authenticator(Arc::new(FakeAuth( + "spiffe://example.test/carbide-system/machine/m1".to_string(), + ))); + let principals = principals_for(middleware, Some("Bearer good")).await; + assert!( + principals.contains(&Principal::SpiffeMachineIdentifier("m1".to_string())), + "expected machine principal, got {principals:?}" + ); + } + + #[tokio::test] + async fn invalid_bearer_token_yields_no_machine_principal() { + let middleware = CertDescriptionMiddleware::::new(None, spiffe_context()) + .with_bearer_authenticator(Arc::new(FakeAuth( + "spiffe://example.test/carbide-system/machine/m1".to_string(), + ))); + let principals = principals_for(middleware, Some("Bearer bogus")).await; + assert!( + !principals + .iter() + .any(|p| matches!(p, Principal::SpiffeMachineIdentifier(_))), + "rejected token must not yield a machine principal, got {principals:?}" + ); + } + + /// Like [`principals_for`], but the request also presents client + /// certificates via [`ConnectionAttributes`]. + async fn principals_for_certs( + middleware: CertDescriptionMiddleware, + auth_header: Option<&str>, + peer_certificates: Vec>, + ) -> Vec { + let inner = tower::service_fn(|req: Request| async move { + let principals = req + .extensions() + .get::>() + .map(|ctx| ctx.principals.clone()) + .unwrap_or_default(); + Ok::<_, Infallible>(principals) + }); + let svc = middleware.layer(inner); + let mut builder = Request::builder(); + if let Some(value) = auth_header { + builder = builder.header(AUTHORIZATION, value); + } + let mut request = builder.body(tonic::body::Body::empty()).unwrap(); + request + .extensions_mut() + .insert(Arc::new(ConnectionAttributes { + peer_address: "192.0.2.9:4433".parse().expect("socket address"), + peer_certificates, + })); + svc.oneshot(request).await.unwrap() + } + + #[tokio::test] + async fn machine_cert_yields_machine_principal_by_default() { + let middleware = CertDescriptionMiddleware::::new(None, spiffe_context()); + let principals = principals_for_certs( + middleware, + None, + vec![spiffe_leaf_certificate("/carbide-system/machine/m1")], + ) + .await; + assert!( + principals.contains(&Principal::SpiffeMachineIdentifier("m1".to_string())), + "expected cert-derived machine principal, got {principals:?}" + ); + } + + #[tokio::test] + async fn machine_cert_ignored_when_machine_certs_disabled() { + // `[node_auth] mtls_enabled = false`: a machine client cert mints no + // machine principal, while a valid bearer token still does — the JWT + // becomes the only node auth path. + let middleware = CertDescriptionMiddleware::::new(None, spiffe_context()) + .with_machine_certs_enabled(false) + .with_bearer_authenticator(Arc::new(FakeAuth( + "spiffe://example.test/carbide-system/machine/m2".to_string(), + ))); + let principals = principals_for_certs( + middleware, + Some("Bearer good"), + vec![spiffe_leaf_certificate("/carbide-system/machine/m1")], + ) + .await; + assert!( + !principals.contains(&Principal::SpiffeMachineIdentifier("m1".to_string())), + "cert-derived machine principal must be dropped, got {principals:?}" + ); + assert!( + principals.contains(&Principal::SpiffeMachineIdentifier("m2".to_string())), + "bearer-derived machine principal must survive, got {principals:?}" + ); + } + + #[tokio::test] + async fn service_cert_unaffected_when_machine_certs_disabled() { + // The gate is scoped to machine certs: service identities on the same + // listener keep authenticating via mTLS. + let middleware = CertDescriptionMiddleware::::new(None, spiffe_context()) + .with_machine_certs_enabled(false); + let principals = principals_for_certs( + middleware, + None, + vec![spiffe_leaf_certificate("/carbide-system/sa/test-service")], + ) + .await; + assert!( + principals + .iter() + .any(|p| matches!(p, Principal::SpiffeServiceIdentifier(_))), + "service cert principal must survive, got {principals:?}" + ); + } + + #[tokio::test] + async fn bearer_ignored_when_no_authenticator_configured() { + // Without an authenticator installed, an Authorization header is ignored + // (nodes keep using mTLS) — no machine principal appears. + let middleware = CertDescriptionMiddleware::::new(None, spiffe_context()); + let principals = principals_for(middleware, Some("Bearer good")).await; + assert!( + !principals + .iter() + .any(|p| matches!(p, Principal::SpiffeMachineIdentifier(_))), + "no authenticator => no bearer principals, got {principals:?}" + ); + } + + #[test] + fn bearer_token_from_headers_parses_scheme() { + let mut headers = hyper::HeaderMap::new(); + assert_eq!(bearer_token_from_headers(&headers), None); + headers.insert(AUTHORIZATION, "Bearer abc.def".parse().unwrap()); + assert_eq!(bearer_token_from_headers(&headers), Some("abc.def")); + headers.insert(AUTHORIZATION, "Basic xyz".parse().unwrap()); + assert_eq!(bearer_token_from_headers(&headers), None); + } } diff --git a/crates/fmds/src/cfg.rs b/crates/fmds/src/cfg.rs index 2bdd5049dd..3ca6e7b93f 100644 --- a/crates/fmds/src/cfg.rs +++ b/crates/fmds/src/cfg.rs @@ -59,6 +59,13 @@ pub struct Options { #[clap(long)] pub client_key: Option, + /// Path of the dpu-agent's local API socket. When set, fmds fetches + /// short-lived node-auth bearer JWTs from the agent and presents them to + /// carbide-api — instead of (or in addition to) the mTLS client cert — + /// so this pod no longer needs the machine private key mounted. + #[clap(long, env = "FMDS_NODE_TOKEN_SOCKET")] + pub node_token_socket: Option, + /// Name of the interface to assign the metadata-service address to. #[clap(long, env = "FMDS_INTERFACE_NAME", default_value = "f_pf0hpf_if")] pub interface_name: String, diff --git a/crates/fmds/src/main.rs b/crates/fmds/src/main.rs index 030f2fed2d..57fe4c2490 100644 --- a/crates/fmds/src/main.rs +++ b/crates/fmds/src/main.rs @@ -26,6 +26,7 @@ use fmds::{http_request_metrics, nic_init}; use forge_tls::client_config::ClientCert; use rpc::fmds::fmds_config_service_server::FmdsConfigServiceServer; use rpc::forge_tls_client::ForgeClientConfig; +use rpc::node_token_socket::SocketTokenSource; use tracing::metadata::LevelFilter; use tracing_subscriber::EnvFilter; use tracing_subscriber::layer::SubscriberExt as _; @@ -79,16 +80,42 @@ async fn main() -> eyre::Result<()> { nic_init::assign_address(&options.interface_name, options.interface_cidr).await?; nic_init::setup_metadata_routing(&options.interface_name, options.interface_cidr).await?; - // Build ForgeClientConfig for phone_home if cert paths are provided - let forge_client_config = match (&options.root_ca, &options.client_cert, &options.client_key) { - (Some(root_ca), Some(client_cert), Some(client_key)) => { - Some(Arc::new(ForgeClientConfig::new( - root_ca.clone(), - Some(ClientCert { - cert_path: client_cert.clone(), - key_path: client_key.clone(), - }), - ))) + // Build ForgeClientConfig for phone_home. Either credential works alone: + // the shared mTLS client cert, and/or node-auth bearer tokens fetched from + // the dpu-agent's local API socket (#355) — the latter lets this pod run + // without the machine private key mounted at all. + let client_cert = match (&options.client_cert, &options.client_key) { + (Some(cert_path), Some(key_path)) => Some(ClientCert { + cert_path: cert_path.clone(), + key_path: key_path.clone(), + }), + (None, None) => None, + // Half a credential is a deployment bug; fail loudly instead of + // silently running without mTLS. + (Some(_), None) => { + eyre::bail!("--client-cert was provided without --client-key") + } + (None, Some(_)) => { + eyre::bail!("--client-key was provided without --client-cert") + } + }; + let forge_client_config = match &options.root_ca { + Some(root_ca) if client_cert.is_some() || options.node_token_socket.is_some() => { + let mut config = ForgeClientConfig::new(root_ca.clone(), client_cert); + if let Some(socket) = &options.node_token_socket { + tracing::info!( + socket = %socket, + "fetching node-auth bearer tokens from the dpu-agent local API" + ); + // Token mode usually runs without a client cert, which would + // otherwise leave the channel on the dummy TLS verifier. The + // bearer token is the client credential; the server still has + // to prove itself against the root CA. + config = config + .require_tls_enforcement() + .with_token_provider(SocketTokenSource::spawn(socket.clone())); + } + Some(Arc::new(config)) } _ => { tracing::warn!( diff --git a/crates/host-support/src/agent_config.rs b/crates/host-support/src/agent_config.rs index 1de547d89a..c292a2113f 100644 --- a/crates/host-support/src/agent_config.rs +++ b/crates/host-support/src/agent_config.rs @@ -118,6 +118,36 @@ pub struct ForgeSystemConfig { pub client_cert: String, #[serde(default = "default_client_key")] pub client_key: String, + /// Unix socket where the agent serves its local API (node tokens for + /// co-located services, issue #355). Works the same containerized (DPF) + /// and as a plain service on DPU OS; override when `/opt/forge` is not + /// the shared credential directory in a deployment. + #[serde(default = "default_local_api_socket")] + pub local_api_socket: String, + /// `aud` stamped on node-auth bearer JWTs (issue #355). Must match the + /// API's `[node_auth] audience`; a site that changes one must change the + /// other, or the API rejects every token this node mints. + #[serde(default = "default_node_auth_audience")] + pub node_auth_audience: String, +} + +impl ForgeSystemConfig { + /// Rejects values that would leave the node unable to authenticate. + /// + /// The `--node-auth-audience` flag validates itself at parse time, but the + /// TOML path had no equivalent, so a blank or whitespace-only audience in + /// a config file reached the minter and produced tokens the API rejects on + /// every request — the silent lockout the audience plumbing exists to + /// prevent. Callers run this after applying any CLI override so both + /// sources are held to the same rule. + pub fn validate(&self) -> Result<(), String> { + if self.node_auth_audience.trim().is_empty() { + return Err( + "forge-system.node-auth-audience: must not be empty or whitespace-only".to_string(), + ); + } + Ok(()) + } } // Called if no `[forge-system]` is provided at all. @@ -129,6 +159,8 @@ impl Default for ForgeSystemConfig { root_ca: default_root_ca(), client_cert: default_client_cert(), client_key: default_client_key(), + local_api_socket: default_local_api_socket(), + node_auth_audience: default_node_auth_audience(), } } } @@ -149,6 +181,14 @@ pub fn default_client_key() -> String { tls_default::default_client_key().to_string() } +pub fn default_local_api_socket() -> String { + ::rpc::node_token_socket::DEFAULT_AGENT_LOCAL_SOCKET.to_string() +} + +pub fn default_node_auth_audience() -> String { + ::rpc::node_jwt::NODE_JWT_AUDIENCE.to_string() +} + #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub struct MachineConfig { @@ -1073,4 +1113,43 @@ interface-id = \"91609f10-c91d-470d-a260-6293ea0c1200\" fs::read_to_string(format!("{TEST_DATA_DIR}/min_agent_config/output.toml")).unwrap(); assert_eq!(observed_output, expected_output); } + + /// The `--node-auth-audience` flag rejects a blank value at parse time; the + /// TOML path has to hold the same line. A blank audience mints tokens the + /// API rejects on every request, which is the silent lockout the whole + /// audience plumbing exists to prevent. + #[test] + fn forge_system_config_rejects_a_blank_node_auth_audience() { + assert!( + ForgeSystemConfig::default().validate().is_ok(), + "the default audience must be valid" + ); + + for blank in ["", " ", "\t"] { + let config = ForgeSystemConfig { + node_auth_audience: blank.to_string(), + ..ForgeSystemConfig::default() + }; + let err = config + .validate() + .expect_err("a blank audience must be rejected"); + assert!( + err.contains("node-auth-audience"), + "the error should name the field, got: {err}" + ); + } + } + + /// A config file that omits the key entirely still gets the default, so + /// validation must not turn an ordinary minimal config into a hard failure. + #[test] + fn a_config_without_an_audience_key_still_validates() { + let config: AgentConfig = toml::from_str( + fs::read_to_string(format!("{TEST_DATA_DIR}/min_agent_config/input.toml")) + .unwrap() + .as_str(), + ) + .unwrap(); + assert!(config.forge_system.validate().is_ok()); + } } diff --git a/crates/host-support/src/registration.rs b/crates/host-support/src/registration.rs index 7c642233eb..e2085137c1 100644 --- a/crates/host-support/src/registration.rs +++ b/crates/host-support/src/registration.rs @@ -312,7 +312,7 @@ pub async fn write_certs( ))?; tracing::info!(%client_cert, "Wrote new machine certificate PEM"); - tokio::fs::write(client_key, machine_certificate.private_key.as_slice()) + write_private_key(client_key, machine_certificate.private_key.as_slice()) .await .wrap_err(format!( "failed to write new machine certificate key to: {client_key}" @@ -323,3 +323,109 @@ pub async fn write_certs( Ok(()) } + +/// Writes the machine private key owner-readable only (0600). The key is +/// created with the restrictive mode from the first byte — not chmod'ed after +/// the fact — so there is no window where it sits world-readable; a pre-existing +/// key file from an older agent (written 0644 via plain `fs::write`) is +/// tightened as well. Everything that legitimately reads the key (dpu-agent, +/// scout, fmds, otelcol) runs as root, so 0600 root-owned loses nobody access. +#[cfg(unix)] +async fn write_private_key(path: &str, key: &[u8]) -> Result<(), std::io::Error> { + use std::os::unix::fs::PermissionsExt; + + use tokio::io::AsyncWriteExt; + + let mut file = tokio::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .mode(0o600) + .open(path) + .await?; + // `mode` only applies at creation; tighten files that already existed. + file.set_permissions(std::fs::Permissions::from_mode(0o600)) + .await?; + file.write_all(key).await?; + file.flush().await +} + +#[cfg(not(unix))] +async fn write_private_key(path: &str, key: &[u8]) -> Result<(), std::io::Error> { + tokio::fs::write(path, key).await +} + +#[cfg(all(test, unix))] +mod tests { + use std::os::unix::fs::PermissionsExt; + + use super::*; + + /// `write_certs` concatenates and writes bytes without parsing them, so + /// these are opaque sentinels rather than PEM. Real-looking `-----BEGIN EC + /// PRIVATE KEY-----` markers would buy nothing here and trip secret + /// scanners, which match the marker and not its contents. + fn test_certificate() -> MachineCertificate { + MachineCertificate { + public_key: b"test-leaf-certificate-bytes".to_vec(), + issuing_ca: b"test-issuing-ca-bytes".to_vec(), + private_key: b"test-machine-private-key-bytes".to_vec(), + } + } + + fn key_mode(path: &str) -> u32 { + std::fs::metadata(path) + .expect("key metadata") + .permissions() + .mode() + & 0o777 + } + + #[tokio::test] + async fn private_key_is_written_owner_only() { + let dir = tempfile::tempdir().expect("tempdir"); + let paths = ClientCert { + cert_path: dir.path().join("cert.pem").to_string_lossy().into_owned(), + key_path: dir.path().join("cert.key").to_string_lossy().into_owned(), + }; + + write_certs(Some(test_certificate()), Some(&paths)) + .await + .expect("write_certs succeeds"); + + assert_eq!(key_mode(&paths.key_path), 0o600, "fresh key must be 0600"); + let cert = std::fs::read_to_string(&paths.cert_path).expect("cert readable"); + assert!( + cert.contains("leaf") && cert.contains("ca"), + "cert file carries leaf + chain" + ); + } + + #[tokio::test] + async fn pre_existing_loose_key_is_tightened_on_rewrite() { + let dir = tempfile::tempdir().expect("tempdir"); + let paths = ClientCert { + cert_path: dir.path().join("cert.pem").to_string_lossy().into_owned(), + key_path: dir.path().join("cert.key").to_string_lossy().into_owned(), + }; + // A key written by an older agent via plain fs::write (umask default). + std::fs::write(&paths.key_path, b"old key").expect("seed old key"); + std::fs::set_permissions(&paths.key_path, std::fs::Permissions::from_mode(0o644)) + .expect("loosen"); + + write_certs(Some(test_certificate()), Some(&paths)) + .await + .expect("write_certs succeeds"); + + assert_eq!( + key_mode(&paths.key_path), + 0o600, + "renewal must tighten an old 0644 key" + ); + let key = std::fs::read_to_string(&paths.key_path).expect("key readable by owner"); + assert!( + key.contains("test-machine-private-key-bytes"), + "key content replaced" + ); + } +} diff --git a/crates/host-support/test/min_agent_config/output.toml b/crates/host-support/test/min_agent_config/output.toml index 3c5db0e597..33a5fafed5 100644 --- a/crates/host-support/test/min_agent_config/output.toml +++ b/crates/host-support/test/min_agent_config/output.toml @@ -3,6 +3,8 @@ api-server = "https://127.0.0.1:8001" root-ca = "/opt/forge/forge_root.pem" client-cert = "/opt/forge/machine_cert.pem" client-key = "/opt/forge/machine_cert.key" +local-api-socket = "/opt/forge/run/agent.sock" +node-auth-audience = "nico-api" [machine] interface-id = "91609f10-c91d-470d-a260-6293ea0c1234" diff --git a/crates/rpc/Cargo.toml b/crates/rpc/Cargo.toml index afd92b72d7..eee3aed98f 100644 --- a/crates/rpc/Cargo.toml +++ b/crates/rpc/Cargo.toml @@ -62,6 +62,8 @@ carbide-uuid = { path = "../uuid" } chrono = { features = ["serde"], workspace = true } const_format = { workspace = true } eyre = { workspace = true } +jsonwebtoken = { workspace = true, features = ["rust_crypto"] } +p256 = { workspace = true } hyper = { workspace = true, features = ["full"] } hyper-rustls = { workspace = true, features = ["http2"] } hyper-util = { features = [ @@ -127,6 +129,8 @@ carbide-api-model = { path = "../api-model", features = ["test-support"] } carbide-libmlx-model = { path = "../libmlx-model", features = ["test-support"] } carbide-test-support = { path = "../test-support" } criterion = { workspace = true } +rcgen = { workspace = true } +tempfile = { workspace = true } [[bench]] name = "machine_convert" diff --git a/crates/rpc/build.rs b/crates/rpc/build.rs index e6afbffd34..ada07c5929 100644 --- a/crates/rpc/build.rs +++ b/crates/rpc/build.rs @@ -1098,6 +1098,7 @@ fn main() -> Result<(), Box> { "proto/site_explorer.proto", "proto/dns.proto", "proto/fmds.proto", + "proto/agent_local.proto", ], &["proto"], ) diff --git a/crates/rpc/proto/agent_local.proto b/crates/rpc/proto/agent_local.proto new file mode 100644 index 0000000000..f3d9822a20 --- /dev/null +++ b/crates/rpc/proto/agent_local.proto @@ -0,0 +1,50 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +package agent_local; + +// Local (on-DPU) API served by forge-dpu-agent over a unix domain socket +// (the agent's `local-api-socket` setting, which defaults to a dedicated run/ +// subdirectory of the shared credentials directory, so consumers can mount the +// socket read-write while the credentials stay read-only). The literal default +// lives in DEFAULT_AGENT_LOCAL_SOCKET and is deliberately not repeated here: +// REST proto normalization rewrites vendor names inside comments, which would +// silently corrupt an inlined filesystem path into one that does not exist. +// This is the consolidation point for +// agent <-> co-located-service communication: services that today mount the +// machine cert/key (or synced files) to talk to nico-api can instead ask the +// agent, and future local needs should add RPCs here rather than new +// sockets, ports, or file drops. +service AgentLocal { + // Returns the current node-auth bearer JWT, minted by the agent from the + // machine's client-certificate key (issue #355). Callers present it as + // `Authorization: Bearer ` to nico-api and must fetch a fresh one + // before `expires_at`. Only the agent ever touches the private key. + rpc GetNodeToken(GetNodeTokenRequest) returns (GetNodeTokenResponse); +} + +message GetNodeTokenRequest { +} + +message GetNodeTokenResponse { + // The signed ES256 JWT. + string token = 1; + // Unix seconds when the token expires. + uint64 expires_at = 2; +} diff --git a/crates/rpc/src/forge_tls_client.rs b/crates/rpc/src/forge_tls_client.rs index fd06f71245..90d5a1de1b 100644 --- a/crates/rpc/src/forge_tls_client.rs +++ b/crates/rpc/src/forge_tls_client.rs @@ -41,6 +41,7 @@ use x509_parser::prelude::{FromDer, X509Certificate}; use crate::forge::VersionRequest; use crate::forge_resolver::resolver::ResolverError; use crate::forge_tls_client::ConfigurationError::CouldNotReadRootCa; +use crate::node_jwt::{BearerAuthService, NodeJwtMinter, NodeTokenProvider}; use crate::protos::forge::forge_client::ForgeClient; use crate::protos::nmx_c::nmx_controller_client::NmxControllerClient; use crate::{forge_resolver, protos}; @@ -96,6 +97,12 @@ pub struct ForgeClientConfig { pub socks_proxy: Option, pub connect_retries_max: Option, pub connect_retries_interval: Option, + /// Optional node-auth token provider (issue #355). When set, each request + /// carries an `Authorization: Bearer ` — either self-signed with the + /// client certificate's own private key ([`NodeJwtMinter`]) or fetched + /// from the dpu-agent's local API (`SocketTokenSource`). Independent of + /// mTLS: the channel may present a client cert, a token, or both. + pub node_token_provider: Option>, } impl ForgeClientConfig { @@ -134,9 +141,66 @@ impl ForgeClientConfig { // MR though, I think. connect_retries_max: Some(3), connect_retries_interval: Some(Duration::from_secs(20)), + node_token_provider: None, } } + /// Restores server-certificate validation on a config built without a + /// client certificate. + /// + /// [`ForgeClientConfig::new`] disables TLS enforcement whenever + /// `client_cert` is `None`, which predates node-auth: back then "no client + /// cert" meant "no credentials at all", and such callers were not expected + /// to reach the API over verified TLS. A node-token client does hold + /// credentials — a bearer JWT brokered by the dpu-agent (issue #355) — and + /// must still authenticate the server against `root_ca_path` rather than + /// fall back to [`DummyTlsVerifier`]. Callers that never opt in keep the + /// previous behavior. + /// + /// `DISABLE_TLS_ENFORCEMENT` continues to win, so local-development + /// overrides work the same as they do for mTLS clients. + #[must_use] + pub fn require_tls_enforcement(mut self) -> Self { + self.enforce_tls = std::env::var("DISABLE_TLS_ENFORCEMENT").is_err(); + self + } + + /// Enables node-auth JWTs: requests built from this config mint and carry + /// short-lived bearer tokens signed with the configured client cert's key. + /// A no-op when no client cert is configured. + /// + /// `audience` must match the API's `[node_auth] audience` — the server + /// rejects any other value outright, so a client pinned to the default + /// would fail every request at a site that changed it. + #[must_use] + pub fn with_node_jwt(mut self, audience: String) -> Self { + self.node_token_provider = self.client_cert.as_ref().map(|client_cert| { + NodeJwtMinter::with_audience( + client_cert.cert_path.clone(), + client_cert.key_path.clone(), + audience, + ) as Arc + }); + self + } + + /// Attaches an explicit node-auth token provider — e.g. a pre-built + /// [`NodeJwtMinter`] the caller also serves through the agent's local + /// API, or a `SocketTokenSource` in a process that holds no key at all. + /// + /// Implies [`require_tls_enforcement`](Self::require_tls_enforcement). + /// A bearer token IS the client's credential, so the server must always + /// be authenticated before one is handed over — a token-only config + /// (no client cert) would otherwise sit on [`DummyTlsVerifier`] and + /// present its token to whatever answered. Enforcing it here rather than + /// asking every caller to remember the pairing keeps the type hard to + /// misuse; `DISABLE_TLS_ENFORCEMENT` still wins for local development. + #[must_use] + pub fn with_token_provider(mut self, provider: Arc) -> Self { + self.node_token_provider = Some(provider); + self.require_tls_enforcement() + } + /// This is required when using `ForgeTlsConfig` on a DPU to communicate with site-controller. /// The mgmt interface exists in the mgmt VRF. `use_mgmt_vrf` sets the /// `SO_BINDTODEVICE` socket option on the client socket used when performing DNS queries @@ -431,6 +495,22 @@ impl<'a> ForgeTlsClient<'a> { error: e, })?; + // `enforce_tls` governs how a TLS handshake is verified, but the + // handshake only happens for an https:// URL — a plaintext one skips + // TLS setup entirely while `BearerAuthService` below still stamps the + // token, putting a live credential on the wire in the clear. Refuse + // that combination outright. `DISABLE_TLS_ENFORCEMENT` is honored for + // parity with the rest of this config so local development still works. + if self.forge_client_config.node_token_provider.is_some() + && uri.scheme() != Some(&tonic::codegen::http::uri::Scheme::HTTPS) + && std::env::var("DISABLE_TLS_ENFORCEMENT").is_err() + { + return Err(ConfigurationError::BearerTokenOverPlaintext { + uri_string: url.as_ref().to_string(), + } + .into()); + } + let connector = self.build_https_client(url.as_ref()).await?; // ping interval + ping timeout should add up to less than tcp_user_timeout, @@ -448,6 +528,13 @@ impl<'a> ForgeTlsClient<'a> { .pool_max_idle_per_host(2) .timer(TokioTimer::new()) .build(connector); + // Stamp a freshly-minted node-auth bearer token onto each request when + // configured (issue #355). A `None` minter is a transparent pass-through, + // so the boxed service type is identical in both modes. + let hyper_client = BearerAuthService::new( + hyper_client, + self.forge_client_config.node_token_provider.clone(), + ); // Inject the issuing span's W3C trace context into every request this client sends // (issue #2438). Wrapping before `boxed_clone` keeps the erased `BoxCloneService` type. let hyper_client = trace_propagation::TraceInjectService::new(hyper_client).boxed_clone(); @@ -744,6 +831,11 @@ pub enum ConfigurationError { }, #[error("could not read root CA cert at {path}: {error}")] CouldNotReadRootCa { path: String, error: io::Error }, + #[error( + "refusing to send node-auth bearer tokens to non-HTTPS URL {uri_string}: \ + the token would travel in cleartext" + )] + BearerTokenOverPlaintext { uri_string: String }, #[error("invalid client cert: {0}")] InvalidClientCert(rustls::Error), #[error("error configuring resolver: {0}")] @@ -769,6 +861,103 @@ mod tests { use super::*; + /// A node-token client presents no client certificate, which by itself + /// drops the channel onto `DummyTlsVerifier`. `require_tls_enforcement` + /// puts it back on root-CA validation. + #[test] + fn require_tls_enforcement_restores_validation_without_a_client_cert() { + if std::env::var("DISABLE_TLS_ENFORCEMENT").is_ok() { + // The override wins by design; the assertions below would not hold. + return; + } + + let config = ForgeClientConfig::new("/etc/carbide/root-ca.pem".to_string(), None); + assert!( + !config.enforce_tls, + "a config without a client cert starts out unenforced" + ); + + let config = config.require_tls_enforcement(); + assert!( + config.enforce_tls, + "opting in must restore server certificate validation" + ); + } + + #[derive(Debug)] + struct FixedToken; + + impl NodeTokenProvider for FixedToken { + fn current(&self) -> Option { + Some("a.b.c".to_string()) + } + } + + /// The bearer token IS the credential, so the server must be authenticated + /// before one is sent. Callers must not have to remember to pair + /// `with_token_provider` with `require_tls_enforcement` — forgetting it + /// would hand tokens to whatever answered the connection. + #[test] + fn with_token_provider_enforces_tls_without_being_asked() { + if std::env::var("DISABLE_TLS_ENFORCEMENT").is_ok() { + // The override wins by design; the assertion below would not hold. + return; + } + + let config = ForgeClientConfig::new("/etc/carbide/root-ca.pem".to_string(), None) + .with_token_provider(Arc::new(FixedToken)); + + assert!( + config.enforce_tls, + "attaching a token provider must restore server validation by itself" + ); + } + + /// TLS setup only runs for an https:// URL, so `enforce_tls` alone does not + /// cover a plaintext one — the token would still be stamped and travel in + /// the clear. Building such a client has to fail. + #[tokio::test] + async fn bearer_tokens_are_refused_over_plaintext() { + if std::env::var("DISABLE_TLS_ENFORCEMENT").is_ok() { + // The override deliberately permits plaintext development. + return; + } + + let config = ForgeClientConfig::new("/etc/carbide/root-ca.pem".to_string(), None) + .with_token_provider(Arc::new(FixedToken)); + + let error = ForgeTlsClient::new(&config) + .build("http://carbide-api.local:8080") + .await + .expect_err("a token client must refuse a plaintext endpoint"); + + assert!( + error.to_string().contains("cleartext"), + "the error should explain the refusal, got: {error}" + ); + } + + /// The same client over https:// must still build — the guard above is + /// scoped to the scheme, not to token clients in general. + #[tokio::test] + async fn bearer_tokens_are_allowed_over_https() { + let config = ForgeClientConfig::new("/etc/carbide/root-ca.pem".to_string(), None) + .with_token_provider(Arc::new(FixedToken)); + + // Reaching a root-CA read means the plaintext guard let it through; + // the file itself need not exist in a unit test. + let result = ForgeTlsClient::new(&config) + .build("https://carbide-api.local:8080") + .await; + + if let Err(error) = result { + assert!( + !error.to_string().contains("cleartext"), + "https must not trip the plaintext guard, got: {error}" + ); + } + } + #[tokio::test] // test_max_retries builds up an instance of hyper client using // the ForgeHttpConnector, which is the same configuration used diff --git a/crates/rpc/src/lib.rs b/crates/rpc/src/lib.rs index 40b3beb4ce..be1140d95f 100644 --- a/crates/rpc/src/lib.rs +++ b/crates/rpc/src/lib.rs @@ -74,13 +74,15 @@ pub use crate::protos::machine_discovery::{ self, BlockDevice, Cpu, DiscoveryInfo, DmiData, NetworkInterface, NvmeDevice, PciDeviceProperties, }; -pub use crate::protos::{fmds, health, scout_firmware_upgrade, site_explorer}; +pub use crate::protos::{agent_local, fmds, health, scout_firmware_upgrade, site_explorer}; pub mod errors; pub mod forge_tls_client; pub mod libmlx; pub mod measured_boot; pub mod network; +pub mod node_jwt; +pub mod node_token_socket; pub mod protos; pub mod secrets; mod site_explorer_report; diff --git a/crates/rpc/src/node_jwt.rs b/crates/rpc/src/node_jwt.rs new file mode 100644 index 0000000000..eb6f5dd9b2 --- /dev/null +++ b/crates/rpc/src/node_jwt.rs @@ -0,0 +1,479 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Client-side node-auth JWT minting (issue NVIDIA/infra-controller#355). +//! +//! The node self-signs short-lived ES256 JWTs with the private key of its +//! EXISTING mTLS client certificate; the certificate chain rides along in the +//! token's `x5c` header (RFC 7515 §4.1.6) so the API can verify it against the +//! same root CA its TLS listener already trusts. No new key material, no key +//! storage, and no refresh RPC: a fresh token is minted locally whenever the +//! cached one nears expiry, and key "rotation" happens for free when the +//! client certificate renews. +//! +//! [`BearerAuthService`] is the tower middleware that stamps the current token +//! onto each outgoing request's `Authorization` header. Minting is best-effort: +//! if the cert/key files are missing or unreadable (e.g. before first +//! registration), requests simply carry no bearer header and the channel's +//! mTLS client cert remains the only credential. + +use std::io::Cursor; +use std::sync::{Arc, RwLock}; +use std::task::{Context, Poll}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use data_encoding::BASE64; +use jsonwebtoken::{Algorithm, EncodingKey, Header}; +use p256::pkcs8::{EncodePrivateKey, LineEnding}; +use serde::Serialize; +use tower::Service; +use x509_parser::prelude::{FromDer, GeneralName, X509Certificate}; + +/// `aud` claim stamped on minted tokens; must match the API's +/// `[node_auth] audience`. +pub const NODE_JWT_AUDIENCE: &str = "nico-api"; + +/// Lifetime of minted tokens. Deliberately short: tokens cost nothing to +/// re-mint locally, so a leaked one ages out in minutes. +pub const NODE_JWT_TTL_SECS: u64 = 300; + +/// A cached token is reused until it has less than this long left, then +/// re-minted. Comfortably above per-request latency, comfortably below TTL. +const REMINT_MARGIN_SECS: u64 = 60; + +#[derive(Debug, thiserror::Error)] +pub enum NodeJwtError { + #[error("could not read client certificate or key: {0}")] + Io(#[from] std::io::Error), + #[error("client certificate file contains no parsable certificate")] + NoCertificate, + #[error("client certificate is not usable for node JWTs: {0}")] + BadCertificate(String), + #[error("client private key is not a usable EC key: {0}")] + BadKey(String), + #[error("client private key does not match the certificate's public key")] + KeyCertMismatch, + #[error("JWT signing failed: {0}")] + Sign(#[from] jsonwebtoken::errors::Error), + #[error("system clock is before the UNIX epoch")] + Clock, +} + +#[derive(Clone)] +struct CachedToken { + token: String, + expires_at: u64, +} + +/// Mints and caches node-auth JWTs from the node's existing mTLS client +/// certificate and private key files. +/// +/// Cert and key are re-read from disk on every mint, so certificate renewal +/// (which rewrites both files in place) is picked up automatically on the +/// next re-mint without any coordination. +pub struct NodeJwtMinter { + cert_path: String, + key_path: String, + /// `aud` stamped on every minted token. Must match the API's + /// `[node_auth] audience`, which is configurable — a minter pinned to the + /// default would be rejected wholesale by a site that changed it. + audience: String, + cached: RwLock>, +} + +/// Manual impl so the cached token (a live credential) never lands in debug +/// output of the client config. +impl std::fmt::Debug for NodeJwtMinter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("NodeJwtMinter") + .field("cert_path", &self.cert_path) + .field("key_path", &self.key_path) + .finish_non_exhaustive() + } +} + +/// Claims carried by a node-auth JWT. `sub` duplicates the certificate's +/// SPIFFE URI SAN; the server derives identity from the *verified certificate* +/// and only cross-checks `sub` against it, so a forged `sub` buys nothing. +#[derive(Debug, Serialize)] +struct NodeClaims<'a> { + sub: &'a str, + aud: &'a str, + iat: u64, + exp: u64, +} + +impl NodeJwtMinter { + /// Mints tokens for the default audience ([`NODE_JWT_AUDIENCE`]). Use + /// [`NodeJwtMinter::with_audience`] when the API's `[node_auth] audience` + /// has been changed from the default. + #[must_use] + pub fn new(cert_path: String, key_path: String) -> Arc { + Self::with_audience(cert_path, key_path, NODE_JWT_AUDIENCE.to_string()) + } + + #[must_use] + pub fn with_audience(cert_path: String, key_path: String, audience: String) -> Arc { + Arc::new(Self { + cert_path, + key_path, + audience, + cached: RwLock::new(None), + }) + } + + /// Returns a currently-valid token, re-minting if the cached one is + /// missing or close to expiry. Returns `None` (and logs at debug) when + /// minting is impossible — e.g. the cert/key files don't exist yet — so + /// callers degrade gracefully to mTLS-only. + pub fn current(&self) -> Option { + self.current_with_expiry().map(|(token, _)| token) + } + + /// Like [`current`](Self::current), but also returns the token's expiry + /// (unix seconds) — used by the agent's local API to tell co-located + /// consumers when to re-fetch. + pub fn current_with_expiry(&self) -> Option<(String, u64)> { + let now = unix_now().ok()?; + if let Ok(guard) = self.cached.read() + && let Some(cached) = guard.as_ref() + && cached.expires_at > now + REMINT_MARGIN_SECS + { + return Some((cached.token.clone(), cached.expires_at)); + } + match self.mint(now) { + Ok(minted) => { + let result = (minted.token.clone(), minted.expires_at); + if let Ok(mut guard) = self.cached.write() { + *guard = Some(minted); + } + Some(result) + } + Err(error) => { + tracing::debug!( + target: "node_auth", + cert_path = %self.cert_path, + %error, + "node-auth: could not mint node JWT; continuing with mTLS only" + ); + None + } + } + } + + fn mint(&self, now: u64) -> Result { + let cert_pem = std::fs::read(&self.cert_path)?; + let key_pem = std::fs::read_to_string(&self.key_path)?; + + let chain = rustls_pemfile::certs(&mut Cursor::new(&cert_pem[..])) + .collect::, _>>() + .map_err(|e| NodeJwtError::BadCertificate(e.to_string()))?; + let leaf = chain.first().ok_or(NodeJwtError::NoCertificate)?; + let sub = spiffe_uri_from_cert(leaf.as_ref())?; + + // Certificate renewal rewrites the two files in sequence, so a mint + // landing in between can pair a new certificate with the old key. + // Signing would still succeed and the API would reject the result, + // and the bad token would sit in the cache for a re-mint margin. Catch + // the mismatch here instead: returning an error means no bearer header + // on this request and a fresh attempt on the next one. + if !key_matches_certificate(&key_pem, leaf.as_ref())? { + return Err(NodeJwtError::KeyCertMismatch); + } + + let mut header = Header::new(Algorithm::ES256); + header.typ = Some("JWT".to_string()); + header.x5c = Some(chain.iter().map(|c| BASE64.encode(c.as_ref())).collect()); + + let expires_at = now + NODE_JWT_TTL_SECS; + let claims = NodeClaims { + sub: &sub, + aud: &self.audience, + iat: now, + exp: expires_at, + }; + let token = jsonwebtoken::encode(&header, &claims, &ec_encoding_key(&key_pem)?)?; + Ok(CachedToken { token, expires_at }) + } +} + +fn unix_now() -> Result { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .map_err(|_| NodeJwtError::Clock) +} + +/// Extracts the single SPIFFE URI SAN from the certificate — the same field +/// the server's authn layer maps to a machine principal for mTLS. +fn spiffe_uri_from_cert(der: &[u8]) -> Result { + let (_, cert) = X509Certificate::from_der(der) + .map_err(|e| NodeJwtError::BadCertificate(format!("X.509 parse error: {e}")))?; + let san = cert + .subject_alternative_name() + .map_err(|e| NodeJwtError::BadCertificate(format!("bad SAN extension: {e}")))? + .ok_or_else(|| NodeJwtError::BadCertificate("no SAN extension".to_string()))?; + san.value + .general_names + .iter() + .find_map(|name| match name { + GeneralName::URI(uri) if uri.starts_with("spiffe://") => Some(uri.to_string()), + _ => None, + }) + .ok_or_else(|| NodeJwtError::BadCertificate("no SPIFFE URI SAN".to_string())) +} + +/// Builds an ES256 signing key from the client key PEM. Vault-issued machine +/// keys are SEC1 (`BEGIN EC PRIVATE KEY`), which `jsonwebtoken` cannot load +/// directly, so those are re-encoded to PKCS#8 first. +/// Whether `key_pem`'s public half is the one certified by `leaf_der`. +/// +/// Both encodings the node may hold are accepted: Vault issues SEC1 ("EC +/// PRIVATE KEY") and renewal may leave PKCS#8. The comparison is on the +/// uncompressed SEC1 point, which is exactly what an EC `SubjectPublicKeyInfo` +/// carries. +fn key_matches_certificate(key_pem: &str, leaf_der: &[u8]) -> Result { + let secret = if key_pem.contains("BEGIN EC PRIVATE KEY") { + p256::SecretKey::from_sec1_pem(key_pem).map_err(|e| NodeJwtError::BadKey(e.to_string()))? + } else { + ::from_pkcs8_pem(key_pem) + .map_err(|e| NodeJwtError::BadKey(e.to_string()))? + }; + + let (_, cert) = X509Certificate::from_der(leaf_der) + .map_err(|e| NodeJwtError::BadCertificate(e.to_string()))?; + let certified = &cert.public_key().subject_public_key.data; + let from_key = { + use p256::elliptic_curve::sec1::ToSec1Point as _; + secret.public_key().to_sec1_point(false) + }; + + Ok(certified.as_ref() == from_key.as_bytes()) +} + +fn ec_encoding_key(key_pem: &str) -> Result { + if key_pem.contains("BEGIN EC PRIVATE KEY") { + let secret = p256::SecretKey::from_sec1_pem(key_pem) + .map_err(|e| NodeJwtError::BadKey(e.to_string()))?; + let pkcs8 = secret + .to_pkcs8_pem(LineEnding::LF) + .map_err(|e| NodeJwtError::BadKey(e.to_string()))?; + EncodingKey::from_ec_pem(pkcs8.as_bytes()).map_err(NodeJwtError::Sign) + } else { + EncodingKey::from_ec_pem(key_pem.as_bytes()).map_err(NodeJwtError::Sign) + } +} + +/// A source of node-auth bearer tokens for outgoing requests. Implemented by +/// [`NodeJwtMinter`] (holds the key, signs locally) and +/// [`SocketTokenSource`](crate::node_token_socket::SocketTokenSource) +/// (fetches from the dpu-agent's local API — the caller never sees the key). +/// +/// `current` runs on the request path, so implementations must be non-blocking: +/// return a cached token or `None`, never wait on I/O. +pub trait NodeTokenProvider: Send + Sync + std::fmt::Debug { + /// Returns a currently-valid token, or `None` if one isn't available + /// (the request then proceeds with whatever else the channel carries). + fn current(&self) -> Option; +} + +impl NodeTokenProvider for NodeJwtMinter { + fn current(&self) -> Option { + NodeJwtMinter::current(self) + } +} + +/// Tower middleware that injects `Authorization: Bearer ` onto each +/// request when a [`NodeTokenProvider`] is configured. A `None` provider is a +/// no-op, so the same client construction path serves both token and +/// mTLS-only modes. +#[derive(Clone)] +pub struct BearerAuthService { + inner: S, + minter: Option>, +} + +impl BearerAuthService { + pub fn new(inner: S, minter: Option>) -> Self { + Self { inner, minter } + } +} + +impl Service> for BearerAuthService +where + S: Service>, +{ + type Response = S::Response; + type Error = S::Error; + type Future = S::Future; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx) + } + + fn call(&mut self, mut req: hyper::Request) -> Self::Future { + if let Some(value) = self + .minter + .as_ref() + .and_then(|minter| minter.current()) + .and_then(|token| hyper::http::HeaderValue::from_str(&format!("Bearer {token}")).ok()) + { + req.headers_mut() + .insert(hyper::header::AUTHORIZATION, value); + } + self.inner.call(req) + } +} + +#[cfg(test)] +mod tests { + use jsonwebtoken::{DecodingKey, Validation}; + use serde::Deserialize; + + use super::*; + + #[derive(Debug, Deserialize)] + struct Claims { + sub: String, + exp: u64, + iat: u64, + } + + const SPIFFE_URI: &str = "spiffe://forge.local/forge-system/machine/fm100xtest"; + + /// Self-signed leaf with a SPIFFE URI SAN plus its PKCS#8 key PEM — + /// stand-ins for the Vault-issued client cert/key pair on a node. + fn cert_and_key() -> (String, String) { + let mut params = rcgen::CertificateParams::default(); + params.subject_alt_names = vec![rcgen::SanType::URI( + rcgen::string::Ia5String::try_from(SPIFFE_URI.to_string()).expect("uri"), + )]; + let key = rcgen::KeyPair::generate().expect("key pair"); + let cert = params.self_signed(&key).expect("certificate"); + (cert.pem(), key.serialize_pem()) + } + + fn write_temp(dir: &tempfile::TempDir, name: &str, contents: &str) -> String { + let path = dir.path().join(name); + std::fs::write(&path, contents).expect("write"); + path.to_string_lossy().into_owned() + } + + fn decode_against_own_cert(token: &str) -> Claims { + // Validate exactly as the server does: pull the leaf from x5c, take + // its SPKI EC point, verify the signature with it. + let header = jsonwebtoken::decode_header(token).expect("header"); + assert_eq!(header.alg, Algorithm::ES256); + let chain = header.x5c_der().expect("x5c decodes").expect("x5c present"); + let (_, cert) = X509Certificate::from_der(&chain[0]).expect("leaf parses"); + let decoding_key = DecodingKey::from_ec_der(&cert.public_key().subject_public_key.data); + let mut validation = Validation::new(Algorithm::ES256); + validation.set_audience(&[NODE_JWT_AUDIENCE]); + jsonwebtoken::decode::(token, &decoding_key, &validation) + .expect("token validates") + .claims + } + + /// Certificate renewal rewrites cert and key in sequence, so a mint can + /// land on a new certificate paired with the previous key. That signs + /// cleanly but the API cannot verify it, so it must never be minted or + /// cached — the next attempt, on a consistent pair, should succeed. + #[test] + fn a_certificate_from_a_different_key_is_never_minted_or_cached() { + let dir = tempfile::tempdir().expect("tempdir"); + let (cert_pem, _) = cert_and_key(); + let (_, other_key_pem) = cert_and_key(); + + let cert_path = write_temp(&dir, "cert.pem", &cert_pem); + let key_path = write_temp(&dir, "key.pem", &other_key_pem); + let minter = NodeJwtMinter::new(cert_path, key_path.clone()); + + assert!( + minter.current().is_none(), + "a token the API could not verify must not be minted" + ); + + // Renewal completes: the matching key lands. + let (matching_cert, matching_key) = cert_and_key(); + std::fs::write(&key_path, &matching_key).expect("write key"); + let cert_path = write_temp(&dir, "cert.pem", &matching_cert); + let minter = NodeJwtMinter::new(cert_path, key_path); + assert!( + minter.current().is_some(), + "a consistent cert/key pair must mint normally" + ); + } + + #[test] + fn mints_a_token_signed_by_the_client_cert_key() { + let (cert_pem, key_pem) = cert_and_key(); + let dir = tempfile::tempdir().expect("tempdir"); + let minter = NodeJwtMinter::new( + write_temp(&dir, "cert.pem", &cert_pem), + write_temp(&dir, "key.pem", &key_pem), + ); + + let token = minter.current().expect("token minted"); + let claims = decode_against_own_cert(&token); + assert_eq!(claims.sub, SPIFFE_URI); + assert_eq!(claims.exp - claims.iat, NODE_JWT_TTL_SECS); + } + + #[test] + fn sec1_key_pem_is_accepted() { + // Vault PKI hands out SEC1-encoded EC keys; re-encode the test key the + // same way and make sure minting still works. + use p256::pkcs8::DecodePrivateKey; + let (cert_pem, key_pem) = cert_and_key(); + let secret = p256::SecretKey::from_pkcs8_pem(&key_pem).expect("pkcs8 parses"); + let sec1_pem = secret + .to_sec1_pem(LineEnding::LF) + .expect("sec1 encodes") + .to_string(); + assert!(sec1_pem.contains("BEGIN EC PRIVATE KEY")); + + let dir = tempfile::tempdir().expect("tempdir"); + let minter = NodeJwtMinter::new( + write_temp(&dir, "cert.pem", &cert_pem), + write_temp(&dir, "key.pem", &sec1_pem), + ); + let token = minter.current().expect("token minted from SEC1 key"); + assert_eq!(decode_against_own_cert(&token).sub, SPIFFE_URI); + } + + #[test] + fn token_is_cached_until_near_expiry() { + let (cert_pem, key_pem) = cert_and_key(); + let dir = tempfile::tempdir().expect("tempdir"); + let minter = NodeJwtMinter::new( + write_temp(&dir, "cert.pem", &cert_pem), + write_temp(&dir, "key.pem", &key_pem), + ); + let first = minter.current().expect("token"); + let second = minter.current().expect("token"); + assert_eq!(first, second, "fresh token must be served from cache"); + } + + #[test] + fn missing_files_yield_none_not_panic() { + let minter = NodeJwtMinter::new( + "/nonexistent/cert.pem".to_string(), + "/nonexistent/key.pem".to_string(), + ); + assert!(minter.current().is_none()); + } +} diff --git a/crates/rpc/src/node_token_socket.rs b/crates/rpc/src/node_token_socket.rs new file mode 100644 index 0000000000..5754d8e5a8 --- /dev/null +++ b/crates/rpc/src/node_token_socket.rs @@ -0,0 +1,281 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Node-auth tokens for co-located services that hold no key (issue #355). +//! +//! On a DPU, only the dpu-agent holds the machine's private key; other NICo +//! pods (fmds, …) obtain bearer tokens from the agent's local API — the +//! `AgentLocal` gRPC service on a unix socket in the shared `/opt/forge` +//! directory — instead of mounting the key to do their own mTLS/minting. +//! +//! [`SocketTokenSource`] keeps a cached token fresh with a background task and +//! serves it synchronously from [`NodeTokenProvider::current`] on the request +//! path. Until the first successful fetch (e.g. the agent hasn't started or +//! registered yet), requests simply go out without a bearer header. + +use std::sync::{Arc, RwLock, Weak}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use hyper_util::rt::TokioIo; +use tokio::net::UnixStream; +use tonic::transport::{Channel, Endpoint, Uri}; +use tower::service_fn; + +use crate::node_jwt::NodeTokenProvider; +use crate::protos::agent_local::GetNodeTokenRequest; +use crate::protos::agent_local::agent_local_client::AgentLocalClient; + +/// Default path of the dpu-agent's local API socket. It lives in a dedicated +/// `run/` subdirectory of the shared `/opt/forge` credentials directory so a +/// containerized consumer can mount just that subdirectory read-write — +/// `connect(2)` needs write access to the socket inode — while the +/// credentials themselves stay on a read-only mount. +pub const DEFAULT_AGENT_LOCAL_SOCKET: &str = "/opt/forge/run/agent.sock"; + +/// The background task re-fetches when less than this long remains on the +/// cached token. The request path keeps serving the cached token down to +/// HALF this margin — the refresher normally replaces it well before that — +/// so a briefly-late refresh doesn't strip requests of their header. +const REFRESH_MARGIN_SECS: u64 = 60; + +/// Delay between fetch attempts while the agent socket is absent or erroring, +/// and the per-attempt deadline for a fetch (connect + RPC) so a stalled +/// handshake can't wedge the refresh loop. +const RETRY_DELAY: Duration = Duration::from_secs(5); + +pub struct SocketTokenSource { + socket_path: String, + cached: RwLock>, +} + +/// Manual impl so the cached bearer token (a live credential) never lands in +/// debug output of the client config, which is itself `Debug`-logged. Mirrors +/// the redaction on [`NodeJwtMinter`](crate::node_jwt::NodeJwtMinter). +impl std::fmt::Debug for SocketTokenSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SocketTokenSource") + .field("socket_path", &self.socket_path) + .finish_non_exhaustive() + } +} + +impl SocketTokenSource { + /// Creates the source and spawns its refresh loop on the current tokio + /// runtime. The loop holds only a weak reference, so dropping the last + /// `Arc` (and the clients built from it) shuts the loop down. + #[must_use] + pub fn spawn(socket_path: String) -> Arc { + let source = Arc::new(Self { + socket_path, + cached: RwLock::new(None), + }); + tokio::spawn(refresh_loop(Arc::downgrade(&source))); + source + } + + async fn fetch(socket_path: &str) -> Result<(String, u64), tonic::Status> { + // Deadline over connect + RPC together: a peer that accepts the + // connection but never answers must fail the attempt, not wedge the + // refresh loop forever. + tokio::time::timeout(RETRY_DELAY, async { + let channel = connect_uds(socket_path) + .await + .map_err(|e| tonic::Status::unavailable(format!("agent local socket: {e}")))?; + let response = AgentLocalClient::new(channel) + .get_node_token(GetNodeTokenRequest {}) + .await? + .into_inner(); + Ok((response.token, response.expires_at)) + }) + .await + .map_err(|_| tonic::Status::deadline_exceeded("agent local API fetch timed out"))? + } +} + +impl NodeTokenProvider for SocketTokenSource { + fn current(&self) -> Option { + let now = unix_now()?; + self.cached + .read() + .ok()? + .as_ref() + .filter(|(_, expires_at)| *expires_at > now + REFRESH_MARGIN_SECS / 2) + .map(|(token, _)| token.clone()) + } +} + +async fn connect_uds(socket_path: &str) -> Result { + let socket_path = socket_path.to_owned(); + // The URI is required by the Endpoint API but unused for UDS connections. + Endpoint::try_from("http://[::]:50051")? + .connect_with_connector(service_fn(move |_: Uri| { + let path = socket_path.clone(); + async move { + let stream = UnixStream::connect(path).await?; + Ok::<_, std::io::Error>(TokioIo::new(stream)) + } + })) + .await +} + +async fn refresh_loop(source: Weak) { + loop { + // Re-upgrade each iteration so the loop exits once every user of the + // source is gone, instead of pinning it alive forever. + let Some(source) = source.upgrade() else { + return; + }; + let socket_path = source.socket_path.clone(); + match SocketTokenSource::fetch(&socket_path).await { + Ok((token, expires_at)) => { + if let Ok(mut guard) = source.cached.write() { + *guard = Some((token, expires_at)); + } + drop(source); + let sleep_secs = unix_now() + .map(|now| { + expires_at + .saturating_sub(now) + .saturating_sub(REFRESH_MARGIN_SECS) + }) + .unwrap_or(0) + .max(RETRY_DELAY.as_secs()); + tokio::time::sleep(Duration::from_secs(sleep_secs)).await; + } + Err(status) => { + tracing::debug!( + target: "node_auth", + socket = %socket_path, + %status, + "node-auth: could not fetch node token from agent local API; will retry" + ); + drop(source); + tokio::time::sleep(RETRY_DELAY).await; + } + } + } +} + +fn unix_now() -> Option { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .ok() +} + +#[cfg(test)] +mod tests { + use tokio_stream::wrappers::UnixListenerStream; + use tonic::{Request, Response, Status}; + + use super::*; + use crate::protos::agent_local::GetNodeTokenResponse; + use crate::protos::agent_local::agent_local_server::{AgentLocal, AgentLocalServer}; + + struct FixedToken(String, u64); + + #[tonic::async_trait] + impl AgentLocal for FixedToken { + async fn get_node_token( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(GetNodeTokenResponse { + token: self.0.clone(), + expires_at: self.1, + })) + } + } + + fn serve(socket: &std::path::Path, token: &str, expires_at: u64) { + let listener = tokio::net::UnixListener::bind(socket).expect("bind uds"); + let service = AgentLocalServer::new(FixedToken(token.to_string(), expires_at)); + tokio::spawn( + tonic::transport::Server::builder() + .add_service(service) + .serve_with_incoming(UnixListenerStream::new(listener)), + ); + } + + async fn wait_for_token(source: &SocketTokenSource) -> Option { + for _ in 0..100 { + if let Some(token) = source.current() { + return Some(token); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + None + } + + #[tokio::test] + async fn fetches_token_from_agent_socket() { + let dir = tempfile::tempdir().expect("tempdir"); + let socket = dir.path().join("agent.sock"); + let expires_at = unix_now().expect("clock") + 300; + serve(&socket, "the.node.token", expires_at); + + let source = SocketTokenSource::spawn(socket.to_string_lossy().into_owned()); + assert_eq!( + wait_for_token(&source).await.as_deref(), + Some("the.node.token") + ); + } + + #[tokio::test] + async fn missing_socket_yields_none_and_no_panic() { + let source = SocketTokenSource::spawn("/nonexistent/agent.sock".to_string()); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!(source.current().is_none()); + } + + #[tokio::test] + async fn debug_output_redacts_the_cached_token() { + let dir = tempfile::tempdir().expect("tempdir"); + let socket = dir.path().join("agent.sock"); + serve( + &socket, + "super.secret.token", + unix_now().expect("clock") + 300, + ); + + let source = SocketTokenSource::spawn(socket.to_string_lossy().into_owned()); + wait_for_token(&source).await.expect("token cached"); + + // ForgeClientConfig is Debug-logged, so a cached token must never + // appear in the provider's debug output. + let rendered = format!("{source:?}"); + assert!( + !rendered.contains("super.secret.token"), + "cached token leaked into Debug: {rendered}" + ); + } + + #[tokio::test] + async fn expired_cached_token_is_not_served() { + let dir = tempfile::tempdir().expect("tempdir"); + let socket = dir.path().join("agent.sock"); + // The agent hands out a token that is already (nearly) expired. + serve(&socket, "stale.token", unix_now().expect("clock")); + + let source = SocketTokenSource::spawn(socket.to_string_lossy().into_owned()); + tokio::time::sleep(Duration::from_millis(200)).await; + assert!( + source.current().is_none(), + "a token inside the refresh margin must not be presented" + ); + } +} diff --git a/crates/rpc/src/protos/mod.rs b/crates/rpc/src/protos/mod.rs index c187ab59bb..6c266dbfc2 100644 --- a/crates/rpc/src/protos/mod.rs +++ b/crates/rpc/src/protos/mod.rs @@ -106,6 +106,12 @@ pub mod fmds { include!(concat!(env!("OUT_DIR"), "/fmds.rs")); } +#[allow(non_snake_case, unknown_lints, clippy::all)] +#[rustfmt::skip] +pub mod agent_local { + include!(concat!(env!("OUT_DIR"), "/agent_local.rs")); +} + #[allow(clippy::all, deprecated)] #[rustfmt::skip] pub mod forge_api_client { diff --git a/crates/scout/src/cfg/command_line.rs b/crates/scout/src/cfg/command_line.rs index 686c9524bd..f3739a58d7 100644 --- a/crates/scout/src/cfg/command_line.rs +++ b/crates/scout/src/cfg/command_line.rs @@ -36,6 +36,16 @@ impl std::fmt::Display for Mode { } } } +/// Rejects an empty or whitespace-only audience at parse time. A blank value +/// would otherwise reach the minter and produce tokens the API cannot match, +/// with nothing in the logs pointing at the flag. +fn non_blank_audience(value: &str) -> Result { + if value.trim().is_empty() { + return Err("node-auth audience must not be empty".to_string()); + } + Ok(value.to_string()) +} + #[derive(Clone, Parser)] #[clap(name = env!("CARGO_BIN_NAME"))] pub(crate) struct Options { @@ -84,6 +94,14 @@ pub(crate) struct Options { )] pub client_key: String, + #[clap( + long, + value_parser = non_blank_audience, + help = "Audience claim stamped on node-auth bearer JWTs; must match the API's [node_auth] audience", + default_value_t = ::rpc::node_jwt::NODE_JWT_AUDIENCE.to_string(), + )] + pub node_auth_audience: String, + // Combined with discovery_retries_max, the default of 60 // seconds worth of discovery_retry_secs provides for 1 // week worth of minutely retries. diff --git a/crates/scout/src/client.rs b/crates/scout/src/client.rs index 437dc215ce..833e5f22e7 100644 --- a/crates/scout/src/client.rs +++ b/crates/scout/src/client.rs @@ -30,7 +30,10 @@ pub(crate) async fn create_forge_client( cert_path: config.client_cert.clone(), key_path: config.client_key.clone(), }), - ); + ) + // Node-auth (#355): also present a self-signed bearer JWT minted from the + // client cert's key. Ignored by the API unless [node_auth] is enabled. + .with_node_jwt(config.node_auth_audience.clone()); let api_config = ApiConfig::new(&config.api, &client_config); let client = forge_tls_client::ForgeTlsClient::retry_build(&api_config) diff --git a/deploy/nico-base/api/config-files/nico-api-config.toml b/deploy/nico-base/api/config-files/nico-api-config.toml index 6b096ede25..ae2f92408e 100644 --- a/deploy/nico-base/api/config-files/nico-api-config.toml +++ b/deploy/nico-base/api/config-files/nico-api-config.toml @@ -43,6 +43,24 @@ hardware_health_reports = "MonitorOnly" enabled = true create_machines = true +[node_auth] +# Node (Scout / DPU-agent) authentication to the API. +# See docs/design/machine-identity/node-auth-jwt.md. +# Bearer JWTs: off by default. When enabled, the API accepts short-lived +# `Authorization: Bearer` tokens that nodes self-sign with their existing +# mTLS client-certificate key (the cert rides in the token's x5c header and +# is verified against the same root CA as the TLS listener). +enabled = false +# Machine mTLS client-certificate authentication: on by default. Disable only +# after the fleet presents bearer tokens (requires enabled = true; the API +# refuses to start with both mechanisms off). Service/admin-CLI certs are +# unaffected. +mtls_enabled = true +# `aud` claim required on presented tokens. +#audience = "nico-api" +# Maximum accepted token lifetime in seconds (nodes mint 300 s tokens). +#max_token_ttl_sec = 900 + [firmware_global] autoupdate = true diff --git a/docs/design/machine-identity/node-auth-jwt.md b/docs/design/machine-identity/node-auth-jwt.md new file mode 100644 index 0000000000..1bfbbea860 --- /dev/null +++ b/docs/design/machine-identity/node-auth-jwt.md @@ -0,0 +1,439 @@ +# Node-Auth: Self-Signed Bearer JWTs for Scout and DPU-Agent + +Design for [#355](https://github.com/NVIDIA/infra-controller/issues/355) +(sub-issue of the Vault-elimination epic +[#195](https://github.com/NVIDIA/infra-controller/issues/195)): Scout and the +DPU-agent authenticate to the API with short-lived bearer JWTs alongside — +and eventually instead of — mTLS client certificates. + +Nodes sign their own tokens with the private key of their **existing** mTLS +client certificate. There is no new key material anywhere, no server-side +signing key or key storage, no issuance or refresh RPCs, and no dependency on +the `machine_identity` (tenant JWT-SVID) subsystem. On a DPU only the +dpu-agent ever holds that key; co-located NICo pods get finished tokens from +it over a local unix socket (see [Key distribution on DPF](#key-distribution-on-dpf-the-agent-is-the-token-broker)). +Approaches that were weighed and rejected are recorded under +[Designs not used](#designs-not-used). + +It is distinct from the tenant-facing [SPIFFE JWT-SVID design](spiffe-svid-sdd.md): +that issues identity tokens *to tenant workloads* via IMDS; this design covers +how *NICo's own node agents* authenticate to the NICo API. + +## How it works + +```text +node (scout / dpu-agent) nico-api +------------------------ -------- +has /opt/forge/machine_cert.pem ── x5c ──► 1. verify x5c chain against the +and /opt/forge/machine_cert.key client-cert root CA (same roots + as the TLS listener) +mint ES256 JWT signed with the 2. verify JWT signature with the +cert's own key, 5-min TTL, verified leaf's public key +cert chain in the x5c header 3. enforce exp / iat / aud, bounded + lifetime +attach as Authorization: Bearer 4. SPIFFE-validate the leaf; map its +on every gRPC request URI SAN through the same + SpiffeContext as mTLS certs + ⇒ identical machine principal, RBAC + unchanged +``` + +Key-less co-located services (fmds) never touch the cert or key at all: they +ask the dpu-agent for its current token over a unix socket and attach that +same header. + +```text +fmds pod dpu-agent nico-api +-------- --------- -------- +no key, no cert ──socket──► holds the key, ── same bearer token ──► +GetNodeToken mints as above (identical +(cached, refreshed verification) + in the background) +``` + +## Auth flow: new DPU → first authorized gRPC call + +**Provisioning & bootstrap (no credentials yet)** + +- A new DPU is provisioned (BFB installed via DPF); the `forge-dpu-agent` DPF + service starts on it with only two auth-relevant inputs from its config: the + API endpoint and the root CA bundle (`forge_system.root_ca`). The cert/key + files at `/opt/forge/machine_cert.pem` / `machine_cert.key` don't exist yet. +- The agent opens a TLS connection to `nico-api` that is **server-auth only** + — it verifies the API's cert against the root CA but presents no client + credential (the API listener uses `allow_unauthenticated()`, so the + connection is accepted with an anonymous principal). + +**First credential: the machine certificate** + +- The agent calls `DiscoverMachine` + (`host-support/registration.rs::register_machine`) carrying its hardware + enumeration (`DiscoveryInfo`); this RPC is reachable pre-credential by + design. +- The API registers/matches the machine and returns a `machine_certificate` + in the response: a Vault-PKI-issued EC P-256 leaf whose SAN is the machine's + SPIFFE URI (`spiffe:////machine/`), plus the + issuing CA and private key. (On attestation-enabled sites, hosts get this + via `AttestQuote` after a TPM challenge instead; DPUs take the discovery + path.) +- `write_certs` persists leaf+issuing-CA to `/opt/forge/machine_cert.pem` and + the key to `/opt/forge/machine_cert.key`. **This existing cert key is the + JWT signing key — nothing else is ever created.** + +**Minting the JWT (client side, `rpc::node_jwt`)** + +- The agent's gRPC client was built with + `ForgeClientConfig::new(root_ca, ClientCert{...}).with_node_jwt()`, so a + `NodeJwtMinter` watches those two file paths. +- On the next outgoing RPC, the minter reads cert+key from disk (re-encoding + Vault's SEC1 key PEM to PKCS#8) and signs an **ES256 JWT**: header + `{alg: ES256, x5c: [leaf, issuing CA]}`, claims + `{sub: , aud: , + iat: now, exp: now+300s}`. The token is cached and re-minted when < 60 s + remain — no refresh RPC, and cert renewal is picked up automatically + because the files are re-read at each mint. +- Renewal writes the certificate and the key in two separate operations, so + the minter compares the key's public half against the one certified by the + leaf before signing. A mismatch (a mint landing between the two writes) + fails the mint rather than caching a token the API will reject. +- `BearerAuthService` stamps `Authorization: Bearer ` onto the request. + (The TLS channel may still present the client cert too — dual-support; each + is sufficient alone.) + +**Validation (server side, requires `[node_auth] enabled = true` + TLS listener)** + +- The authn middleware sees the Bearer header and hands it to + `NodeJwtValidator`, which checks in order: + - header `alg` is exactly ES256 (no algorithm substitution); + - the `x5c` chain verifies against the **same root CA file the TLS listener + uses for client certs** (`[tls] root_cafile_path`) — path building, + validity window, client-auth EKU; + - the JWT signature verifies with the *verified leaf's* public key; + - claims: `exp`/`iat`/`aud` enforced, and lifetime bounded by + `max_token_ttl_sec` (default 900 s) so a client can't stretch `exp`; + - the leaf passes SPIFFE validation (leaf-only, single URI SAN), and `sub` + must equal that SAN — identity comes from the verified cert, never from + the claim. + +**Authorization & the first call** + +- The validated SPIFFE URI is mapped through the same `SpiffeContext` as mTLS + certs → a `SpiffeMachineIdentifier` principal, byte-identical to what the + cert path would have minted. +- Casbin RBAC evaluates that principal exactly as before (machine class → + Agent/Scout role rules) — **no RBAC changes** — and the handler executes. + That is the first authorized gRPC call on bearer auth. + +Note: the very first *authorized* call after discovery could ride either +credential, since the agent holds both from the same moment — the JWT only +becomes load-bearing once `mtls_enabled = false`. + +## Q1 — How does the agent get a private key that is trusted to create the JWT? + +**It already has one.** The node's Vault-issued mTLS client certificate key +(`/opt/forge/machine_cert.key`, EC P-256 — Vault PKI role `key_type=ec, +key_bits=256`) signs the JWT, and the certificate itself rides along in the +token's `x5c` header (RFC 7515 §4.1.6). The key is trusted because the +certificate chains to the root CA the API already trusts for client certs — +the JWT is effectively "mTLS at the application layer". + +Bootstrapping is unchanged: a machine obtains its first certificate through +the existing discovery/attestation flow (`DiscoverMachine` / `AttestQuote` +respond with the machine certificate — see the auth-flow walkthrough above), +and from that moment it can mint tokens. Minting is best-effort: before the +cert exists, requests simply carry no bearer header. + +## Q2 — JWT side by side with mTLS + +The authn middleware (`CertDescriptionMiddleware` in `crates/authn`) mints +principals from **both** sources on every request: the TLS-layer client cert, +and the `Authorization: Bearer` token (validated by `NodeJwtValidator`). Both +paths converge on the same SPIFFE URI → `SpiffeMachineIdentifier` mapping +through the same `SpiffeContext`, so RBAC (Casbin policy, role mapping) is +completely unchanged. A node presenting both credentials gets the same +principal twice — harmless. Clients attach the bearer token unconditionally +(`ForgeClientConfig::with_node_jwt()` in scout and dpu-agent); a server with +node-auth disabled ignores the header, so rollout order doesn't matter. + +## Q3 — JWT off by default, configured in the API config + +```toml +[node_auth] +enabled = false # master switch for accepting bearer JWTs +audience = "nico-api" # `aud` required on presented tokens +max_token_ttl_sec = 900 # upper bound on client-chosen lifetimes (cap 86400) +``` + +When `enabled = true`, startup requires a TLS listener (bearer tokens are +never accepted over plaintext) and a readable `[tls] root_cafile_path`; +missing prerequisites fail startup rather than silently degrading. The whole +preflight runs *before* DPF resource creation, so a misconfiguration can't +mutate cluster state on its way to failing. + +`audience` is the one value that must be set on **both** ends: the API +validates `aud` against it, and each node stamps it. Nodes take it from their +own config — scout via `--node-auth-audience`, the agent via `[forge-system] +node-auth-audience` (or the `--node-auth-audience` flag, which is how DPF +deploys it, since containerized agents run with no config file and the API +templates the value from its own `[node_auth] audience`). A site that changes +one end and not the other has every token rejected. + +## Q4 — mTLS on by default, disableable in the API config + +```toml +[node_auth] +mtls_enabled = true +``` + +When `mtls_enabled = false`, the middleware stops minting machine principals +from client certificates — bearer JWTs become the only node auth path. The +gate is scoped to **machine** certs: service and admin-CLI certs on the same +listener are unaffected. `enabled = false` + `mtls_enabled = false` is +rejected at startup (node lockout). + +Note the trust chain is still the certificate PKI: disabling mTLS here +disables the *transport-layer* cert authentication, not cert issuance. Nodes +must keep renewing certificates because the JWT is signed by the cert key. + +## Q5 — Key regeneration and public-key exchange + +**Regeneration** is the existing client-certificate renewal: when +`ClientCertRenewer` rotates the cert/key files, the minter picks the new pair +up on its next re-mint (it re-reads both files from disk each time). No +coordination, no state. + +**Public-key exchange: the x5c header.** Every token carries the certificate +chain that vouches for its signing key, and the API verifies that chain +against the root CA bundle it already holds (`[tls] root_cafile_path`). There +is no JWKS endpoint, no key registry, and no key distribution problem — CA +rotation is handled wherever the root bundle is handled today. + +**CA rotation** moves both consumers of that bundle together. The TLS +listener already re-reads the file every five minutes for cert-manager +rotations; the validator's trust anchors are held behind a lock and refreshed +on that same tick, so a token chaining to a freshly rotated CA is accepted +without an API restart. A failed reload keeps the previous anchors — a bundle +caught mid-write must not disarm node auth. + +**Compromise response** is likewise the PKI's: a stolen key/cert pair is the +same incident as a stolen mTLS cert today. Tokens age out in minutes +(`exp - iat ≤ max_token_ttl_sec`, client mints 5-minute tokens), and that +expiry — together with chain and EKU validation — is what actually bounds a +stolen key. Revocation does not: the validator calls +`allow_unknown_revocation_status()`, so a holder of the private key keeps +minting acceptable tokens until the certificate itself expires. Cutting a +compromised key off sooner needs revocation checking we do not do yet. + +## Q6 — JWT best-practice checklist + +| Practice | How it's honored | +| --- | --- | +| Asymmetric signing, no `alg` confusion | ES256 only; both the header check and `Validation` pin the algorithm, so `none`/HS256 substitution is rejected. | +| Identity never comes from claims | The principal derives from the **chain-verified certificate's SPIFFE SAN**; `sub` is only cross-checked against it. A forged `sub` buys nothing. | +| Short-lived tokens | Clients mint 300 s tokens; the server enforces `exp - iat` and `exp - now` ≤ `max_token_ttl_sec` (default 900 s, hard cap 86400), so a client cannot stretch `exp`. | +| `exp` / `iat` / `aud` enforced | Required claims; validated by `jsonwebtoken` plus the bounded-lifetime check. | +| Chain validation, not pinning | `x5c` verified with rustls `WebPkiClientVerifier` (path building, validity window, client-auth EKU) against the same roots as the TLS listener. | +| SPIFFE leaf constraints | `carbide_authn::validate_x509_certificate` re-checks leaf-ness, key usage, and the single-URI-SAN rule — same code path as mTLS certs. | +| No bearer tokens over plaintext | Enforced at both ends. Server: startup refuses `enabled = true` on a non-TLS listener, the middleware only installs the validator when the listener is TLS-terminated, and a failed acceptor rebuild keeps the previous acceptor rather than falling back to cleartext. Client: `with_token_provider` implies `require_tls_enforcement`, and building a token client against a non-HTTPS URL is an error. | +| No key material at rest beyond the PKI | The server holds no signing key; the client holds only what it already had, and `write_certs` persists the key file owner-only (0600). Credentials never in logs (both token caches have a redacting `Debug`). | +| Least exposure for the signing key | On a DPU the key stays in the dpu-agent alone; consumers get finished short-lived tokens over a local socket and never mount the credentials directory. | + +## Component map + +| Piece | Where | +| --- | --- | +| Client mint + cache + header injection | `crates/rpc/src/node_jwt.rs` (`NodeJwtMinter`, `BearerAuthService`, `NodeTokenProvider`) | +| Client opt-in | `ForgeClientConfig::with_node_jwt()` / `with_token_provider()` (`crates/rpc/src/forge_tls_client.rs`); called in scout `client.rs` and dpu-agent `lib.rs` | +| Key-less token source | `crates/rpc/src/node_token_socket.rs` (`SocketTokenSource`), consumed by fmds via `--node-token-socket` | +| Broker service | `AgentLocal/GetNodeToken` in `crates/rpc/proto/agent_local.proto`, served by `crates/agent/src/local_api.rs` | +| Server validation | `crates/api-core/src/node_auth.rs` (`NodeJwtValidator`) | +| Config | `NodeAuthConfig` in `crates/api-core/src/cfg/file.rs` (`[node_auth]`); node side in `crates/host-support/src/agent_config.rs` and the two `command_line.rs` | +| Middleware hook | `BearerTokenAuthenticator` trait + machine-cert gate in `crates/authn/src/middleware.rs` | +| Wiring | `crates/api-core/src/setup.rs` (preflight, validator construction), `listener.rs` (middleware install, trust-anchor reload), `dpf_services.rs` (fmds token mode) | +| Charts | `bluefield/charts/nico-fmds` (`useNodeTokens`), `bluefield/charts/nico-dpu-agent` | + +All of it logs under one target: `RUST_LOG=node_auth=debug` turns on the +whole feature's tracing, and every message carries a `node-auth:` prefix. + +## Key distribution on DPF: the agent is the token broker + +The machine cert/key live at `/opt/forge` on the DPU, a hostPath directory +several NICo pods mount. Rather than share the key with each of them, the +dpu-agent is the only holder and hands out finished tokens. + +- **The socket.** The agent serves `AgentLocal/GetNodeToken` on a unix socket + at `/opt/forge/run/agent.sock`, mode 0600 — a dedicated `run/` subdirectory + so a consumer can mount just that path read-write (`connect(2)` needs write + access to the socket inode) while credentials stay elsewhere. `bind` creates + the socket at umask permissions and the 0600 chmod only lands afterwards, so + the directory — not the socket — is what closes that window; the agent + therefore requires the socket's parent to be dedicated, creating it `0700` or + refusing a directory that holds anything else. (Without that rule, + `local-api-socket = /run/agent.sock` would take `/run` to `0700` and lock + every non-root service on the box out of its runtime files.) The path is + configurable via `[forge-system] local-api-socket`; the server retries + forever rather than dying once, since a bare-metal boot can start the agent + before its directory exists. This socket is the consolidation point for + future agent ↔ co-located-service traffic, in place of new sockets, ports, + or file drops. +- **The consumer.** `SocketTokenSource` implements the same + `NodeTokenProvider` trait as the minter, so a client built with + `with_token_provider()` is indistinguishable on the wire from one that + minted its own. A background task keeps the cached token fresh (refresh at + 60 s remaining, request-path cutoff at half that, per-attempt deadline + covering connect + RPC); the request path never blocks, and before the + first successful fetch requests simply carry no bearer header. +- **The trust anchor, without the key.** fmds still needs the root CA, so the + agent mirrors it to `/pub/`, a directory holding nothing else. + Token-mode fmds pods mount `pub/` plus `run/` and **do not reference the + credentials volume at all**, not even in the init container — the machine + key is absent from the pod rather than merely read-only, which matters + because the container runs as UID 0. It is a directory mount, not a + `subPath`, so the atomic-rename CA replacement still propagates into a + running pod. The agent republishes on every start as well as from the init + container, so a CA replaced out of band propagates too. +- **The switch.** fmds helm values are rendered by the API, so `useNodeTokens` + is derived from the one setting that makes tokens meaningful: `[node_auth] + enabled`. With node-auth off, the chart renders exactly as before. Rollout + note: enabling it requires an agent image that serves the socket, so + agent and fmds images must be deployed together. + +Scope: this covers *authentication to nico-api*. otelcol also uses the machine +cert for TLS client auth to its OTLP gateway, which a nico-api bearer token +cannot replace, so the key only fully disappears from other pods once that +ingest path has its own credential story. + +## Design decisions (resolved questions) + +1. **How does the server learn the public key?** → from the token itself + (`x5c`), verified against the existing root CA — the one key-distribution + mechanism the system already operates. +2. **Why not drop mTLS immediately, since the JWT proves the same key?** → + dual-support de-risks rollout and keeps requirement 4 orthogonal; and the + JWT still depends on the cert PKI, so cert issuance must outlive transport + mTLS. +3. **Client-side enable knob?** → none. Nodes always mint when they hold a + cert; a disabled server ignores the header (verified by middleware test). + One switch (`[node_auth] enabled`) controls the feature. The audience is + the sole value that must agree on both ends. +4. **Replay window** → a captured token is replayable for ≤ 5 minutes against + the same API over TLS only. Accepted; `jti`/nonce tracking or DPoP-style + proof-of-possession is the hardening path if needed. +5. **RSA machine certs** → not supported (Vault PKI role is EC P-256 + everywhere); the validator rejects non-EC leaves with a clear debug reason. + +## Known issues + +**Disabling `[node_auth]` after fmds has been in token mode is unsequenced.** +The two halves of the switch take effect on different clocks: + +- The API stops accepting bearer tokens *immediately*, when it restarts with + `enabled = false`. +- fmds returns to cert mode only *eventually*. The re-apply is real — + `create_initialization_objects` upserts the DPUServiceConfiguration by + forced server-side apply, so the new `useNodeTokens: false` does land — but + DPF then has to re-render and roll the DaemonSet across the fleet. + +In between, fmds pods are still running token mode with no client cert and +tokens the API now rejects: `phone_home` and machine-identity signing take +401s until the roll reaches each node. Config serving and instance metadata +are unaffected, and it self-heals once the roll completes. + +Nothing is corrupted by the transition. The agent writes `machine_cert.pem` / +`.key` and the base-path root CA unconditionally, in both modes, so cert mode +works the moment a pod rolls; leftover `pub/` and `run/` directories are +inert. + +There is no way to stage it today, because `useNodeTokens` is derived from +`[node_auth] enabled` alone. The safe order — move fmds off tokens, confirm +the roll landed, *then* stop accepting tokens at the API — would need an +explicit per-service override defaulting to the derived value. The enable +direction has the mirror window but fails safe: the init container blocks +waiting for the CA in `pub/` rather than starting into a broken state. + +## Designs not used + +**Server-issued tokens.** A site-level signing key in the credential store, a +`RefreshNodeToken` RPC, and DPU device identity as the refresh anchor. It +brings back everything this design deletes: a server-side key to store, +rotate, and share across HA replicas; issuance and refresh RPCs to build, +version, and rate-limit; and a bootstrap question of its own (what +credential authorizes the *first* refresh). Reusing the node's existing +client-cert key gets the same authenticated principal with no new secret +anywhere. Worth revisiting only if per-node keys are removed from the +architecture, at which point nothing is left to self-sign with. + +**A JWKS endpoint or key registry.** The obvious way to publish per-node +public keys, and unnecessary: `x5c` carries the certificate with every token +and the API already holds the root CA that vouches for it. A registry would +add a distribution channel that can go stale, be unreachable, or disagree +with the PKI — and CA rotation would have to be handled in two places instead +of one. + +**Reusing the tenant JWT-SVID subsystem.** `machine_identity` issues SPIFFE +JWT-SVIDs to *tenant workloads* via IMDS. Node agents authenticating to the +NICo API is a different problem with a different trust root, and coupling +them would make NICo's own control plane depend on a tenant-facing service +being healthy. + +**Kubernetes Secret for the machine key.** Secret mounts are tmpfs, so the +key stops touching DPU flash, and sharing becomes explicit, per-pod, and +auditable through the K8s API. But in DPF the DPU-cluster control plane +(kamaji-hosted etcd) runs on the x86 management cluster, so the key gains a +durable copy — plus backups — *off the DPU*, requiring etcd encryption at +rest to be guaranteed; the agent needs new K8s API rights to create and +update the Secret; and scout (pre-DPF, live-image) can't use this path at +all, so the file mechanism would survive alongside it. The token broker +addresses the same concern — key exposure surface — without moving the key +anywhere, so this became moot rather than merely deferred. + +**Sharing the key by hostPath, read-only.** What token mode replaced. A +read-only mount of the credentials directory stops nothing: the container +runs as UID 0 and can read `machine_cert.key` regardless. Whether the mount +is read-only is not the control; whether the file is in the pod's namespace +at all is. + +**Mounting just the CA file by `subPath`.** A tempting one-line fix for the +above — mount the single file rather than the directory. `subPath` +bind-mounts an inode, and `install_bootstrap_ca` replaces the CA by atomic +rename, so a running pod would keep the old anchor forever and silently fail +after a CA rotation. Hence the dedicated `pub/` directory. + +**A network endpoint for the broker.** There is precedent for it — the agent +already dials `nico-dhcp-server` over gRPC through a k8s `Service` — but the +token broker is a different kind of service, and a port is the wrong shape +for it: + +- *The socket is the authorization check.* `GetNodeToken` ignores its request + entirely: there is no authentication in the handler, because anyone who can + connect is already on the node with the path mounted. On a TCP port that + same RPC hands a full machine identity to every pod in the DPU cluster, and + to anything that can route to the DPU. +- *TLS can't fix that.* Protecting the endpoint means mTLS with the machine + cert — which the consumer does not have, that being the entire premise. The + transport cannot be secured by the credential it exists to distribute, so a + port needs a second credential system to bootstrap the first. The SPIFFE + Workload API is a unix socket for this reason. +- *Locality.* Agent and consumer are DaemonSets on the same node; the call is + same-node by construction. A `Service` routes it through cluster DNS and a + VIP whose control plane (kamaji-hosted etcd) runs on the x86 management + cluster — making "can this node authenticate at all" depend on remote + infrastructure. A socket is a path. +- *No listening port* to firewall or expose by accident, which matters on a + DPU where `nico-otelcol` runs with `hostNetwork: true`. + +The cost of the choice is real: `connect(2)` needs write access to the socket +inode, which is why the socket sits in its own `run/` subdirectory on a +read-write mount. And it serves co-located consumers only — anything off-DPU +needing a node token is not covered by this design. + +Note what mode 0600 does and does not buy. Everything on the DPU runs as +root, so the UID check separates nobody; the real control is which pods mount +the directory. The gain over sharing the key is the blast radius of a leak — +a five-minute token instead of a long-lived private key — not a hard +boundary. + +**Certificate revocation checking.** Not implemented — see Q5. Token expiry +and the certificate's own lifetime are what bound a compromised key; if the +incident model needs a faster cutoff than that, revocation is the work, and +this design does not do it. diff --git a/helm/charts/nico-api/files/carbide-api-config.toml b/helm/charts/nico-api/files/carbide-api-config.toml index 22391afc63..626aa2ff6d 100644 --- a/helm/charts/nico-api/files/carbide-api-config.toml +++ b/helm/charts/nico-api/files/carbide-api-config.toml @@ -41,6 +41,24 @@ hardware_health_reports = "MonitorOnly" enabled = true create_machines = true +[node_auth] +# Node (Scout / DPU-agent) authentication to the API. +# See docs/design/machine-identity/node-auth-jwt.md. +# Bearer JWTs: off by default. When enabled, the API accepts short-lived +# `Authorization: Bearer` tokens that nodes self-sign with their existing +# mTLS client-certificate key (the cert rides in the token's x5c header and +# is verified against the same root CA as the TLS listener). +enabled = false +# Machine mTLS client-certificate authentication: on by default. Disable only +# after the fleet presents bearer tokens (requires enabled = true; the API +# refuses to start with both mechanisms off). Service/admin-CLI certs are +# unaffected. +mtls_enabled = true +# `aud` claim required on presented tokens. +#audience = "nico-api" +# Maximum accepted token lifetime in seconds (nodes mint 300 s tokens). +#max_token_ttl_sec = 900 + [firmware_global] autoupdate = true diff --git a/rest-api/proto/core/gen/v1/agent_local_nico.pb.go b/rest-api/proto/core/gen/v1/agent_local_nico.pb.go new file mode 100644 index 0000000000..17ed48ed4b --- /dev/null +++ b/rest-api/proto/core/gen/v1/agent_local_nico.pb.go @@ -0,0 +1,180 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: agent_local_nico.proto + +package core + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type GetNodeTokenRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetNodeTokenRequest) Reset() { + *x = GetNodeTokenRequest{} + mi := &file_agent_local_nico_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetNodeTokenRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetNodeTokenRequest) ProtoMessage() {} + +func (x *GetNodeTokenRequest) ProtoReflect() protoreflect.Message { + mi := &file_agent_local_nico_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetNodeTokenRequest.ProtoReflect.Descriptor instead. +func (*GetNodeTokenRequest) Descriptor() ([]byte, []int) { + return file_agent_local_nico_proto_rawDescGZIP(), []int{0} +} + +type GetNodeTokenResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The signed ES256 JWT. + Token string `protobuf:"bytes,1,opt,name=token,proto3" json:"token,omitempty"` + // Unix seconds when the token expires. + ExpiresAt uint64 `protobuf:"varint,2,opt,name=expires_at,json=expiresAt,proto3" json:"expires_at,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetNodeTokenResponse) Reset() { + *x = GetNodeTokenResponse{} + mi := &file_agent_local_nico_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetNodeTokenResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetNodeTokenResponse) ProtoMessage() {} + +func (x *GetNodeTokenResponse) ProtoReflect() protoreflect.Message { + mi := &file_agent_local_nico_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetNodeTokenResponse.ProtoReflect.Descriptor instead. +func (*GetNodeTokenResponse) Descriptor() ([]byte, []int) { + return file_agent_local_nico_proto_rawDescGZIP(), []int{1} +} + +func (x *GetNodeTokenResponse) GetToken() string { + if x != nil { + return x.Token + } + return "" +} + +func (x *GetNodeTokenResponse) GetExpiresAt() uint64 { + if x != nil { + return x.ExpiresAt + } + return 0 +} + +var File_agent_local_nico_proto protoreflect.FileDescriptor + +const file_agent_local_nico_proto_rawDesc = "" + + "\n" + + "\x16agent_local_nico.proto\x12\vagent_local\"\x15\n" + + "\x13GetNodeTokenRequest\"K\n" + + "\x14GetNodeTokenResponse\x12\x14\n" + + "\x05token\x18\x01 \x01(\tR\x05token\x12\x1d\n" + + "\n" + + "expires_at\x18\x02 \x01(\x04R\texpiresAt2a\n" + + "\n" + + "AgentLocal\x12S\n" + + "\fGetNodeToken\x12 .agent_local.GetNodeTokenRequest\x1a!.agent_local.GetNodeTokenResponseB8Z6github.com/NVIDIA/infra-controller/rest-api/proto/coreb\x06proto3" + +var ( + file_agent_local_nico_proto_rawDescOnce sync.Once + file_agent_local_nico_proto_rawDescData []byte +) + +func file_agent_local_nico_proto_rawDescGZIP() []byte { + file_agent_local_nico_proto_rawDescOnce.Do(func() { + file_agent_local_nico_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_agent_local_nico_proto_rawDesc), len(file_agent_local_nico_proto_rawDesc))) + }) + return file_agent_local_nico_proto_rawDescData +} + +var file_agent_local_nico_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_agent_local_nico_proto_goTypes = []any{ + (*GetNodeTokenRequest)(nil), // 0: agent_local.GetNodeTokenRequest + (*GetNodeTokenResponse)(nil), // 1: agent_local.GetNodeTokenResponse +} +var file_agent_local_nico_proto_depIdxs = []int32{ + 0, // 0: agent_local.AgentLocal.GetNodeToken:input_type -> agent_local.GetNodeTokenRequest + 1, // 1: agent_local.AgentLocal.GetNodeToken:output_type -> agent_local.GetNodeTokenResponse + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_agent_local_nico_proto_init() } +func file_agent_local_nico_proto_init() { + if File_agent_local_nico_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_agent_local_nico_proto_rawDesc), len(file_agent_local_nico_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_agent_local_nico_proto_goTypes, + DependencyIndexes: file_agent_local_nico_proto_depIdxs, + MessageInfos: file_agent_local_nico_proto_msgTypes, + }.Build() + File_agent_local_nico_proto = out.File + file_agent_local_nico_proto_goTypes = nil + file_agent_local_nico_proto_depIdxs = nil +} diff --git a/rest-api/proto/core/gen/v1/agent_local_nico_grpc.pb.go b/rest-api/proto/core/gen/v1/agent_local_nico_grpc.pb.go new file mode 100644 index 0000000000..e6e7b43d03 --- /dev/null +++ b/rest-api/proto/core/gen/v1/agent_local_nico_grpc.pb.go @@ -0,0 +1,156 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.1 +// - protoc (unknown) +// source: agent_local_nico.proto + +package core + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + AgentLocal_GetNodeToken_FullMethodName = "/agent_local.AgentLocal/GetNodeToken" +) + +// AgentLocalClient is the client API for AgentLocal service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// Local (on-DPU) API served by nico-dpu-agent over a unix domain socket +// (the agent's `local-api-socket` setting, which defaults to a dedicated run/ +// subdirectory of the shared credentials directory, so consumers can mount the +// socket read-write while the credentials stay read-only). The literal default +// lives in DEFAULT_AGENT_LOCAL_SOCKET and is deliberately not repeated here: +// REST proto normalization rewrites vendor names inside comments, which would +// silently corrupt an inlined filesystem path into one that does not exist. +// This is the consolidation point for +// agent <-> co-located-service communication: services that today mount the +// machine cert/key (or synced files) to talk to nico-api can instead ask the +// agent, and future local needs should add RPCs here rather than new +// sockets, ports, or file drops. +type AgentLocalClient interface { + // Returns the current node-auth bearer JWT, minted by the agent from the + // machine's client-certificate key (issue #355). Callers present it as + // `Authorization: Bearer ` to nico-api and must fetch a fresh one + // before `expires_at`. Only the agent ever touches the private key. + GetNodeToken(ctx context.Context, in *GetNodeTokenRequest, opts ...grpc.CallOption) (*GetNodeTokenResponse, error) +} + +type agentLocalClient struct { + cc grpc.ClientConnInterface +} + +func NewAgentLocalClient(cc grpc.ClientConnInterface) AgentLocalClient { + return &agentLocalClient{cc} +} + +func (c *agentLocalClient) GetNodeToken(ctx context.Context, in *GetNodeTokenRequest, opts ...grpc.CallOption) (*GetNodeTokenResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetNodeTokenResponse) + err := c.cc.Invoke(ctx, AgentLocal_GetNodeToken_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// AgentLocalServer is the server API for AgentLocal service. +// All implementations should embed UnimplementedAgentLocalServer +// for forward compatibility. +// +// Local (on-DPU) API served by nico-dpu-agent over a unix domain socket +// (the agent's `local-api-socket` setting, which defaults to a dedicated run/ +// subdirectory of the shared credentials directory, so consumers can mount the +// socket read-write while the credentials stay read-only). The literal default +// lives in DEFAULT_AGENT_LOCAL_SOCKET and is deliberately not repeated here: +// REST proto normalization rewrites vendor names inside comments, which would +// silently corrupt an inlined filesystem path into one that does not exist. +// This is the consolidation point for +// agent <-> co-located-service communication: services that today mount the +// machine cert/key (or synced files) to talk to nico-api can instead ask the +// agent, and future local needs should add RPCs here rather than new +// sockets, ports, or file drops. +type AgentLocalServer interface { + // Returns the current node-auth bearer JWT, minted by the agent from the + // machine's client-certificate key (issue #355). Callers present it as + // `Authorization: Bearer ` to nico-api and must fetch a fresh one + // before `expires_at`. Only the agent ever touches the private key. + GetNodeToken(context.Context, *GetNodeTokenRequest) (*GetNodeTokenResponse, error) +} + +// UnimplementedAgentLocalServer should be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedAgentLocalServer struct{} + +func (UnimplementedAgentLocalServer) GetNodeToken(context.Context, *GetNodeTokenRequest) (*GetNodeTokenResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetNodeToken not implemented") +} +func (UnimplementedAgentLocalServer) testEmbeddedByValue() {} + +// UnsafeAgentLocalServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to AgentLocalServer will +// result in compilation errors. +type UnsafeAgentLocalServer interface { + mustEmbedUnimplementedAgentLocalServer() +} + +func RegisterAgentLocalServer(s grpc.ServiceRegistrar, srv AgentLocalServer) { + // If the following call panics, it indicates UnimplementedAgentLocalServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&AgentLocal_ServiceDesc, srv) +} + +func _AgentLocal_GetNodeToken_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetNodeTokenRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AgentLocalServer).GetNodeToken(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AgentLocal_GetNodeToken_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AgentLocalServer).GetNodeToken(ctx, req.(*GetNodeTokenRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// AgentLocal_ServiceDesc is the grpc.ServiceDesc for AgentLocal service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var AgentLocal_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "agent_local.AgentLocal", + HandlerType: (*AgentLocalServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetNodeToken", + Handler: _AgentLocal_GetNodeToken_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "agent_local_nico.proto", +} diff --git a/rest-api/proto/core/src/v1/agent_local_nico.proto b/rest-api/proto/core/src/v1/agent_local_nico.proto new file mode 100644 index 0000000000..4959b84471 --- /dev/null +++ b/rest-api/proto/core/src/v1/agent_local_nico.proto @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +syntax = "proto3"; + +package agent_local; + +option go_package = "github.com/NVIDIA/infra-controller/rest-api/proto/core"; + +// Local (on-DPU) API served by nico-dpu-agent over a unix domain socket +// (the agent's `local-api-socket` setting, which defaults to a dedicated run/ +// subdirectory of the shared credentials directory, so consumers can mount the +// socket read-write while the credentials stay read-only). The literal default +// lives in DEFAULT_AGENT_LOCAL_SOCKET and is deliberately not repeated here: +// REST proto normalization rewrites vendor names inside comments, which would +// silently corrupt an inlined filesystem path into one that does not exist. +// This is the consolidation point for +// agent <-> co-located-service communication: services that today mount the +// machine cert/key (or synced files) to talk to nico-api can instead ask the +// agent, and future local needs should add RPCs here rather than new +// sockets, ports, or file drops. +service AgentLocal { + // Returns the current node-auth bearer JWT, minted by the agent from the + // machine's client-certificate key (issue #355). Callers present it as + // `Authorization: Bearer ` to nico-api and must fetch a fresh one + // before `expires_at`. Only the agent ever touches the private key. + rpc GetNodeToken(GetNodeTokenRequest) returns (GetNodeTokenResponse); +} + +message GetNodeTokenRequest { +} + +message GetNodeTokenResponse { + // The signed ES256 JWT. + string token = 1; + // Unix seconds when the token expires. + uint64 expires_at = 2; +}