diff --git a/.gitignore b/.gitignore index e41df4cebf..2ad01b0934 100644 --- a/.gitignore +++ b/.gitignore @@ -68,3 +68,8 @@ devenv.local.yaml # pre-commit .pre-commit-config.yaml + +# Trust anchor the dpu-agent mirrors for key-less consumers (issue #355). The +# agent tests point `[forge-system] root-ca` at dev/certs/forge_root.pem, so +# running them publishes a copy beside it. Runtime artifact, never committed. +dev/certs/pub/ diff --git a/Cargo.lock b/Cargo.lock index 15ca49bb2e..f081652948 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1098,6 +1098,7 @@ dependencies = [ "prost", "prost-types", "rand 0.10.1", + "rcgen", "regex", "reqwest 0.13.4", "resolv-conf", @@ -1551,6 +1552,7 @@ dependencies = [ "rcgen", "serde", "thiserror 2.0.18", + "tokio", "tonic", "tower", "tracing", @@ -2951,13 +2953,16 @@ dependencies = [ "hyper-util", "ipnetwork", "itertools 0.14.0", + "jsonwebtoken", "log", "mac_address", "nonempty", "once_cell", + "p256 0.14.0", "prettytable-rs", "prost", "prost-types", + "rcgen", "regex", "resolv-conf", "rustls", @@ -2967,6 +2972,7 @@ dependencies = [ "serde_yaml", "sha2 0.11.0", "sqlx", + "tempfile", "thiserror 2.0.18", "tokio", "tokio-stream", diff --git a/bluefield/charts/nico-fmds/templates/daemonset.yaml b/bluefield/charts/nico-fmds/templates/daemonset.yaml index e8de5fadf0..e9e2e75acd 100644 --- a/bluefield/charts/nico-fmds/templates/daemonset.yaml +++ b/bluefield/charts/nico-fmds/templates/daemonset.yaml @@ -1,3 +1,28 @@ +{{/* + Token mode pins these paths to the dpu-agent's defaults. + + The agent publishes the trust anchor and serves its socket at paths taken + from its own [forge-system] config, and DPF deploys it with no config file + at all -- so it always uses /opt/forge. It has no CLI flag for either path. + Rendering fmds against a different certsDir/rootCaFile therefore points it + at files the agent will never create: the init container waits forever for a + CA that is not coming, or the token source connects to a socket that does + not exist. + + Failing here turns that silent hang into a render-time error. The overrides + remain available in cert mode, where fmds reads the credentials directly and + nothing has to agree with the agent. +*/}} +{{- if .Values.useNodeTokens }} + {{- $dir := .Values.certsDir | default "/opt/forge" }} + {{- $rootCa := .Values.rootCaFile | default "forge_root.pem" }} + {{- if ne $dir "/opt/forge" }} + {{- fail (printf "useNodeTokens requires the dpu-agent's default certsDir (/opt/forge); got %q. The agent has no way to be told otherwise, so fmds would wait for a CA that is never published." $dir) }} + {{- end }} + {{- if ne $rootCa "forge_root.pem" }} + {{- fail (printf "useNodeTokens requires the dpu-agent's default rootCaFile (forge_root.pem); got %q. The agent publishes under its own filename, so fmds would wait for a file that is never created." $rootCa) }} + {{- end }} +{{- end }} apiVersion: apps/v1 kind: DaemonSet metadata: @@ -51,29 +76,51 @@ spec: - /busybox/sh - -c - | - {{- $dir := .Values.certsDir | default "/opt/nico" }} - {{- $rootCa := .Values.rootCaFile | default "nico_root.pem" }} + {{- $dir := .Values.certsDir | default "/opt/forge" }} + {{- $rootCa := .Values.rootCaFile | default "forge_root.pem" }} + {{- if .Values.useNodeTokens }} + echo "Waiting for root CA in {{ $dir }}/pub ..." + while [ ! -f {{ $dir }}/pub/{{ $rootCa }} ]; do + sleep 5 + done + echo "Root CA found, starting nico-fmds (node tokens from the dpu-agent)." + {{- else }} echo "Waiting for certificates in {{ $dir }} ..." while [ ! -f {{ $dir }}/{{ $rootCa }} ] || [ ! -f {{ $dir }}/machine_cert.pem ] || [ ! -f {{ $dir }}/machine_cert.key ]; do sleep 5 done echo "Certificates found, starting nico-fmds." + {{- end }} volumeMounts: + {{- if .Values.useNodeTokens }} + # Token mode never needs the credentials directory, not even to + # wait: the agent publishes the trust anchor to pub/, which holds + # nothing else. + - name: nico-certs-pub + mountPath: {{ .Values.certsDir | default "/opt/forge" }}/pub + readOnly: true + {{- else }} - name: nico-certs - mountPath: {{ .Values.certsDir | default "/opt/nico" }} + mountPath: {{ .Values.certsDir | default "/opt/forge" }} readOnly: true + {{- end }} containers: - name: nico-fmds securityContext: {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" args: - {{- $dir := .Values.certsDir | default "/opt/nico" }} - {{- $rootCa := .Values.rootCaFile | default "nico_root.pem" }} + {{- $dir := .Values.certsDir | default "/opt/forge" }} + {{- $rootCa := .Values.rootCaFile | default "forge_root.pem" }} - "--grpc-address=$(POD_IP):50052" + {{- if .Values.useNodeTokens }} + - "--root-ca={{ $dir }}/pub/{{ $rootCa }}" + - "--node-token-socket={{ $dir }}/run/agent.sock" + {{- else }} - "--root-ca={{ $dir }}/{{ $rootCa }}" - "--client-cert={{ $dir }}/machine_cert.pem" - "--client-key={{ $dir }}/machine_cert.key" + {{- end }} imagePullPolicy: {{ .Values.image.pullPolicy }} {{- with toYaml .Values.serviceDaemonSet.resources }} resources: @@ -106,14 +153,47 @@ spec: fieldRef: fieldPath: metadata.namespace volumeMounts: + {{- $dir := .Values.certsDir | default "/opt/forge" }} + {{- $rootCa := .Values.rootCaFile | default "forge_root.pem" }} + {{- if .Values.useNodeTokens }} + # Token mode: the bearer JWT is the credential, so this container + # gets the trust anchor and nothing else. pub/ holds only the CA, + # so the machine key is absent from the pod — and because this is a + # directory mount rather than a subPath, the agent's atomic + # republish of the CA reaches a running pod. + - name: nico-certs-pub + mountPath: {{ $dir }}/pub + readOnly: true + # Read-only: connecting to a unix socket does not write to the + # filesystem, and Linux exempts socket inodes from the read-only + # mount check, so connect(2) works here. The socket's own 0600 mode + # is what gates access. + - name: nico-agent-run + mountPath: {{ $dir }}/run + readOnly: true + {{- else }} - name: nico-certs - mountPath: {{ .Values.certsDir | default "/opt/nico" }} + mountPath: {{ $dir }} readOnly: true + {{- end }} volumes: + {{- if .Values.useNodeTokens }} + # No credentials-directory volume at all in token mode: nothing in this + # pod is entitled to the machine key. + - name: nico-certs-pub + hostPath: + path: {{ .Values.certsDir | default "/opt/forge" }}/pub + type: DirectoryOrCreate + - name: nico-agent-run + hostPath: + path: {{ .Values.certsDir | default "/opt/forge" }}/run + type: DirectoryOrCreate + {{- else }} - name: nico-certs hostPath: - path: {{ .Values.certsDir | default "/opt/nico" }} + path: {{ .Values.certsDir | default "/opt/forge" }} type: DirectoryOrCreate + {{- end }} {{- with .Values.tolerations }} tolerations: {{- toYaml . | nindent 8 }} diff --git a/bluefield/charts/nico-fmds/tests/node_tokens_test.yaml b/bluefield/charts/nico-fmds/tests/node_tokens_test.yaml new file mode 100644 index 0000000000..84bfc287fa --- /dev/null +++ b/bluefield/charts/nico-fmds/tests/node_tokens_test.yaml @@ -0,0 +1,190 @@ +suite: node-auth token mode +templates: + - daemonset.yaml +tests: + # The point of token mode is that the machine private key is not reachable + # from this pod at all. The key sits beside the root CA in certsDir, and the + # container runs as UID 0, so a read-only mount of that directory would not + # be enough -- the volume must be absent entirely. + - it: should not mount the credentials directory anywhere in token mode + set: + image: + repository: test + tag: test + useNodeTokens: true + asserts: + - notContains: + path: spec.template.spec.volumes + content: + name: nico-certs + hostPath: + path: /opt/forge + type: DirectoryOrCreate + - notContains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: nico-certs + mountPath: /opt/forge + readOnly: true + - notContains: + path: spec.template.spec.initContainers[0].volumeMounts + content: + name: nico-certs + mountPath: /opt/forge + readOnly: true + + # pub/ is mounted as a directory, not as a subPath of the CA file: the agent + # installs the CA by atomic rename, and a subPath mount would pin the old + # inode so a rotation never reached a running pod. + - it: should mount only the published CA directory and the agent socket + set: + image: + repository: test + tag: test + useNodeTokens: true + asserts: + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: nico-certs-pub + mountPath: /opt/forge/pub + readOnly: true + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: nico-agent-run + mountPath: /opt/forge/run + readOnly: true + - lengthEqual: + path: spec.template.spec.containers[0].volumeMounts + count: 2 + - contains: + path: spec.template.spec.volumes + content: + name: nico-certs-pub + hostPath: + path: /opt/forge/pub + type: DirectoryOrCreate + # The socket mount is useless without the volume backing it, and a mount + # naming a volume that does not exist fails the pod at admission rather + # than here — so pin the pair, not just the mount. + - contains: + path: spec.template.spec.volumes + content: + name: nico-agent-run + hostPath: + path: /opt/forge/run + type: DirectoryOrCreate + - lengthEqual: + path: spec.template.spec.volumes + count: 2 + + - it: should read the CA from pub/ and authenticate with a token, not a cert + set: + image: + repository: test + tag: test + useNodeTokens: true + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --root-ca=/opt/forge/pub/forge_root.pem + - contains: + path: spec.template.spec.containers[0].args + content: --node-token-socket=/opt/forge/run/agent.sock + - notContains: + path: spec.template.spec.containers[0].args + content: --client-cert=/opt/forge/machine_cert.pem + - notContains: + path: spec.template.spec.containers[0].args + content: --client-key=/opt/forge/machine_cert.key + + # The init container gates the main one; in token mode it has no business + # waiting on the machine cert/key, which may never arrive in this pod. + - it: should wait only for the published CA in token mode + set: + image: + repository: test + tag: test + useNodeTokens: true + asserts: + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: '/opt/forge/pub/forge_root\.pem' + - notMatchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: 'machine_cert\.key' + - contains: + path: spec.template.spec.initContainers[0].volumeMounts + content: + name: nico-certs-pub + mountPath: /opt/forge/pub + readOnly: true + - lengthEqual: + path: spec.template.spec.initContainers[0].volumeMounts + count: 1 + + # Token mode pins the paths to the agent's defaults, because the agent has no + # flag for them and DPF gives it no config file. Rendering fmds elsewhere + # would leave the init container waiting for a CA that is never published, so + # the chart refuses at template time rather than at 3am. + - it: should refuse a certsDir override in token mode + set: + image: + repository: test + tag: test + useNodeTokens: true + certsDir: /srv/creds + asserts: + - failedTemplate: + errorMessage: 'useNodeTokens requires the dpu-agent''s default certsDir (/opt/forge); got "/srv/creds". The agent has no way to be told otherwise, so fmds would wait for a CA that is never published.' + + - it: should refuse a rootCaFile override in token mode + set: + image: + repository: test + tag: test + useNodeTokens: true + rootCaFile: site_root.pem + asserts: + - failedTemplate: + errorMessage: 'useNodeTokens requires the dpu-agent''s default rootCaFile (forge_root.pem); got "site_root.pem". The agent publishes under its own filename, so fmds would wait for a file that is never created.' + + # The same overrides stay available in cert mode, where fmds reads the + # credentials itself and nothing has to agree with the agent. + - it: should still honour certsDir in cert mode + set: + image: + repository: test + tag: test + certsDir: /srv/creds + rootCaFile: site_root.pem + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --root-ca=/srv/creds/site_root.pem + + # mTLS remains the default, and must not acquire the token-mode plumbing. + - it: should keep the credentials mount and no pub volume by default + set: + image: + repository: test + tag: test + asserts: + - contains: + path: spec.template.spec.volumes + content: + name: nico-certs + hostPath: + path: /opt/forge + type: DirectoryOrCreate + - notContains: + path: spec.template.spec.volumes + content: + name: nico-certs-pub + hostPath: + path: /opt/forge/pub + type: DirectoryOrCreate + - notContains: + path: spec.template.spec.containers[0].args + content: --node-token-socket=/opt/forge/run/agent.sock diff --git a/bluefield/charts/nico-fmds/values.yaml b/bluefield/charts/nico-fmds/values.yaml index d88ca3dfd1..e795a6d4d0 100644 --- a/bluefield/charts/nico-fmds/values.yaml +++ b/bluefield/charts/nico-fmds/values.yaml @@ -13,6 +13,30 @@ exposedPorts: certsDir: /opt/forge rootCaFile: forge_root.pem +### Node-auth (issue #355). +# When true, fmds authenticates to nico-api with short-lived bearer JWTs +# fetched from the dpu-agent's local API socket (/run/agent.sock) +# instead of the machine mTLS cert/key. +# +# The pod then mounts exactly two paths: /pub, which holds the trust +# anchor and nothing else, and the socket directory. It does not mount the +# credentials directory at all, so the machine private key is absent from the +# pod rather than merely read-only. That distinction matters because the +# container runs as UID 0, for which a read-only mount is no barrier. +# +# pub/ is mounted as a directory rather than a subPath so that the agent's +# atomic republish of a rotated CA reaches a running pod. +# +# Requires a dpu-agent that serves the local API and publishes the CA, and +# [node_auth] enabled on the API. +# +# certsDir and rootCaFile must keep their defaults in this mode: the agent +# takes those paths from its own config, DPF gives it no config file, and it +# has no CLI flag for either -- so it always publishes under /opt/forge. The +# chart fails at template time rather than let fmds wait for a CA that is +# never coming. Both remain overridable in cert mode. +useNodeTokens: false + ### Service specific values ### # Prometheus /metrics listen port (must match nico-otelcol scrape target). metricsPort: 8888 diff --git a/crates/agent/Cargo.toml b/crates/agent/Cargo.toml index d1e2d44d6c..891a318f6a 100644 --- a/crates/agent/Cargo.toml +++ b/crates/agent/Cargo.toml @@ -140,6 +140,7 @@ tonic-prost-build = { workspace = true } carbide-instrument = { path = "../instrument", features = ["test-support"] } carbide-test-support = { path = "../test-support" } ctor = { workspace = true } +rcgen = { workspace = true } prost = { workspace = true } rustls-pki-types = { workspace = true } diff --git a/crates/agent/example_agent_config.toml b/crates/agent/example_agent_config.toml index 28fd03b079..c433f9db65 100644 --- a/crates/agent/example_agent_config.toml +++ b/crates/agent/example_agent_config.toml @@ -18,6 +18,9 @@ api-server = "http://localhost:50051" pxe-server = "http://127.0.0.1:8080" root-ca = "/home/amaslennikov/pcarbide/credentials/localhost.pem" +# Unix socket where the agent serves node tokens to co-located services +# (issue #355). Same default containerized (DPF) and on plain DPU OS. +#local-api-socket = "/opt/forge/run/agent.sock" [machine] interface-id = "3048d658-0b13-4a5e-91cc-f47e45a031f9" diff --git a/crates/agent/proto/agent_local.proto b/crates/agent/proto/agent_local.proto new file mode 100644 index 0000000000..0802f03b7c --- /dev/null +++ b/crates/agent/proto/agent_local.proto @@ -0,0 +1,67 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +package agent_local; + +// Local (on-DPU) API served by forge-dpu-agent over a unix domain socket +// (the agent's `local-api-socket` setting, which defaults to a dedicated run/ +// subdirectory of the shared credentials directory, so consumers can mount the +// socket read-write while the credentials stay read-only). The literal default +// lives in DEFAULT_AGENT_LOCAL_SOCKET and is deliberately not repeated here: +// REST proto normalization rewrites vendor names inside comments, which would +// silently corrupt an inlined filesystem path into one that does not exist. +// This is the consolidation point for +// agent <-> co-located-service communication: services that today mount the +// machine cert/key (or synced files) to talk to nico-api can instead ask the +// agent, and future local needs should add RPCs here rather than new +// sockets, ports, or file drops. +service AgentLocal { + // Returns the current node-auth bearer JWT, minted by the agent from the + // machine's client-certificate key (issue #355). Callers present it as + // `Authorization: Bearer ` to nico-api. Only the agent ever touches + // the private key. The empty request carries neither a caller identity nor + // a current token; access is controlled by the local socket's mount and + // permissions. + // + // Freshness: the returned token is valid at the moment of the call but is + // NOT freshly minted per request -- the agent serves a cached token and + // re-mints only once it nears expiry, so successive calls usually return the + // identical string. A cached token is returned only with more than 60 + // seconds remaining; otherwise the agent mints a replacement with its + // normal 300-second lifetime. `expires_at` is therefore the only thing a + // caller may rely on: re-fetch before it, and do not assume a new call + // yields a new token or a reset lifetime. Callers that need continuous + // coverage should refresh with a margin rather than at the boundary. + // + // If the agent cannot mint a token, it returns UNAVAILABLE. This method + // applies no server-side deadline or retry policy; callers choose their own. + rpc GetNodeToken(GetNodeTokenRequest) returns (GetNodeTokenResponse); +} + +message GetNodeTokenRequest { + // Intentionally empty. The connection to the protected local socket is the + // authorization boundary. +} + +message GetNodeTokenResponse { + // The signed ES256 JWT. + string token = 1; + // Unix seconds when the token expires. + uint64 expires_at = 2; +} diff --git a/crates/agent/src/lib.rs b/crates/agent/src/lib.rs index 0e337169fa..ea53912c55 100644 --- a/crates/agent/src/lib.rs +++ b/crates/agent/src/lib.rs @@ -65,6 +65,7 @@ mod host_machine_id; mod instance_metadata_endpoint; pub mod instrumentation; pub mod lldp; +mod local_api; mod machine_inventory_updater; mod main_loop; mod managed_files; @@ -91,7 +92,21 @@ pub const FMDS_MINIMUM_HBN_VERSION: &str = "1.5.0-doca2.2.0"; /// supported configuration path, DPUs running older HBN versions cannot be configured. pub const NVUE_MINIMUM_HBN_VERSION: &str = "2.0.0-doca2.5.0"; -const BOOTSTRAP_CA_OUTPUT_PATH: &str = "/opt/forge/forge_root.pem"; +/// Subdirectory holding a second copy of the trust anchor, and nothing else. +/// +/// Co-located services need the root CA but must never see the machine private +/// key that lives beside it (issue #355). A container can only be given one or +/// the other: mounting the credentials directory exposes the key, and mounting +/// the CA file alone by `subPath` bind-mounts its inode, so the atomic rename +/// in `install_bootstrap_ca` would never reach a running consumer. A directory +/// containing only the CA gives both properties — no key, and renames resolve. +/// +/// Placed beside whichever CA path is configured rather than at a fixed +/// location, so it tracks a deployment that moves the credentials directory — +/// token-mode fmds watches `/pub/` and would otherwise +/// block forever. With the defaults on both sides this resolves to +/// `/opt/forge/pub/forge_root.pem`. +const BOOTSTRAP_CA_PUBLIC_SUBDIR: &str = "pub"; const MOUNTED_BOOTSTRAP_CA_PATH: &str = "/var/run/secrets/nico-bootstrap-ca/ca.pem"; const MAX_BOOTSTRAP_CA_BYTES: usize = 1024 * 1024; const BOOTSTRAP_CA_DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(30); @@ -269,10 +284,97 @@ fn install_bootstrap_ca(contents: &[u8], output_path: &Path) -> eyre::Result<()> Ok(()) } -async fn provision_bootstrap_ca(options: &command_line::InitContainerOptions) -> eyre::Result<()> { +async fn provision_bootstrap_ca( + options: &command_line::InitContainerOptions, + root_ca_path: &str, +) -> eyre::Result<()> { let contents = acquire_bootstrap_ca(options.bootstrap_ca_source, &options.bootstrap_ca_url).await?; - install_bootstrap_ca(&contents, Path::new(BOOTSTRAP_CA_OUTPUT_PATH)) + // Installed where the agent will later look for it, rather than at a + // constant of its own. The two were equal by coincidence -- `[forge-system] + // root-ca` defaults to the same path -- and nothing kept them that way, so + // a deployment that moved the CA would have had the init container write + // one place and every reader look in another. + let output_path = Path::new(root_ca_path); + install_bootstrap_ca(&contents, output_path)?; + publish_bootstrap_ca(&contents, output_path) +} + +/// Mirrors the trust anchor into a `pub/` subdirectory beside `source_ca_path` +/// for key-less consumers. Same atomic install, so a consumer with the +/// directory mounted picks up a replacement without restarting. +/// +/// Derived from the configured CA rather than a constant: token-mode fmds +/// waits on `/pub/`, so a deployment that moves the +/// credentials directory would otherwise have the agent publish to the default +/// location while fmds watched the configured one — leaving its init container +/// blocked forever with nothing to say why. +fn publish_bootstrap_ca(contents: &[u8], source_ca_path: &Path) -> eyre::Result<()> { + let dir = source_ca_path + .parent() + .ok_or_else(|| eyre::eyre!("CA path has no parent: {}", source_ca_path.display()))?; + let file_name = source_ca_path + .file_name() + .ok_or_else(|| eyre::eyre!("CA path has no file name: {}", source_ca_path.display()))?; + let parent = dir.join(BOOTSTRAP_CA_PUBLIC_SUBDIR); + std::fs::create_dir_all(&parent) + .wrap_err_with(|| format!("failed to create public CA directory {}", parent.display()))?; + install_bootstrap_ca(contents, &parent.join(file_name)) +} + +/// How often the published copy of the trust anchor is reconciled against the +/// configured one. Matches the API listener's own reload cadence, so a rotation +/// reaches key-less consumers on roughly the same clock as it reaches the +/// listener and the token validator. +const CA_REPUBLISH_INTERVAL: Duration = Duration::from_secs(5 * 60); + +/// Copies the configured trust anchor into its `pub/` mirror when the two +/// differ. +/// +/// Publishing only at startup is not enough. `pub/` is the only trust anchor a +/// token-mode consumer has — it does not mount the credentials directory — so a +/// CA rotated while the agent is running would leave fmds pinned to the old +/// issuer indefinitely, and it would stop being able to verify the API the +/// moment that issuer is retired. The API listener re-reads the same material +/// every few minutes; the published copy has to keep pace. +/// +/// Compares before writing so a steady state costs a read rather than an atomic +/// rename every interval, and best-effort throughout: a missing or unreadable +/// CA is the pre-registration case, and a failed write leaves the previous copy +/// in place for the next pass. +fn republish_bootstrap_ca_if_changed(root_ca_path: &str) { + let source = Path::new(root_ca_path); + let Ok(contents) = std::fs::read(source) else { + tracing::debug!( + target: "node_auth", + path = %root_ca_path, + "node-auth: no root CA to publish for co-located services yet" + ); + return; + }; + + let published = source + .parent() + .zip(source.file_name()) + .map(|(dir, name)| dir.join(BOOTSTRAP_CA_PUBLIC_SUBDIR).join(name)); + if let Some(published) = &published + && std::fs::read(published).is_ok_and(|existing| existing == contents) + { + return; + } + + match publish_bootstrap_ca(&contents, source) { + Ok(()) => tracing::info!( + target: "node_auth", + path = %root_ca_path, + "node-auth: published the root CA for co-located services" + ), + Err(error) => tracing::warn!( + target: "node_auth", + %error, + "node-auth: could not publish the root CA for co-located services" + ), + } } pub async fn start(cmdline: command_line::Options) -> eyre::Result<()> { @@ -303,6 +405,19 @@ pub async fn start(cmdline: command_line::Options) -> eyre::Result<()> { tracing::warn!("Pretending local host is a DPU. Dev only."); } + // Published once here so it exists before anything else starts, then kept + // current by `main_loop::run_single_iteration`, which reconciles it beside + // certificate renewal. + republish_bootstrap_ca_if_changed(&agent.forge_system.root_ca); + + // Node-auth (#355): the agent is the only process on the DPU that holds + // the machine key. This minter signs bearer JWTs for the agent's own API + // calls AND backs the local API socket that brokers tokens to co-located + // services (fmds, ...). Ignored by the API unless [node_auth] is enabled. + let node_jwt_minter = ::rpc::node_jwt::NodeJwtMinter::new( + agent.forge_system.client_cert.clone(), + agent.forge_system.client_key.clone(), + ); let forge_client_config = Arc::new( ForgeClientConfig::new( agent.forge_system.root_ca.clone(), @@ -311,6 +426,7 @@ pub async fn start(cmdline: command_line::Options) -> eyre::Result<()> { key_path: agent.forge_system.client_key.clone(), }), ) + .with_token_provider(node_jwt_minter.clone()) .use_mgmt_vrf()?, ); @@ -325,6 +441,19 @@ pub async fn start(cmdline: command_line::Options) -> eyre::Result<()> { tracing::warn!("Upgrades disabled. Dev only"); } + let local_api = { + let minter = node_jwt_minter.clone(); + let socket_path = agent.forge_system.local_api_socket.clone(); + async move { + loop { + if let Err(error) = local_api::serve(minter.clone(), &socket_path).await { + tracing::warn!(target: "node_auth", %error, "node-auth: agent local API server failed; retrying"); + } + tokio::time::sleep(std::time::Duration::from_secs(10)).await; + } + } + }; + let Registration { machine_id, factory_mac_address, @@ -339,15 +468,18 @@ pub async fn start(cmdline: command_line::Options) -> eyre::Result<()> { factory_mac_address: "11:22:33:44:55:66".parse().unwrap(), }, }; - main_loop::setup_and_run( - machine_id, - factory_mac_address, - forge_client_config, - agent, - *options, - ) - .await - .wrap_err("main_loop error exit")?; + let main_loop_result = tokio::select! { + result = main_loop::setup_and_run( + machine_id, + factory_mac_address, + forge_client_config, + agent, + *options, + ) => result.wrap_err("main_loop error exit"), + () = local_api => unreachable!("the agent local API retry loop cannot finish"), + }; + + main_loop_result?; tracing::info!("Agent exit"); } @@ -372,7 +504,7 @@ pub async fn start(cmdline: command_line::Options) -> eyre::Result<()> { // Init-container entry point: provision the CA + snapshot hardware to the shared volume. // Output path is fixed (HW_CACHE_PATH) so the main container can always find it. Some(AgentCommand::InitContainer(options)) => { - provision_bootstrap_ca(&options).await?; + provision_bootstrap_ca(&options, &agent.forge_system.root_ca).await?; enumerate_and_save_hardware().await?; util::save_host_nameservers()?; } diff --git a/crates/agent/src/local_api.rs b/crates/agent/src/local_api.rs new file mode 100644 index 0000000000..74e41eed05 --- /dev/null +++ b/crates/agent/src/local_api.rs @@ -0,0 +1,537 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! The dpu-agent's local API: an `AgentLocal` gRPC service on a unix socket, +//! by default in a dedicated `run/` subdirectory of the shared `/opt/forge` +//! credentials directory (issue #355) — so containerized consumers mount the +//! socket read-write while the credentials stay on a read-only mount. +//! +//! This is the consolidation point for agent <-> co-located-service +//! communication on the DPU. First RPC: `GetNodeToken`, which hands +//! co-located NICo services (fmds, …) a short-lived node-auth bearer JWT so +//! they can authenticate to nico-api without mounting the machine's private +//! key — only the agent ever touches the key. Future local needs should add +//! RPCs here rather than new sockets, ports, or file drops. + +use std::sync::Arc; + +use ::rpc::agent_local::agent_local_server::{AgentLocal, AgentLocalServer}; +use ::rpc::agent_local::{GetNodeTokenRequest, GetNodeTokenResponse}; +use ::rpc::node_jwt::NodeJwtMinter; +use eyre::WrapErr; +use tokio_stream::wrappers::UnixListenerStream; +use tonic::{Request, Response, Status}; + +struct AgentLocalService { + minter: Arc, +} + +#[tonic::async_trait] +impl AgentLocal for AgentLocalService { + async fn get_node_token( + &self, + _request: Request, + ) -> Result, Status> { + // Minting reads the cert/key from disk on cache miss — cheap enough to + // do inline, and it means the endpoint starts working the moment the + // machine certificate lands without any coordination. + match self.minter.current_with_expiry() { + Some((token, expires_at)) => { + Ok(Response::new(GetNodeTokenResponse { token, expires_at })) + } + None => Err(Status::unavailable( + "no node token available yet; machine certificate not present or unreadable", + )), + } + } +} + +/// Prepares the socket's parent directory, which must be dedicated to this +/// socket and nothing else. +/// +/// The directory has to be unreachable to other users *before* `bind`: `bind` +/// creates the socket with umask-derived permissions — typically +/// world-connectable — and the 0600 chmod only lands afterward. The listener +/// is already bound in that window, so a connection from any local user would +/// queue in the backlog and be served a full machine identity the moment +/// accepting starts. An unreachable parent closes the window; the socket's own +/// 0600 is then defence in depth. +/// +/// But `local-api-socket` is operator-configurable, so applying 0700 to +/// whatever the parent happens to be is its own hazard: `/run/agent.sock` +/// would take `/run` to 0700 and lock every non-root service on the box out of +/// its runtime files. Only a directory this function created, or one holding +/// nothing but this socket, is ours to restrict — anything else is rejected +/// with an actionable message rather than silently modified. +/// +/// The directory is also required to be a real directory owned by this +/// process. A symlink would redirect the 0700 onto a target chosen by whoever +/// planted it, and a directory owned by another user is one we have no +/// business restricting — in both cases the configured path is wrong in a way +/// worth reporting rather than working around. This does not close every race: +/// the path is resolved afresh by `read_dir`, `set_permissions` and `bind`, so +/// an attacker who can already swap directories underneath us between those +/// calls is not stopped by it. On the DPU every process that can do that is +/// already root, which is why this stays a validity check rather than growing +/// into `openat`/`fchmod` plumbing. +fn prepare_socket_dir(dir: &std::path::Path, socket_path: &str) -> eyre::Result<()> { + use std::os::unix::fs::{DirBuilderExt, FileTypeExt, MetadataExt, PermissionsExt}; + + let Some(metadata) = optional_symlink_metadata(dir) + .wrap_err(format!("inspecting socket directory {}", dir.display()))? + else { + // Ancestors keep their normal modes; only the socket directory itself + // is created restricted, and created that way from the first instant + // rather than chmod-ed after the fact. + if let Some(ancestor) = dir.parent() { + std::fs::create_dir_all(ancestor) + .wrap_err(format!("creating socket directory {}", ancestor.display()))?; + } + return std::fs::DirBuilder::new() + .mode(0o700) + .create(dir) + .wrap_err(format!("creating socket directory {}", dir.display())); + }; + + // `symlink_metadata` does not follow the final component, so this catches a + // symlink instead of reporting whatever it points at. + if metadata.file_type().is_symlink() { + eyre::bail!( + "socket directory {} is a symlink; point local-api-socket at a real \ + directory, so restricting it to 0700 cannot be redirected elsewhere", + dir.display() + ); + } + if !metadata.is_dir() { + eyre::bail!( + "socket path parent {} exists but is not a directory", + dir.display() + ); + } + let euid = nix::unistd::geteuid().as_raw(); + if metadata.uid() != euid { + eyre::bail!( + "socket directory {} is owned by uid {}, not this process (uid {}); \ + point local-api-socket at a directory this agent owns", + dir.display(), + metadata.uid(), + euid + ); + } + + let socket_name = std::path::Path::new(socket_path).file_name(); + for entry in + std::fs::read_dir(dir).wrap_err(format!("reading socket directory {}", dir.display()))? + { + let entry = entry.wrap_err(format!("reading socket directory {}", dir.display()))?; + let name = entry.file_name(); + if socket_name == Some(name.as_os_str()) { + if !entry + .file_type() + .wrap_err(format!("inspecting socket path {}", entry.path().display()))? + .is_socket() + { + eyre::bail!( + "socket path {} exists and is not a socket; refusing to restrict its parent \ + directory — point local-api-socket at a path the agent owns", + entry.path().display() + ); + } + } else { + eyre::bail!( + "socket directory {} is shared with other files (found {}); \ + point local-api-socket at a directory used for nothing else, \ + so restricting it to 0700 cannot lock other services out", + dir.display(), + name.to_string_lossy() + ); + } + } + + std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o700)).wrap_err(format!( + "restricting socket directory permissions on {}", + dir.display() + )) +} + +/// `symlink_metadata`, with "does not exist" as `Ok(None)` rather than an error. +fn optional_symlink_metadata(path: &std::path::Path) -> std::io::Result> { + match std::fs::symlink_metadata(path) { + Ok(metadata) => Ok(Some(metadata)), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e), + } +} + +/// Clears the socket left behind by a previous run, since `bind` fails on an +/// existing path even when nothing is listening. +/// +/// Only an actual socket is removed. The path is operator-configurable and the +/// dedicated-directory rule above matches on filename alone, so a regular file, +/// directory or symlink sitting at that name would otherwise be silently +/// deleted by a privileged process — a fine primitive to hand an attacker who +/// can write the directory, and a data-loss footgun for an operator who +/// mistypes the path. Anything that is not a socket is reported instead. +fn remove_stale_socket(socket_path: &str) -> eyre::Result<()> { + use std::os::unix::fs::FileTypeExt; + + let path = std::path::Path::new(socket_path); + let Some(metadata) = optional_symlink_metadata(path) + .wrap_err(format!("inspecting socket path {socket_path}"))? + else { + return Ok(()); + }; + + if !metadata.file_type().is_socket() { + eyre::bail!( + "{socket_path} exists and is not a socket; refusing to remove it — \ + point local-api-socket at a path the agent owns" + ); + } + + std::fs::remove_file(path).wrap_err(format!("removing stale socket {socket_path}")) +} + +/// Binds the local API socket and serves until the process exits. The socket +/// is created mode 0600 (root-only): every legitimate consumer on the DPU +/// runs as root, and nothing else on the shared directory should be able to +/// obtain machine credentials. +/// +/// Deployment-agnostic: containerized (DPF) the socket lands on the mounted +/// `/opt/forge` volume; as a plain service on DPU OS (non-DPF) the parent +/// directory is created if the agent starts before anything else touched it. +pub(crate) async fn serve(minter: Arc, socket_path: &str) -> eyre::Result<()> { + use std::os::unix::fs::PermissionsExt; + + if let Some(parent) = std::path::Path::new(socket_path).parent() { + prepare_socket_dir(parent, socket_path)?; + } + remove_stale_socket(socket_path)?; + + let listener = tokio::net::UnixListener::bind(socket_path) + .wrap_err(format!("binding agent local API socket {socket_path}"))?; + std::fs::set_permissions(socket_path, std::fs::Permissions::from_mode(0o600)) + .wrap_err(format!("restricting socket permissions on {socket_path}"))?; + tracing::info!(target: "node_auth", socket = %socket_path, "node-auth: serving agent local API (node tokens)"); + + tonic::transport::Server::builder() + .add_service(AgentLocalServer::new(AgentLocalService { minter })) + .serve_with_incoming(UnixListenerStream::new(listener)) + .await + .wrap_err("agent local API server exited") +} + +#[cfg(test)] +mod tests { + use ::rpc::node_jwt::NodeTokenProvider; + use ::rpc::node_token_socket::SocketTokenSource; + + use super::*; + + const SPIFFE_URI: &str = "spiffe://forge.local/forge-system/machine/fm100xtest"; + + fn write_cert_and_key(dir: &tempfile::TempDir) -> (String, String) { + let mut params = rcgen::CertificateParams::default(); + params.subject_alt_names = vec![rcgen::SanType::URI( + rcgen::string::Ia5String::try_from(SPIFFE_URI.to_string()).expect("uri"), + )]; + let key = rcgen::KeyPair::generate().expect("key pair"); + let cert = params.self_signed(&key).expect("certificate"); + let cert_path = dir.path().join("cert.pem"); + let key_path = dir.path().join("cert.key"); + std::fs::write(&cert_path, cert.pem()).expect("write cert"); + std::fs::write(&key_path, key.serialize_pem()).expect("write key"); + ( + cert_path.to_string_lossy().into_owned(), + key_path.to_string_lossy().into_owned(), + ) + } + + /// End-to-end broker flow: agent serves tokens minted from the machine + /// cert; a key-less consumer obtains one through `SocketTokenSource`. + #[tokio::test] + async fn keyless_consumer_gets_token_via_socket() { + let dir = tempfile::tempdir().expect("tempdir"); + let (cert_path, key_path) = write_cert_and_key(&dir); + // Its own subdirectory, as in production (`/run/agent.sock`): + // the socket never shares a directory with the credentials. + let socket = dir.path().join("run").join("agent.sock"); + let socket_str = socket.to_string_lossy().into_owned(); + + let minter = NodeJwtMinter::new(cert_path, key_path); + let expected = minter.current().expect("agent side can mint"); + tokio::spawn({ + let socket_str = socket_str.clone(); + async move { serve(minter, &socket_str).await } + }); + + let source = SocketTokenSource::spawn(socket_str); + let mut got = None; + for _ in 0..100 { + if let Some(token) = source.current() { + got = Some(token); + break; + } + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + assert_eq!(got.as_deref(), Some(expected.as_str())); + } + + #[tokio::test] + async fn socket_is_root_only() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().expect("tempdir"); + let (cert_path, key_path) = write_cert_and_key(&dir); + let socket = dir.path().join("run").join("agent.sock"); + let socket_str = socket.to_string_lossy().into_owned(); + + tokio::spawn({ + let socket_str = socket_str.clone(); + async move { serve(NodeJwtMinter::new(cert_path, key_path), &socket_str).await } + }); + // Wait for the mode, not merely for the socket to appear: `bind` + // creates it at umask permissions and the chmod lands afterward, so + // sampling on existence alone would read the pre-chmod mode whenever + // the poll happened to fall inside that window — a flake that looks + // like a real permissions regression. + let mut mode = None; + for _ in 0..100 { + if let Ok(metadata) = std::fs::metadata(&socket) { + let observed = metadata.permissions().mode() & 0o777; + if observed == 0o600 { + mode = Some(observed); + break; + } + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert_eq!( + mode, + Some(0o600), + "the socket must end up root-only; last seen: {:?}", + std::fs::metadata(&socket).map(|m| m.permissions().mode() & 0o777) + ); + } + + /// The socket's own mode is applied only after `bind`, so the directory is + /// what actually keeps other users out during that window. Anyone who got + /// in would be handed a full machine identity. + #[tokio::test] + async fn socket_directory_is_root_only_before_the_socket_exists() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().expect("tempdir"); + let (cert_path, key_path) = write_cert_and_key(&dir); + // A pre-existing, world-traversable directory: the agent must tighten + // it rather than assume a fresh one. + let run_dir = dir.path().join("run"); + std::fs::create_dir(&run_dir).expect("create run dir"); + std::fs::set_permissions(&run_dir, std::fs::Permissions::from_mode(0o755)) + .expect("loosen run dir"); + let socket = run_dir.join("agent.sock"); + let socket_str = socket.to_string_lossy().into_owned(); + + tokio::spawn({ + let socket_str = socket_str.clone(); + async move { serve(NodeJwtMinter::new(cert_path, key_path), &socket_str).await } + }); + for _ in 0..100 { + if socket.exists() { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + + let mode = std::fs::metadata(&run_dir) + .expect("run dir metadata") + .permissions() + .mode() + & 0o777; + assert_eq!(mode, 0o700, "socket directory must exclude other users"); + } + + /// Tightening the parent is only safe when the parent belongs to us. + /// `local-api-socket = /run/agent.sock` would otherwise take `/run` to + /// 0700 and lock every non-root service on the box out of its runtime + /// files, so a directory holding anything else is refused outright. + #[test] + fn shared_socket_directory_is_refused_rather_than_restricted() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().expect("tempdir"); + let shared = dir.path().join("run"); + std::fs::create_dir(&shared).expect("create shared dir"); + std::fs::write(shared.join("someone-elses.pid"), b"1234").expect("write neighbour"); + std::fs::set_permissions(&shared, std::fs::Permissions::from_mode(0o755)) + .expect("loosen shared dir"); + let socket = shared.join("agent.sock"); + + let err = prepare_socket_dir(&shared, &socket.to_string_lossy()) + .expect_err("a shared directory must not be accepted"); + assert!( + err.to_string().contains("someone-elses.pid"), + "the error should name the file that made it shared, got: {err}" + ); + + let mode = std::fs::metadata(&shared) + .expect("shared dir metadata") + .permissions() + .mode() + & 0o777; + assert_eq!( + mode, 0o755, + "a refused directory must be left exactly as it was found" + ); + } + + /// A directory we create ourselves is restricted from its first instant, + /// with no window at a looser mode for anyone to slip through. + #[test] + fn fresh_socket_directory_is_created_root_only() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().expect("tempdir"); + let run_dir = dir.path().join("nested").join("run"); + let socket = run_dir.join("agent.sock"); + + prepare_socket_dir(&run_dir, &socket.to_string_lossy()).expect("prepare fresh dir"); + + let mode = std::fs::metadata(&run_dir) + .expect("run dir metadata") + .permissions() + .mode() + & 0o777; + assert_eq!(mode, 0o700, "a freshly created socket directory is 0700"); + } + + /// The stale socket from a previous run is ours, so it must not be + /// mistaken for a neighbour and block startup. + #[test] + fn a_stale_socket_does_not_make_the_directory_look_shared() { + let dir = tempfile::tempdir().expect("tempdir"); + let run_dir = dir.path().join("run"); + std::fs::create_dir(&run_dir).expect("create run dir"); + let socket = run_dir.join("agent.sock"); + // A real socket, as a previous run would have left: the type is what + // `remove_stale_socket` keys on, so a placeholder file would not + // exercise the same path. + let listener = std::os::unix::net::UnixListener::bind(&socket).expect("stale socket"); + drop(listener); + + prepare_socket_dir(&run_dir, &socket.to_string_lossy()) + .expect("a directory holding only our own socket is still dedicated"); + remove_stale_socket(&socket.to_string_lossy()).expect("our own stale socket is removable"); + assert!(!socket.exists(), "the stale socket should be gone"); + } + + /// Removing whatever happens to sit at the configured path would hand an + /// attacker who can write the directory a privileged unlink, and would + /// quietly destroy an operator's file after a mistyped path. + #[test] + fn a_non_socket_at_the_socket_path_is_refused_not_deleted() { + let dir = tempfile::tempdir().expect("tempdir"); + let occupied = dir.path().join("agent.sock"); + std::fs::write(&occupied, b"not a socket").expect("write regular file"); + + let err = remove_stale_socket(&occupied.to_string_lossy()) + .expect_err("a regular file must not be removed"); + assert!( + err.to_string().contains("not a socket"), + "the error should say why, got: {err}" + ); + assert!(occupied.exists(), "the file must be left untouched"); + } + + /// An existing entry with the configured socket name must be a socket + /// before it makes the parent look dedicated. Otherwise we would tighten + /// the directory and only then reject the regular file or symlink. + #[test] + fn a_non_socket_in_the_socket_directory_is_refused_before_restricting() { + use std::os::unix::fs::PermissionsExt; + + for occupant in ["regular file", "symlink"] { + let dir = tempfile::tempdir().expect("tempdir"); + let run_dir = dir.path().join("run"); + std::fs::create_dir(&run_dir).expect("create run dir"); + std::fs::set_permissions(&run_dir, std::fs::Permissions::from_mode(0o755)) + .expect("loosen run dir"); + let socket = run_dir.join("agent.sock"); + match occupant { + "regular file" => { + std::fs::write(&socket, b"not a socket").expect("write regular file"); + } + "symlink" => { + std::os::unix::fs::symlink("someone-elses.sock", &socket) + .expect("create symlink"); + } + _ => unreachable!("test cases are exhaustive"), + } + + let err = prepare_socket_dir(&run_dir, &socket.to_string_lossy()) + .expect_err("a non-socket must not make the directory look dedicated"); + assert!( + err.to_string().contains("not a socket"), + "{occupant}: the error should say why, got: {err}" + ); + let mode = std::fs::metadata(&run_dir) + .expect("run dir metadata") + .permissions() + .mode() + & 0o777; + assert_eq!( + mode, 0o755, + "{occupant}: a refused directory must be left exactly as it was found" + ); + } + } + + /// A missing socket is the normal first-boot case, not an error. + #[test] + fn a_missing_socket_path_is_not_an_error() { + let dir = tempfile::tempdir().expect("tempdir"); + let socket = dir.path().join("agent.sock"); + remove_stale_socket(&socket.to_string_lossy()).expect("nothing to remove is fine"); + } + + /// A symlinked directory would redirect the 0700 onto a target chosen by + /// whoever planted it. + #[test] + fn a_symlinked_socket_directory_is_refused() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().expect("tempdir"); + let real = dir.path().join("elsewhere"); + std::fs::create_dir(&real).expect("create target dir"); + std::fs::set_permissions(&real, std::fs::Permissions::from_mode(0o755)) + .expect("loosen target"); + let link = dir.path().join("run"); + std::os::unix::fs::symlink(&real, &link).expect("symlink"); + + let err = prepare_socket_dir(&link, &link.join("agent.sock").to_string_lossy()) + .expect_err("a symlinked directory must be refused"); + assert!( + err.to_string().contains("symlink"), + "the error should say why, got: {err}" + ); + + let mode = std::fs::metadata(&real) + .expect("target metadata") + .permissions() + .mode() + & 0o777; + assert_eq!( + mode, 0o755, + "the symlink target must not be re-permissioned" + ); + } +} diff --git a/crates/agent/src/main_loop.rs b/crates/agent/src/main_loop.rs index 510fcdae8f..d217b7852d 100644 --- a/crates/agent/src/main_loop.rs +++ b/crates/agent/src/main_loop.rs @@ -424,6 +424,7 @@ pub(super) async fn setup_and_run( has_logged_stable: false, version_check_time: std::time::Instant::now(), inventory_updater_time: std::time::Instant::now(), + ca_republish_time: std::time::Instant::now(), started_at: std::time::Instant::now(), inventory_updater_config, options, @@ -462,6 +463,7 @@ struct MainLoop { started_at: std::time::Instant, version_check_time: std::time::Instant, inventory_updater_time: std::time::Instant, + ca_republish_time: std::time::Instant, inventory_updater_config: MachineInventoryUpdaterConfig, options: command_line::RunOptions, agent_config: AgentConfig, @@ -1222,6 +1224,17 @@ impl MainLoop { .renew_certificates_if_necessary(None) .await; + // Beside renewal because the two revolve around the same files: + // renewal rewrites the credentials, and key-less consumers need the + // resulting trust anchor mirrored into `pub/` (issue #355). Driving it + // from here rather than a spawned task means it inherits the loop's + // shutdown handling, and a panic takes the agent down for a restart + // instead of silently leaving consumers on a stale anchor. + if now > self.ca_republish_time { + self.ca_republish_time = now.add(crate::CA_REPUBLISH_INTERVAL); + crate::republish_bootstrap_ca_if_changed(&self.agent_config.forge_system.root_ca); + } + if now > self.inventory_updater_time { self.inventory_updater_time = now.add(self.inventory_updater_config.update_inventory_interval); diff --git a/crates/agent/src/tests/bootstrap_ca.rs b/crates/agent/src/tests/bootstrap_ca.rs index 3455983046..a4a5f398be 100644 --- a/crates/agent/src/tests/bootstrap_ca.rs +++ b/crates/agent/src/tests/bootstrap_ca.rs @@ -27,7 +27,8 @@ use url::Url; use crate::{ MAX_BOOTSTRAP_CA_BYTES, download_bootstrap_ca, download_bootstrap_ca_with_timeout, - install_bootstrap_ca, read_bootstrap_ca_file, + install_bootstrap_ca, publish_bootstrap_ca, read_bootstrap_ca_file, + republish_bootstrap_ca_if_changed, }; const VALID_CA: &[u8] = include_bytes!(concat!( @@ -224,3 +225,114 @@ fn oversized_bootstrap_ca_preserves_existing_file() { assert!(install_bootstrap_ca(&oversized, &output).is_err()); assert_eq!(std::fs::read(&output).unwrap(), b"old trust anchor"); } + +/// The published copy has to land beside whichever CA path is configured. +/// Token-mode fmds waits on `/pub/`, so publishing to a +/// fixed location would leave its init container blocked forever on any +/// deployment that moved the credentials directory — with nothing logged to +/// explain it. +#[test] +fn published_ca_follows_the_configured_credentials_directory() { + let directory = tempfile::tempdir().unwrap(); + let creds = directory.path().join("srv").join("creds"); + std::fs::create_dir_all(&creds).unwrap(); + let source = creds.join("site_root.pem"); + + publish_bootstrap_ca(VALID_CA, &source).unwrap(); + + let published = creds.join("pub").join("site_root.pem"); + assert_eq!( + std::fs::read(&published).unwrap(), + VALID_CA, + "the CA must be published to {}", + published.display() + ); +} + +/// Republishing over an existing copy is the rotation path, and must land +/// atomically rather than leaving a truncated file a consumer could read. +#[test] +fn publishing_replaces_a_previous_published_ca() { + let directory = tempfile::tempdir().unwrap(); + let source = directory.path().join("forge_root.pem"); + let published_dir = directory.path().join("pub"); + std::fs::create_dir_all(&published_dir).unwrap(); + std::fs::write(published_dir.join("forge_root.pem"), b"old trust anchor").unwrap(); + + publish_bootstrap_ca(VALID_CA, &source).unwrap(); + + assert_eq!( + std::fs::read(published_dir.join("forge_root.pem")).unwrap(), + VALID_CA + ); +} + +/// A CA rotated while the agent is running must reach the published copy. +/// `pub/` is the only trust anchor a token-mode consumer has, so leaving it on +/// the old issuer would break fmds the moment that issuer is retired — the same +/// stale-snapshot failure the API-side validator reload exists to prevent. +#[test] +fn republish_picks_up_a_rotated_ca() { + let directory = tempfile::tempdir().unwrap(); + let source = directory.path().join("forge_root.pem"); + let published = directory.path().join("pub").join("forge_root.pem"); + + // Both anchors must be real certificates: publishing validates, so a + // placeholder would be rejected and the test would prove nothing. + let old_ca = a_self_signed_ca(); + std::fs::write(&source, &old_ca).unwrap(); + republish_bootstrap_ca_if_changed(&source.to_string_lossy()); + assert_eq!(std::fs::read(&published).unwrap(), old_ca); + + // Rotation, out of band, with the agent already running. + std::fs::write(&source, VALID_CA).unwrap(); + republish_bootstrap_ca_if_changed(&source.to_string_lossy()); + assert_eq!( + std::fs::read(&published).unwrap(), + VALID_CA, + "the published copy must follow the configured one" + ); +} + +/// Steady state must not rewrite. The reconcile runs on a timer, and an atomic +/// rename every interval would churn the inode consumers are watching for no +/// reason. +#[test] +fn republish_is_a_no_op_when_the_ca_is_unchanged() { + use std::os::unix::fs::MetadataExt; + + let directory = tempfile::tempdir().unwrap(); + let source = directory.path().join("forge_root.pem"); + let published = directory.path().join("pub").join("forge_root.pem"); + std::fs::write(&source, VALID_CA).unwrap(); + + republish_bootstrap_ca_if_changed(&source.to_string_lossy()); + let first = std::fs::metadata(&published).unwrap().ino(); + + republish_bootstrap_ca_if_changed(&source.to_string_lossy()); + let second = std::fs::metadata(&published).unwrap().ino(); + + assert_eq!(first, second, "an unchanged CA must not be republished"); +} + +/// Before registration there is no CA at all. That is the normal first-boot +/// state, not an error, and it must not panic the republish loop. +#[test] +fn republish_tolerates_a_missing_ca() { + let directory = tempfile::tempdir().unwrap(); + let missing = directory.path().join("forge_root.pem"); + republish_bootstrap_ca_if_changed(&missing.to_string_lossy()); + assert!(!directory.path().join("pub").exists()); +} + +/// A second, distinct trust anchor — publishing validates its input, so +/// rotation tests need real certificates on both sides of the change. +fn a_self_signed_ca() -> Vec { + let params = rcgen::CertificateParams::default(); + let key = rcgen::KeyPair::generate().expect("key pair"); + params + .self_signed(&key) + .expect("certificate") + .pem() + .into_bytes() +} diff --git a/crates/api-core/src/api.rs b/crates/api-core/src/api.rs index 1be2996819..b6abd0f31a 100644 --- a/crates/api-core/src/api.rs +++ b/crates/api-core/src/api.rs @@ -90,6 +90,10 @@ pub struct Api { pub(crate) component_manager: Option, pub(crate) bms_client: OnceLock>, pub(crate) secrets_context: Option, + /// Validator for node-auth bearer JWTs (issue #355). `Some` only when + /// `[node_auth] enabled`; installed into the authn middleware by the + /// listener. + pub(crate) node_jwt_validator: Option>, } pub(crate) type ScoutStreamType = diff --git a/crates/api-core/src/cfg/README.md b/crates/api-core/src/cfg/README.md index 0584db0967..f226ab472e 100644 --- a/crates/api-core/src/cfg/README.md +++ b/crates/api-core/src/cfg/README.md @@ -137,6 +137,7 @@ Use `site_explorer.dpu_policy` instead. | `dhcp_lease_expiry_handling` | `bool` | `false` | `networking` | Enables IP cleanup when a DHCP lease expires. | | `certificates` | `CertificatesConfig` | *(default)* | `security` | Certificate vending backend, selected independently of the credential store; the default shares the credential Vault (see [CertificatesConfig](#certificatesconfig)). | | `allow_insecure_discovery` | `bool` | `false` | `machines` | Allows machines to submit discovery without enforcing the request comes from the expected IP address. Needed for *Integration tests only*, should otherwise not be used. | +| `node_auth` | `NodeAuthConfig` | *(default)* | `security` | How Scout and the DPU-agent authenticate: bearer JWTs, machine mTLS client certificates, or both during a migration (see [NodeAuthConfig](#nodeauthconfig)). | --- @@ -330,6 +331,27 @@ extracted identifier contains characters such as `/` or `.`. | `identity_keyfile_path` | `String` | `""` | Server identity private key. | | `admin_root_cafile_path` | `String` | `""` | Admin root CA for admin client validation. | +### `NodeAuthConfig` + +Node (Scout / DPU-agent) authentication. Bearer tokens are off by default, so +the default is machine mTLS exactly as before. See +`docs/design/machine-identity/node-auth-jwt.md`. + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `enabled` | `bool` | `false` | Accept `Authorization: Bearer` node JWTs. Nodes self-sign these with their existing mTLS client-certificate key and carry the certificate in the token's `x5c` header; the API verifies it against `[tls] root_cafile_path`. Requires a TLS listener -- the API refuses to accept bearer tokens over plaintext. | +| `mtls_enabled` | `bool` | `true` | Accept machine mTLS client certificates as node identity. Turn off only once the fleet presents bearer tokens; startup fails if this and `enabled` are both false. Scoped to machine certificates -- service and admin-CLI certificates are unaffected. Requires a TLS listener to mean anything: a plaintext `listen_mode` presents no peer certificates, so this silently authenticates nobody. | +| `max_token_ttl_sec` | `u32` | `900` | Longest accepted token lifetime, in seconds. Clients mint 300 s tokens; this caps how far a client may push `exp`. Must be greater than zero and at most 86400. | +| `fmds_use_node_tokens` | `Option` | *(unset)* | Whether DPF-deployed fmds is rendered in token mode. Unset follows `enabled`, which is what almost every site wants. Set it to `false` while `enabled` is still `true` to move fmds back to client certificates *first* -- the supported way to stage a disable, since the API stops accepting tokens the moment it restarts while fmds keeps presenting them until DPF has rolled every DaemonSet. `true` with `enabled = false` is refused at startup. | + +Both mechanisms need `listen_mode = "tls"`. Bearer tokens are refused over +plaintext explicitly, at startup; machine mTLS simply has no certificates to +inspect, because a plaintext listener hands the middleware an empty peer-cert +list. The `enabled = false` + `mtls_enabled = false` lockout check therefore +guarantees a working node-auth path *only on a TLS listener* -- on plaintext, +`mtls_enabled = true` satisfies the check while authenticating nobody. No +shipped configuration selects a plaintext mode. + ### `AuthConfig` | Field | Type | Default | Description | diff --git a/crates/api-core/src/cfg/file.rs b/crates/api-core/src/cfg/file.rs index 03138eaf1c..4a5d9dc30f 100644 --- a/crates/api-core/src/cfg/file.rs +++ b/crates/api-core/src/cfg/file.rs @@ -529,6 +529,12 @@ pub struct CarbideConfig { #[serde(default)] pub machine_identity: MachineIdentityConfig, + /// Node-auth: bearer JWTs that Scout / DPU-agent self-sign with their + /// existing mTLS client-certificate key, accepted alongside (or instead + /// of) mTLS. Section `[node_auth]`. + #[serde(default)] + pub node_auth: NodeAuthConfig, + /// Disables role-based access control enforcement. /// Intended for testing and development only. #[serde(default)] @@ -2195,6 +2201,145 @@ impl Default for MachineIdentityConfig { } } +/// Node-auth (Scout / DPU-agent bearer JWT) configuration. +/// Loaded from `[node_auth]` section in config. +/// +/// There is no server-side signing key: nodes self-sign tokens with their +/// existing mTLS client-certificate key and the API validates the embedded +/// `x5c` chain against the client-cert root CA (see +/// `docs/design/machine-identity/node-auth-jwt.md`). +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct NodeAuthConfig { + /// Master switch. When false, no bearer authenticator is installed and + /// nodes keep authenticating via mTLS client certs only. + #[serde(default = "node_auth_default_enabled")] + pub enabled: bool, + /// Maximum accepted token lifetime, in seconds. Nodes mint 5-minute + /// tokens; this bounds how far a (compromised) client can stretch `exp`. + #[serde(default = "node_auth_default_max_token_ttl_sec")] + pub max_token_ttl_sec: u32, + /// Whether machine mTLS client certificates are accepted as node identity. + /// On by default. Scoped to MACHINE certs only — service and admin-CLI + /// client certs on the same listener are unaffected. Disable only once the + /// fleet presents bearer tokens. + #[serde(default = "node_auth_default_mtls_enabled")] + pub mtls_enabled: bool, + /// Whether DPF-deployed fmds is rendered in token mode, overriding the + /// value otherwise derived from [`enabled`](Self::enabled). + /// + /// Unset (the default) means "follow `enabled`", which is what a site + /// wants almost always. The override exists because the two halves of a + /// change do not land at the same time: the API stops accepting bearer + /// tokens the moment it restarts, while fmds keeps presenting them until + /// DPF has rolled every DaemonSet. Without a separate knob there is no way + /// to order those steps, so disabling node-auth necessarily opens a window + /// where fmds is authenticating with a credential the API no longer takes. + /// + /// Setting it to `false` while `enabled` is still `true` moves fmds back + /// to client certificates first; once that roll has landed, `enabled` can + /// be turned off with nothing depending on tokens. See "Disabling + /// node-auth" in `docs/design/machine-identity/node-auth-jwt.md`. + /// + /// `true` with `enabled = false` is rejected at startup: fmds would + /// present tokens to an API that refuses them, which is the outage the + /// override exists to prevent. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fmds_use_node_tokens: Option, +} + +/// Upper bound on accepted token lifetime. Node tokens are minted locally on +/// demand, so anything beyond a day is almost certainly a misconfiguration. +pub const NODE_AUTH_MAX_TOKEN_TTL_SEC: u32 = 86_400; + +impl NodeAuthConfig { + /// Whether DPF-deployed fmds should be rendered in token mode. + /// + /// The override when set, otherwise [`enabled`](Self::enabled). Call this + /// rather than reading `enabled` directly wherever fmds helm values are + /// built, so the staging path stays available. + #[must_use] + pub fn fmds_use_node_tokens(&self) -> bool { + self.fmds_use_node_tokens.unwrap_or(self.enabled) + } + + /// Validates node-auth settings. Call unconditionally at startup: the + /// lockout check applies even when [`enabled`](Self::enabled) is false. + /// + /// That check assumes a TLS listener. On a plaintext `listen_mode` the + /// accept path yields no peer certificates, so `mtls_enabled = true` + /// satisfies it while authenticating nobody; only the bearer half of the + /// dependency is enforced here, because refusing plaintext outright would + /// break local development for a mode nothing ships. + pub fn validate(&self) -> eyre::Result<()> { + if !self.enabled && !self.mtls_enabled { + return Err(eyre::eyre!( + "[node_auth] enabled = false and mtls_enabled = false would leave nodes with no \ + way to authenticate; enable at least one of bearer tokens or machine mTLS" + )); + } + // Checked before the `enabled` early-return below: this combination is + // precisely the one where `enabled` is false, and it would deploy fmds + // to present tokens the API refuses. + if self.fmds_use_node_tokens == Some(true) && !self.enabled { + return Err(eyre::eyre!( + "[node_auth] fmds_use_node_tokens = true requires enabled = true; fmds would \ + present bearer tokens to an API that does not accept them" + )); + } + if !self.enabled { + // Remaining checks only constrain token validation. + return Ok(()); + } + if self.max_token_ttl_sec == 0 { + return Err(eyre::eyre!( + "[node_auth] max_token_ttl_sec must be greater than zero" + )); + } + // A cap below what the shipped clients mint accepts at startup and then + // rejects every token the fleet presents -- and with mtls_enabled = + // false that is a total lockout, with nothing naming the setting. The + // clients mint a fixed lifetime, so the cap has to clear it. + if u64::from(self.max_token_ttl_sec) < ::rpc::node_jwt::NODE_JWT_TTL_SECS { + return Err(eyre::eyre!( + "[node_auth] max_token_ttl_sec {} is below the {} s lifetime node clients \ + mint, so every token they present would be rejected", + self.max_token_ttl_sec, + ::rpc::node_jwt::NODE_JWT_TTL_SECS + )); + } + if self.max_token_ttl_sec > NODE_AUTH_MAX_TOKEN_TTL_SEC { + return Err(eyre::eyre!( + "[node_auth] max_token_ttl_sec {} exceeds maximum {NODE_AUTH_MAX_TOKEN_TTL_SEC}", + self.max_token_ttl_sec + )); + } + Ok(()) + } +} + +fn node_auth_default_enabled() -> bool { + false +} +fn node_auth_default_max_token_ttl_sec() -> u32 { + 900 +} +fn node_auth_default_mtls_enabled() -> bool { + true +} + +impl Default for NodeAuthConfig { + fn default() -> Self { + Self { + enabled: node_auth_default_enabled(), + max_token_ttl_sec: node_auth_default_max_token_ttl_sec(), + mtls_enabled: node_auth_default_mtls_enabled(), + // Unset: follow `enabled`. Only a site staging a change sets it. + fmds_use_node_tokens: None, + } + } +} + impl From for model::tenant::IdentityConfigValidationBounds { fn from(mi: MachineIdentityConfig) -> Self { Self { @@ -3942,6 +4087,119 @@ mod tests { use super::*; use crate::test_support::network_segment::FIXTURE_TENANT_ORG_ID; + /// Disabling both bearer tokens and machine mTLS would lock every node out + /// of the API; validation must refuse the combination, and each mechanism + /// alone must pass. + #[test] + fn node_auth_rejects_all_methods_disabled() { + let both_off = NodeAuthConfig { + enabled: false, + mtls_enabled: false, + ..NodeAuthConfig::default() + }; + assert!(both_off.validate().is_err()); + + assert!(NodeAuthConfig::default().validate().is_ok()); + let jwt_only = NodeAuthConfig { + enabled: true, + mtls_enabled: false, + ..NodeAuthConfig::default() + }; + assert!(jwt_only.validate().is_ok()); + } + + #[test] + fn node_auth_rejects_the_removed_audience_setting() { + let error = toml::from_str::("audience = \"nico-api-eu\"") + .expect_err("the node-auth audience is fixed"); + assert!(error.to_string().contains("unknown field `audience`")); + } + + /// A cap below the clients' fixed 300 s lifetime is accepted-looking and + /// fatal: every token the fleet mints exceeds it, so all of them are + /// rejected. Startup has to refuse rather than let the fleet discover it. + #[test] + fn max_token_ttl_below_the_client_lifetime_is_rejected() { + let too_small = NodeAuthConfig { + enabled: true, + max_token_ttl_sec: 60, + ..NodeAuthConfig::default() + }; + let err = too_small + .validate() + .expect_err("a cap under the client TTL must be refused"); + assert!( + err.to_string().contains("max_token_ttl_sec"), + "the error should name the setting, got: {err}" + ); + + // Exactly the client lifetime is the boundary, and is fine. + let exact = NodeAuthConfig { + enabled: true, + max_token_ttl_sec: u32::try_from(::rpc::node_jwt::NODE_JWT_TTL_SECS).expect("fits"), + ..NodeAuthConfig::default() + }; + assert!(exact.validate().is_ok()); + + // Disabled: the cap is never consulted, so it must not block startup. + let disabled = NodeAuthConfig { + enabled: false, + max_token_ttl_sec: 60, + ..NodeAuthConfig::default() + }; + assert!(disabled.validate().is_ok()); + } + + /// Unset means "follow `enabled`", which is what almost every site wants. + /// The override exists so a disable can be staged: fmds goes back to client + /// certificates while the API still accepts tokens. + #[test] + fn fmds_token_mode_follows_enabled_unless_overridden() { + let enabled = NodeAuthConfig { + enabled: true, + ..NodeAuthConfig::default() + }; + assert!(enabled.fmds_use_node_tokens(), "unset follows enabled=true"); + + let disabled = NodeAuthConfig::default(); + assert!( + !disabled.fmds_use_node_tokens(), + "unset follows enabled=false" + ); + + // The staging step: API still accepting tokens, fmds moved off them. + let staging = NodeAuthConfig { + enabled: true, + fmds_use_node_tokens: Some(false), + ..NodeAuthConfig::default() + }; + assert!(!staging.fmds_use_node_tokens()); + assert!( + staging.validate().is_ok(), + "moving fmds off tokens early is the supported path" + ); + } + + /// The inverse is the outage the override exists to prevent: fmds + /// presenting bearer tokens to an API that does not accept them. Refuse it + /// at startup rather than deploying it. + #[test] + fn fmds_token_mode_cannot_outrun_the_api() { + let ahead = NodeAuthConfig { + enabled: false, + mtls_enabled: true, + fmds_use_node_tokens: Some(true), + ..NodeAuthConfig::default() + }; + let err = ahead + .validate() + .expect_err("fmds must not present tokens the API refuses"); + assert!( + err.to_string().contains("fmds_use_node_tokens"), + "the error should name the setting, got: {err}" + ); + } + const TEST_DATA_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/src/cfg/test_data"); /// Verifies legacy entries remain valid while typed DPF identities require and preserve their diff --git a/crates/api-core/src/dpf_services.rs b/crates/api-core/src/dpf_services.rs index eb7719905b..537141821d 100644 --- a/crates/api-core/src/dpf_services.rs +++ b/crates/api-core/src/dpf_services.rs @@ -32,7 +32,7 @@ use carbide_dpf::{ use crate::cfg::file::{ DpfBootstrapCaObjectKind, DpfDpuAgentBootstrapCa, DpfExtraService, - DpfResolvedMandatoryServicesConfig, DpfServiceConfig, + DpfResolvedMandatoryServicesConfig, DpfServiceConfig, NodeAuthConfig, }; /// Default DOCA helm registry (DPUServiceTemplate source.repoURL). @@ -375,6 +375,56 @@ fn set_hbn_sf_count(helm_values: &mut serde_json::Value, sf_count: usize) { } } +/// Restores a value the API owns after the operator overlay has been merged. +/// +/// `apply_helm_values` merges `extra_helm_values` last, so an overlay wins over +/// anything generated above it. That is right for tuning a chart, and wrong for +/// the handful of values that encode an agreement between two components: the +/// API validates them at startup, so letting a Helm overlay change one deploys +/// a fleet the API will reject while passing every check. Reassert them, and +/// say so when an overlay tried. +fn reassert_api_owned_value( + helm_values: &mut serde_json::Value, + service: &str, + path: &[&str], + authoritative: serde_json::Value, +) { + let (last, parents) = path.split_last().expect("path must not be empty"); + let mut node = helm_values; + for key in parents { + // An overlay can put anything here, including a scalar or null, so + // descend defensively: assuming an object would turn a malformed + // `extra_helm_values` into a panic during DPF resource creation, which + // is a poor way to report a typo in someone's config. + let entry = node + .as_object_mut() + .expect("generated Helm values must be an object") + .entry((*key).to_string()) + .or_insert_with(|| serde_json::json!({})); + if !entry.is_object() { + *entry = serde_json::json!({}); + } + node = entry; + } + let object = node + .as_object_mut() + .expect("generated Helm values must be an object"); + match object.get(*last) { + Some(existing) if *existing == authoritative => {} + Some(overridden) => tracing::warn!( + target: "node_auth", + service, + setting = path.join("."), + %overridden, + authoritative = %authoritative, + "node-auth: ignoring an extra_helm_values override of a value the API owns; \ + it is validated at startup and both ends must agree" + ), + None => {} + } + object.insert((*last).to_string(), authoritative); +} + /// DOCA HBN service definition. pub(crate) fn doca_hbn_service( cfg: &DpfServiceConfig, @@ -564,17 +614,31 @@ pub(crate) fn dhcp_server_service( } /// Forge FMDS service definition. +/// +/// `use_node_tokens` defaults to the API's `[node_auth] enabled` switch and can +/// be overridden by `fmds_use_node_tokens`: when set, fmds is deployed fetching +/// bearer JWTs from the dpu-agent's local API socket instead of mounting the +/// machine cert/key (issue #355). Requires a dpu-agent image that serves the +/// local API. pub(crate) fn fmds_service( cfg: &DpfServiceConfig, dpu_interfaces: &[DpuServiceInterfaceTemplateDefinition], + use_node_tokens: bool, ) -> ServiceDefinition { let mut helm_values = serde_json::json!({ "image": { "repository": cfg.docker_repo_url, "tag": cfg.docker_image_tag, - } + }, + "useNodeTokens": use_node_tokens, }); apply_helm_values(&mut helm_values, cfg); + reassert_api_owned_value( + &mut helm_values, + FMDS_SERVICE_NAME, + &["useNodeTokens"], + serde_json::json!(use_node_tokens), + ); ServiceDefinition { helm_values: Some(helm_values), @@ -764,17 +828,27 @@ pub(crate) fn doca_xplane_service(cfg: &DpfServiceConfig) -> ServiceDefinition { } /// Build the full list of resolved mandatory DPU services. +/// +/// `node_auth` mirrors the API's `[node_auth]` section: fmds token mode follows +/// `enabled` unless `fmds_use_node_tokens` overrides it (issue #355). pub(crate) fn mandatory_services( resolved: &DpfResolvedMandatoryServicesConfig, bootstrap_ca: &DpfDpuAgentBootstrapCa, interfaces: &[DpuServiceInterfaceTemplateDefinition], + node_auth: &NodeAuthConfig, ) -> Vec { let mut service_vec = vec![ dts_service(&resolved.base.dts), doca_hbn_service(&resolved.base.doca_hbn, interfaces), dhcp_server_service(&resolved.base.dhcp_server, interfaces), dpu_agent_service(&resolved.base.dpu_agent, bootstrap_ca), - fmds_service(&resolved.base.fmds, interfaces), + // Not `node_auth.enabled` directly: an operator staging a disable + // moves fmds off tokens first, while the API still accepts them. + fmds_service( + &resolved.base.fmds, + interfaces, + node_auth.fmds_use_node_tokens(), + ), otelcol_service(&resolved.base.otel), ]; @@ -854,7 +928,7 @@ mod tests { // DHCP receives both configured entries, while FMDS receives only the PF. let dhcp = dhcp_server_service(&default_dhcp_server_service(), &interfaces); - let fmds = fmds_service(&default_fmds_service(), &interfaces); + let fmds = fmds_service(&default_fmds_service(), &interfaces, false); assert_eq!( dhcp.interfaces .iter() @@ -955,7 +1029,10 @@ mod tests { fn dpu_agent_bootstrap_ca_helm_values_follow_site_policy() { value_scenarios!( run = |policy| { - dpu_agent_helm_values(&default_dpu_agent_service(), &policy) + dpu_agent_helm_values( + &default_dpu_agent_service(), + &policy, + ) .get("bootstrapCa") .cloned() }; @@ -1462,4 +1539,67 @@ mod tests { println!("wrote {}", template_path.display()); println!("wrote {}", configuration_path.display()); } + /// The override has to reach the rendered helm values, not just the + /// resolver — staging a disable is worthless if `mandatory_services` still + /// reads `enabled` and deploys fmds in token mode anyway. + #[test] + fn fmds_helm_values_honor_the_token_mode_override() { + // Every field carries a serde default, so an empty object yields the + // stock service set without spelling all six out here. + let resolved = DpfResolvedMandatoryServicesConfig { + base: serde_json::from_value(serde_json::json!({})) + .expect("mandatory services build from their serde defaults"), + extra: BTreeMap::new(), + }; + let bootstrap_ca = DpfDpuAgentBootstrapCa::default(); + + let fmds_mode = |node_auth: &NodeAuthConfig| { + mandatory_services(&resolved, &bootstrap_ca, &[], node_auth) + .into_iter() + .find(|s| s.name == FMDS_SERVICE_NAME) + .and_then(|s| s.helm_values) + .and_then(|v| v.get("useNodeTokens").and_then(serde_json::Value::as_bool)) + .expect("fmds renders useNodeTokens") + }; + + let derived_on = NodeAuthConfig { + enabled: true, + ..NodeAuthConfig::default() + }; + assert!(fmds_mode(&derived_on), "enabled=true derives token mode"); + + let staged_off = NodeAuthConfig { + enabled: true, + fmds_use_node_tokens: Some(false), + ..NodeAuthConfig::default() + }; + assert!( + !fmds_mode(&staged_off), + "the override must win over the derived value" + ); + + assert!( + !fmds_mode(&NodeAuthConfig::default()), + "node-auth off still renders cert mode" + ); + } + /// Same reasoning for token mode: an overlay setting it true while the API + /// does not accept bearer tokens would deploy keyless fmds pods whose only + /// credential is refused, bypassing the startup validation entirely. + #[test] + fn an_overlay_cannot_turn_on_fmds_token_mode() { + let mut cfg = default_fmds_service(); + cfg.extra_helm_values = serde_json::json!({ "useNodeTokens": true }) + .as_object() + .cloned(); + + let service = fmds_service(&cfg, &[], false); + let values = service.helm_values.expect("fmds renders helm values"); + + assert_eq!( + values.get("useNodeTokens"), + Some(&serde_json::json!(false)), + "token mode follows the validated config, not the overlay" + ); + } } diff --git a/crates/api-core/src/lib.rs b/crates/api-core/src/lib.rs index 55ea60f7cc..c98b37e724 100644 --- a/crates/api-core/src/lib.rs +++ b/crates/api-core/src/lib.rs @@ -68,6 +68,7 @@ mod machine_validation; mod measured_boot; mod mqtt_state_change_hook; mod network_segment; +mod node_auth; mod scout_stream; pub mod secrets; mod setup; diff --git a/crates/api-core/src/listener.rs b/crates/api-core/src/listener.rs index 2dbfce7046..9858ae356c 100644 --- a/crates/api-core/src/listener.rs +++ b/crates/api-core/src/listener.rs @@ -17,7 +17,7 @@ use std::net::SocketAddr; use std::sync::Arc; -use std::time::Instant; +use std::time::{Duration, Instant}; use ::rpc::forge as rpc; use carbide_authn::SpiffeContext; @@ -72,8 +72,31 @@ pub(crate) struct ApiTlsConfig { pub(crate) admin_root_cafile_path: String, } +/// Cadence for re-reading the TLS identity and client-CA bundle, so +/// cert-manager rotations are picked up without a restart. +const TLS_REFRESH_INTERVAL: Duration = Duration::from_secs(5 * 60); + +/// Cadence after a failed rebuild. Shorter than [`TLS_REFRESH_INTERVAL`], so a +/// file caught mid-write recovers in seconds rather than minutes — but not +/// zero: retrying on every accepted connection would let a persistently broken +/// identity file amplify inbound traffic into a rebuild per connection. +const TLS_REFRESH_RETRY_DELAY: Duration = Duration::from_secs(15); + +/// Reads the client-CA bundle. Split out so one read can feed both the TLS +/// acceptor and the node-auth validator: each building from its own read lets a +/// cert-manager rotation land between them, which would install an acceptor +/// trusting one generation of the bundle and a token validator trusting +/// another, until the next refresh happened to catch them together. +/// +/// this function blocks, don't use it in a raw async context +fn read_client_ca(tls_config: &ApiTlsConfig) -> Option> { + std::fs::read(&tls_config.root_cafile_path) + .inspect_err(|error| tracing::error!(?error, "error reading root ca cert file")) + .ok() +} + /// this function blocks, don't use it in a raw async context -fn get_tls_acceptor(tls_config: &ApiTlsConfig) -> Option { +fn get_tls_acceptor(tls_config: &ApiTlsConfig, client_ca_pem: &[u8]) -> Option { let certs = { let fd = match std::fs::File::open(&tls_config.identity_pemfile_path) { Ok(fd) => fd, @@ -111,22 +134,14 @@ fn get_tls_acceptor(tls_config: &ApiTlsConfig) -> Option { let roots = { let mut roots = RootCertStore::empty(); - match std::fs::read(&tls_config.root_cafile_path) { - Ok(pem_file) => { - let mut cert_cursor = std::io::Cursor::new(&pem_file[..]); - let certs_to_add = rustls_pemfile::certs(&mut cert_cursor) - .collect::, _>>() - .inspect_err(|error| { - tracing::error!(?error, "error parsing root ca cert file"); - }) - .ok()?; - let (_added, _ignored) = roots.add_parsable_certificates(certs_to_add); - } - Err(error) => { - tracing::error!(?error, "error reading root ca cert file"); - return None; - } - } + let mut cert_cursor = std::io::Cursor::new(client_ca_pem); + let certs_to_add = rustls_pemfile::certs(&mut cert_cursor) + .collect::, _>>() + .inspect_err(|error| { + tracing::error!(?error, "error parsing root ca cert file"); + }) + .ok()?; + let (_added, _ignored) = roots.add_parsable_certificates(certs_to_add); if let Ok(pem_file) = std::fs::read(&tls_config.admin_root_cafile_path) { let mut cert_cursor = std::io::Cursor::new(&pem_file[..]); @@ -305,7 +320,10 @@ pub(crate) async fn start( let tls_config_clone = tls_config.clone(); let tls_acceptor = tokio::task::Builder::new() .name("get_tls_acceptor init") - .spawn_blocking(move || get_tls_acceptor(&tls_config_clone))? + .spawn_blocking(move || { + let client_ca = read_client_ca(&tls_config_clone)?; + get_tls_acceptor(&tls_config_clone, &client_ca) + })? .await?; (Some(tls_config), tls_acceptor, false) } @@ -339,8 +357,64 @@ pub(crate) async fn start( ), ))?; - let cert_description_layer: CertDescriptionMiddleware = - CertDescriptionMiddleware::new(extra_cli_certs, spiffe_context); + let cert_description_layer: CertDescriptionMiddleware = { + let machine_certs_enabled = api_service.runtime_config.node_auth.mtls_enabled; + if !machine_certs_enabled { + tracing::warn!( + target: "node_auth", + "node-auth: mtls_enabled = false: machine client certificates will NOT be \ + accepted as node identity; nodes must present bearer tokens" + ); + } + let layer = CertDescriptionMiddleware::new(extra_cli_certs, spiffe_context) + .with_machine_certs_enabled(machine_certs_enabled); + // When node-auth is enabled, accept bearer JWTs in addition to mTLS + // client certs (dual-support during the mTLS→JWT migration). Bearer + // tokens must only be accepted over TLS — never plaintext — so guard the + // authenticator on the listener actually being TLS-terminated. + // + // `tls_config.is_some()` alone is not that guarantee: it says TLS was + // configured, not that `get_tls_acceptor` could build one. An + // unreadable identity certificate or key drops the accept loop onto its + // plaintext branch. + // + // That is a failure on its own terms, node-auth or not: operators and + // clients both treat a TLS-configured port as encrypted, and silently + // serving cleartext there is worse than not coming up. Node-auth only + // sharpens it — the bearer authenticator is armed once, here, on the + // premise that this listener terminates TLS, and once + // mtls_enabled = false there is no other credential to fall back to. + match ( + &api_service.node_jwt_validator, + tls_config.is_some(), + tls_acceptor.is_some(), + ) { + (node_jwt_validator, true, false) => { + eyre::bail!( + "the TLS acceptor could not be built from the configured identity \ + certificate and key; refusing to start, because the listener would \ + serve plaintext on a TLS-configured port{}", + if node_jwt_validator.is_some() { + " while accepting node-auth bearer tokens" + } else { + "" + } + ); + } + (Some(node_jwt_validator), true, true) => { + tracing::info!(target: "node_auth", "node-auth: bearer token authentication enabled"); + layer.with_bearer_authenticator(node_jwt_validator.clone()) + } + (Some(_), false, _) => { + tracing::warn!( + target: "node_auth", + "node-auth: enabled but listener is not TLS; refusing to accept bearer tokens over plaintext" + ); + layer + } + (None, _, _) => layer, + } + }; let casbin_layer = if let Some(auth_config) = auth_config { if let Some(casbin_policy_file) = &auth_config.casbin_policy_file { let casbin_authorizer = Arc::new( @@ -416,6 +490,15 @@ pub(crate) async fn start( let mut tls_acceptor_created = Instant::now(); let mut initialize_tls_acceptor = true; + // How long until the next refresh attempt. Normally the rotation cadence; + // shortened after a failed rebuild so recovery does not wait out a full + // interval — but still a delay, because retrying on every accepted + // connection would turn a half-written identity file into one full rebuild + // (file reads, PEM parsing, a blocking task) per inbound connection. + let mut tls_refresh_after = TLS_REFRESH_INTERVAL; + // Refreshed alongside the TLS acceptor below; both read the same client-CA + // bundle, so they must not drift apart. + let node_jwt_validator = api_service.node_jwt_validator.clone(); join_set .build_task() @@ -440,23 +523,53 @@ pub(crate) async fn start( // the file on disk and only refresh if it's actually necessary to do so, // and emit a metric for the remaining duration on the cert - // hard refresh our certs every five minutes - // they may have been rewritten on disk by cert-manager and we want to honor the new cert. + // hard refresh our certs on the interval below (shortened after + // a failed rebuild); they may have been rewritten on disk by + // cert-manager and we want to honor the new cert. if let (Some(tls_config), true) = ( tls_config.as_ref(), - initialize_tls_acceptor - || tls_acceptor_created.elapsed() - > tokio::time::Duration::from_secs(5 * 60), + initialize_tls_acceptor || tls_acceptor_created.elapsed() > tls_refresh_after, ) { carbide_instrument::emit(TlsCertsRefreshed); initialize_tls_acceptor = false; tls_acceptor_created = Instant::now(); - tls_acceptor = tokio::task::Builder::new() - .name("get_tls_acceptor refresh") + // Node-auth JWTs chain to the same client-CA bundle the TLS + // listener verifies client certs against, so the acceptor + // and the validator's trust anchors have to move as one. + // Two ways that can go wrong, and both matter once + // mtls_enabled = false leaves tokens as the only + // credential: anchors left stale reject tokens issued under + // the new CA, and an acceptor dropped to `None` puts the + // listener on its plaintext branch while the bearer + // authenticator keeps accepting JWTs in the clear. + // + // So do every fallible step first and swap nothing until + // both succeed. Committing one without the other would + // leave the TLS path trusting one generation of the bundle + // and the token path another. + // One read of the client-CA bundle feeds both builders. + // Reading it separately in each would let a rotation land + // between them, so the pair could be committed together and + // still disagree about which generation they trust. + let (rebuilt_acceptor, rebuilt_jwt_roots) = tokio::task::Builder::new() + .name("tls trust rebuild") .spawn_blocking({ let tls_config = tls_config.clone(); - move || get_tls_acceptor(&tls_config) + let node_jwt_validator = node_jwt_validator.clone(); + move || { + let Some(client_ca) = read_client_ca(&tls_config) else { + return (None, Ok(None)); + }; + let acceptor = get_tls_acceptor(&tls_config, &client_ca); + let roots = match node_jwt_validator.as_ref() { + None => Ok(None), + Some(validator) => { + validator.build_roots_from_pem(&client_ca).map(Some) + } + }; + (acceptor, roots) + } }) // Safety: spawn_blocking only returns Error if run outside the tokio runtime .expect("Failed to spawn blocking task") @@ -464,6 +577,34 @@ pub(crate) async fn start( // Safety: Awaiting a JoinHandle only fails if the task panicked, and we want to // propagate panics .expect("task panicked"); + + match (rebuilt_acceptor, rebuilt_jwt_roots) { + (Some(acceptor), Ok(roots)) => { + // Commit phase: nothing below this line can fail. + if let (Some(validator), Some(roots)) = + (node_jwt_validator.as_ref(), roots) + { + validator.install_roots(roots); + } + tls_acceptor = Some(acceptor); + tls_refresh_after = TLS_REFRESH_INTERVAL; + } + (acceptor, roots) => { + // Come back sooner than the rotation cadence, but + // on a timer rather than on the next connection: + // the previous pair is still serving, so there is + // no urgency worth spending a rebuild per inbound + // connection on while the files stay broken. + tls_refresh_after = TLS_REFRESH_RETRY_DELAY; + tracing::error!( + target: "node_auth", + tls_acceptor_rebuilt = acceptor.is_some(), + jwt_roots_rebuilt = roots.is_ok(), + "node-auth: could not rebuild both the TLS acceptor and the \ + token trust anchors; keeping the previous pair and retrying" + ); + } + } } let tls_acceptor = tls_acceptor.clone(); diff --git a/crates/api-core/src/node_auth.rs b/crates/api-core/src/node_auth.rs new file mode 100644 index 0000000000..a99724030b --- /dev/null +++ b/crates/api-core/src/node_auth.rs @@ -0,0 +1,661 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Node-auth: validation of self-signed bearer JWTs from Scout / DPU-agent +//! (issue NVIDIA/infra-controller#355, simple variant). +//! +//! Nodes sign short-lived ES256 JWTs with the private key of their EXISTING +//! mTLS client certificate and carry the certificate chain in the token's +//! `x5c` header. [`NodeJwtValidator`] verifies, in order: +//! +//! 1. the `x5c` chain against the same root CAs the TLS listener trusts for +//! client certificates (chain of trust, validity window, client-auth EKU); +//! 2. the JWT signature against the verified leaf's public key (algorithm +//! pinned to ES256 — the only key type Vault PKI issues to machines); +//! 3. the registered claims: `exp` (with a bounded lifetime), `aud`; +//! 4. the SPIFFE constraints on the leaf and that the token's `sub` matches +//! the leaf's SPIFFE URI SAN — identity always derives from the verified +//! certificate, never from an attacker-controlled claim. +//! +//! The resulting SPIFFE URI is mapped by the authn middleware through the +//! SAME `SpiffeContext` as mTLS client certs, so a JWT and a cert for the +//! same machine yield an identical principal and reuse the existing RBAC +//! unchanged. There is no server-side key material and no issuance path: +//! "public key exchange" is the existing certificate PKI. + +use std::sync::{Arc, RwLock}; + +use carbide_authn::middleware::BearerTokenAuthenticator; +use jsonwebtoken::{Algorithm, DecodingKey, Validation, decode, decode_header}; +use rustls::RootCertStore; +use rustls::server::WebPkiClientVerifier; +use rustls::server::danger::ClientCertVerifier; +use rustls_pki_types::{CertificateDer, UnixTime}; +use serde::Deserialize; +use x509_parser::prelude::{FromDer, X509Certificate}; + +use crate::cfg::file::NodeAuthConfig; + +#[derive(Debug, thiserror::Error)] +pub(crate) enum NodeAuthError { + #[error("could not read root CA file {path}: {error}")] + RootCaRead { path: String, error: std::io::Error }, + #[error("root CA file {path} contains no usable trust anchors")] + NoTrustAnchors { path: String }, + #[error("could not build certificate verifier: {0}")] + Verifier(String), +} + +/// Why a presented bearer token was rejected. Only ever logged at debug — +/// rejection simply means the request proceeds without a bearer principal. +#[derive(Debug, thiserror::Error)] +enum RejectReason { + #[error("malformed JWT: {0}")] + Malformed(jsonwebtoken::errors::Error), + #[error("unexpected algorithm {0:?}; only ES256 is accepted")] + Algorithm(Algorithm), + #[error("no x5c certificate chain in the JWT header")] + NoChain, + #[error("x5c chain did not verify against the trusted roots: {0}")] + Chain(rustls::Error), + #[error("leaf certificate could not be parsed as X.509")] + LeafCertificateParse, + #[error("signature/claims validation failed: {0}")] + Claims(jsonwebtoken::errors::Error), + #[error("token lifetime exceeds the allowed maximum")] + Lifetime, + #[error("leaf certificate fails SPIFFE validation: {0}")] + Spiffe(String), + #[error("token `sub` does not match the certificate's SPIFFE URI")] + SubjectMismatch, + #[error("system clock is before the UNIX epoch")] + Clock, +} + +/// Registered claims checked on node tokens. `iat` is required during +/// deserialization so the bounded-lifetime check cannot be dodged by omitting +/// it. +#[derive(Debug, Deserialize)] +struct NodeClaims { + sub: String, + iat: u64, + exp: u64, +} + +/// Validates node-auth JWTs against the client-certificate PKI. +pub(crate) struct NodeJwtValidator { + /// Kept so the trust anchors can be re-read when the bundle rotates. + root_cafile_path: String, + /// Swapped in place by [`NodeJwtValidator::install_roots`]. The validator + /// is shared (the authn layer holds one `Arc` for the process lifetime), + /// so the refresh has to be interior, not a rebuild of the whole struct. + /// Held behind a lock rather than an `ArcSwap` because `arc-swap` cannot + /// store an unsized `dyn ClientCertVerifier`; the guard is released before + /// the (comparatively expensive) chain verification runs. + cert_verifier: RwLock>, + validation: Validation, + max_token_ttl_sec: u64, +} + +impl NodeJwtValidator { + /// Builds a validator trusting the given root CA bundle — the same file + /// the TLS listener uses to verify mTLS client certificates. + pub(crate) fn from_root_ca_file( + root_cafile_path: &str, + cfg: &NodeAuthConfig, + ) -> Result { + let cert_verifier = Self::build_verifier(root_cafile_path)?; + + let mut validation = Validation::new(Algorithm::ES256); + validation.set_audience(&[::rpc::node_jwt::NODE_JWT_AUDIENCE]); + // `jsonwebtoken` validates these registered claims. `iat` is required + // by `NodeClaims` and checked explicitly below because the library does + // not validate it. + validation.set_required_spec_claims(&["exp", "sub", "aud"]); + + Ok(Self { + root_cafile_path: root_cafile_path.to_string(), + cert_verifier: RwLock::new(cert_verifier), + validation, + max_token_ttl_sec: u64::from(cfg.max_token_ttl_sec), + }) + } + + /// Re-reads the root CA bundle from disk and returns a verifier built from + /// it, without installing it. + /// + /// The TLS listener reloads the same file every five minutes to pick up + /// cert-manager rotations; without this the validator would keep its + /// startup snapshot and start rejecting tokens whose `x5c` chains to the + /// new CA — which, with `mtls_enabled = false`, locks nodes out until the + /// API restarts. + /// + /// The fallible read is split from [`install_roots`](Self::install_roots) + /// so the listener can build this and its new TLS acceptor before + /// committing either. Both derive from this same bundle, and the listener + /// must never end up trusting one generation of it on the TLS path and + /// another on the token path — nor drop the acceptor and serve plaintext + /// while bearer auth stays armed. A failed build leaves the previous + /// verifier in place: a half-written bundle must not disarm node auth. + /// Test-only convenience. Production goes through + /// [`build_roots_from_pem`](Self::build_roots_from_pem) so the listener can + /// share one read of the bundle with the TLS acceptor. + #[cfg(test)] + pub(crate) fn build_roots(&self) -> Result, NodeAuthError> { + let pem = + std::fs::read(&self.root_cafile_path).map_err(|error| NodeAuthError::RootCaRead { + path: self.root_cafile_path.clone(), + error, + })?; + self.build_roots_from_pem(&pem) + } + + /// Builds anchors from a bundle the caller already read. + /// + /// The listener uses this so the TLS acceptor and this validator are built + /// from the *same* bytes: reading the file twice lets a rotation land + /// between the two reads, leaving each path trusting a different generation + /// of the client CA. + pub(crate) fn build_roots_from_pem( + &self, + pem: &[u8], + ) -> Result, NodeAuthError> { + Self::verifier_from_pem(&self.root_cafile_path, pem) + } + + /// Installs anchors from [`build_roots`](Self::build_roots). Infallible, so + /// it is safe to call in a commit phase alongside other swaps. + pub(crate) fn install_roots(&self, cert_verifier: Arc) { + *self + .cert_verifier + .write() + .unwrap_or_else(|poisoned| poisoned.into_inner()) = cert_verifier; + } + + fn build_verifier( + root_cafile_path: &str, + ) -> Result, NodeAuthError> { + let pem = std::fs::read(root_cafile_path).map_err(|error| NodeAuthError::RootCaRead { + path: root_cafile_path.to_string(), + error, + })?; + Self::verifier_from_pem(root_cafile_path, &pem) + } + + /// Parses an already-read bundle into a verifier. `root_cafile_path` is + /// carried only for error messages. + fn verifier_from_pem( + root_cafile_path: &str, + pem: &[u8], + ) -> Result, NodeAuthError> { + let mut roots = RootCertStore::empty(); + let certs = rustls_pemfile::certs(&mut std::io::Cursor::new(pem)) + .collect::, _>>() + .map_err(|e| NodeAuthError::Verifier(format!("root CA parse error: {e}")))?; + let (added, _ignored) = roots.add_parsable_certificates(certs); + if added == 0 { + return Err(NodeAuthError::NoTrustAnchors { + path: root_cafile_path.to_string(), + }); + } + + WebPkiClientVerifier::builder_with_provider( + Arc::new(roots), + Arc::new(rustls::crypto::aws_lc_rs::default_provider()), + ) + .allow_unknown_revocation_status() + .build() + .map_err(|e| NodeAuthError::Verifier(e.to_string())) + } + + fn validate(&self, token: &str) -> Result { + let header = decode_header(token).map_err(RejectReason::Malformed)?; + if header.alg != Algorithm::ES256 { + return Err(RejectReason::Algorithm(header.alg)); + } + + // 1. The certificate chain must verify against the trusted roots. + let chain = header + .x5c_der() + .map_err(RejectReason::Malformed)? + .filter(|chain| !chain.is_empty()) + .ok_or(RejectReason::NoChain)?; + let leaf = CertificateDer::from(chain[0].clone()); + let intermediates: Vec = chain[1..] + .iter() + .map(|der| CertificateDer::from(der.clone())) + .collect(); + let cert_verifier = self + .cert_verifier + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone(); + cert_verifier + .verify_client_cert(&leaf, &intermediates, UnixTime::now()) + .map_err(RejectReason::Chain)?; + + // 2. The token must be signed by the verified leaf's key. + let (_, x509) = X509Certificate::from_der(leaf.as_ref()) + .map_err(|_| RejectReason::LeafCertificateParse)?; + let decoding_key = DecodingKey::from_ec_der(&x509.public_key().subject_public_key.data); + let claims = decode::(token, &decoding_key, &self.validation) + .map_err(RejectReason::Claims)? + .claims; + + // 3. Bounded lifetime: the client controls `iat` and `exp`, so reject + // invalid ordering and cap how far in the future `exp` may reach. + // `jsonwebtoken` already rejected expired tokens above. + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(|_| RejectReason::Clock)? + .as_secs(); + // The wall-clock arm gets the same skew tolerance `jsonwebtoken` already + // applies to `exp`, taken from the same field so the two cannot drift + // apart. It also bounds an `iat` that is slightly ahead of the API's + // clock. Without it, `max_token_ttl_sec` set to exactly the client's + // lifetime -- the smallest value startup accepts -- rejects every token + // the moment the API's clock sits a second behind the node's, which is + // ordinary between hosts. The `exp - iat` arm is unaffected: it compares + // two claims from one clock, so skew cannot reach it. + let skew = self.validation.leeway; + if claims.iat > claims.exp + || claims.iat > now + skew + || claims.exp - claims.iat > self.max_token_ttl_sec + || claims.exp > now + self.max_token_ttl_sec + skew + { + return Err(RejectReason::Lifetime); + } + + // 4. Identity comes from the verified certificate, with `sub` only + // cross-checked against it. + let spiffe_id = carbide_authn::validate_x509_certificate(leaf.as_ref()) + .map_err(|e| RejectReason::Spiffe(e.to_string()))?; + let spiffe_uri = spiffe_id.to_string(); + if claims.sub != spiffe_uri { + return Err(RejectReason::SubjectMismatch); + } + Ok(spiffe_uri) + } +} + +impl BearerTokenAuthenticator for NodeJwtValidator { + fn spiffe_id_from_bearer(&self, token: &str) -> Option { + match self.validate(token) { + Ok(spiffe_uri) => Some(spiffe_uri), + Err(reason) => { + tracing::debug!(target: "node_auth", %reason, "node-auth: rejected bearer token"); + None + } + } + } +} + +#[cfg(test)] +mod tests { + use base64::Engine as _; + use carbide_test_support::{Check, check_values}; + use rpc::node_jwt::NodeJwtMinter; + + use super::*; + + const TRUST_DOMAIN: &str = "forge.local"; + const MACHINE_PATH: &str = "/forge-system/machine/fm100xtest"; + + struct TestPki { + ca_pem: String, + cert_pem: String, + key_pem: String, + } + + /// A CA plus a leaf it issued carrying the machine SPIFFE URI SAN — + /// stand-ins for the Vault PKI root and a node's client certificate. + fn test_pki(spiffe_path: &str) -> TestPki { + let mut ca_params = rcgen::CertificateParams::default(); + ca_params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + ca_params + .distinguished_name + .push(rcgen::DnType::CommonName, "test root"); + let ca_key = rcgen::KeyPair::generate().expect("ca key"); + let ca_cert = ca_params.clone().self_signed(&ca_key).expect("ca cert"); + let issuer = rcgen::Issuer::new(ca_params, ca_key); + + let mut leaf_params = rcgen::CertificateParams::default(); + leaf_params.subject_alt_names = vec![rcgen::SanType::URI( + rcgen::string::Ia5String::try_from(format!("spiffe://{TRUST_DOMAIN}{spiffe_path}")) + .expect("uri"), + )]; + leaf_params.use_authority_key_identifier_extension = true; + leaf_params + .extended_key_usages + .push(rcgen::ExtendedKeyUsagePurpose::ClientAuth); + let leaf_key = rcgen::KeyPair::generate().expect("leaf key"); + let leaf_cert = leaf_params + .signed_by(&leaf_key, &issuer) + .expect("leaf cert"); + + TestPki { + ca_pem: ca_cert.pem(), + cert_pem: leaf_cert.pem(), + key_pem: leaf_key.serialize_pem(), + } + } + + fn write_temp(dir: &tempfile::TempDir, name: &str, contents: &str) -> String { + let path = dir.path().join(name); + std::fs::write(&path, contents).expect("write"); + path.to_string_lossy().into_owned() + } + + fn validator_for(dir: &tempfile::TempDir, ca_pem: &str) -> NodeJwtValidator { + let ca_path = write_temp(dir, "ca.pem", ca_pem); + NodeJwtValidator::from_root_ca_file(&ca_path, &NodeAuthConfig::default()) + .expect("validator builds") + } + + fn mint_with(dir: &tempfile::TempDir, pki: &TestPki) -> String { + let minter = NodeJwtMinter::new( + write_temp(dir, "cert.pem", &pki.cert_pem), + write_temp(dir, "key.pem", &pki.key_pem), + ); + minter.current().expect("token minted") + } + + fn header_with_certificate_chain(pki: &TestPki, algorithm: Algorithm) -> jsonwebtoken::Header { + let certs = rustls_pemfile::certs(&mut std::io::Cursor::new(&pki.cert_pem)) + .collect::, _>>() + .expect("test certificate parses"); + let mut header = jsonwebtoken::Header::new(algorithm); + header.x5c = Some( + certs + .iter() + .map(|certificate| { + base64::engine::general_purpose::STANDARD.encode(certificate.as_ref()) + }) + .collect(), + ); + header + } + + fn leaf_signed_token(pki: &TestPki, claims: serde_json::Value) -> String { + let encoding_key = jsonwebtoken::EncodingKey::from_ec_pem(pki.key_pem.as_bytes()) + .expect("test leaf key parses"); + jsonwebtoken::encode( + &header_with_certificate_chain(pki, Algorithm::ES256), + &claims, + &encoding_key, + ) + .expect("test token encodes") + } + + fn unix_now() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system clock is after UNIX epoch") + .as_secs() + } + + /// A rotated client-CA bundle has to be picked up without a restart: the + /// TLS listener re-reads the same file every five minutes, and tokens + /// chaining to the new CA must start verifying once it does. + #[test] + fn rotated_root_ca_is_honored_after_refresh() { + let dir = tempfile::tempdir().expect("tempdir"); + let old_pki = test_pki(MACHINE_PATH); + let ca_path = write_temp(&dir, "ca.pem", &old_pki.ca_pem); + let validator = NodeJwtValidator::from_root_ca_file(&ca_path, &NodeAuthConfig::default()) + .expect("validator builds"); + + // A second CA, as though cert-manager had rotated the bundle. + let new_pki = test_pki(MACHINE_PATH); + let new_dir = tempfile::tempdir().expect("tempdir"); + let token = mint_with(&new_dir, &new_pki); + assert_eq!( + validator.spiffe_id_from_bearer(&token), + None, + "a token from the not-yet-trusted CA must be rejected" + ); + + std::fs::write(&ca_path, &new_pki.ca_pem).expect("rotate the bundle on disk"); + validator.install_roots(validator.build_roots().expect("roots reload")); + + assert_eq!( + validator.spiffe_id_from_bearer(&token).as_deref(), + Some(format!("spiffe://{TRUST_DOMAIN}{MACHINE_PATH}").as_str()), + "after the refresh the rotated CA must be trusted" + ); + } + + /// A bundle that cannot be read or parsed must not disarm bearer auth. + #[test] + fn a_broken_bundle_leaves_the_previous_roots_in_place() { + let dir = tempfile::tempdir().expect("tempdir"); + let pki = test_pki(MACHINE_PATH); + let ca_path = write_temp(&dir, "ca.pem", &pki.ca_pem); + let validator = NodeJwtValidator::from_root_ca_file(&ca_path, &NodeAuthConfig::default()) + .expect("validator builds"); + let token = mint_with(&dir, &pki); + + std::fs::write(&ca_path, "not a certificate").expect("truncate the bundle"); + validator + .build_roots() + .expect_err("a bundle with no trust anchors must not be accepted"); + + assert_eq!( + validator.spiffe_id_from_bearer(&token).as_deref(), + Some(format!("spiffe://{TRUST_DOMAIN}{MACHINE_PATH}").as_str()), + "the previous roots must still verify tokens" + ); + } + + #[test] + fn client_minted_token_round_trips_to_the_cert_spiffe_uri() { + let dir = tempfile::tempdir().expect("tempdir"); + let pki = test_pki(MACHINE_PATH); + let validator = validator_for(&dir, &pki.ca_pem); + + let token = mint_with(&dir, &pki); + assert_eq!( + validator.spiffe_id_from_bearer(&token).as_deref(), + Some(format!("spiffe://{TRUST_DOMAIN}{MACHINE_PATH}").as_str()) + ); + } + + #[test] + fn token_from_an_untrusted_ca_is_rejected() { + let dir = tempfile::tempdir().expect("tempdir"); + let pki = test_pki(MACHINE_PATH); + let other_pki = test_pki(MACHINE_PATH); + // Validator trusts a DIFFERENT root than the one that issued the cert. + let validator = validator_for(&dir, &other_pki.ca_pem); + + let token = mint_with(&dir, &pki); + assert!(validator.spiffe_id_from_bearer(&token).is_none()); + } + + #[test] + fn garbage_and_missing_chain_tokens_are_rejected() { + let dir = tempfile::tempdir().expect("tempdir"); + let pki = test_pki(MACHINE_PATH); + let validator = validator_for(&dir, &pki.ca_pem); + + assert!(validator.spiffe_id_from_bearer("not.a.jwt").is_none()); + + // Structurally valid ES256 JWT without an x5c header. + let key = rcgen::KeyPair::generate().expect("key"); + let encoding_key = jsonwebtoken::EncodingKey::from_ec_pem(key.serialize_pem().as_bytes()) + .expect("encoding key"); + let claims = serde_json::json!({ + "sub": format!("spiffe://{TRUST_DOMAIN}{MACHINE_PATH}"), + "aud": "nico-api", "iat": 0u64, "exp": u64::MAX / 2, + }); + let no_chain = jsonwebtoken::encode( + &jsonwebtoken::Header::new(Algorithm::ES256), + &claims, + &encoding_key, + ) + .expect("token"); + assert!(validator.spiffe_id_from_bearer(&no_chain).is_none()); + } + + /// Both the identity cross-check and algorithm pin are independent of the + /// x5c-chain verification. Keep explicit coverage so an otherwise-valid + /// certificate cannot accidentally make either attacker-controlled header + /// or claim authoritative. + #[test] + fn certificate_backed_tokens_reject_subject_mismatch_and_other_algorithms() { + #[derive(Debug, Eq, PartialEq)] + enum Rejection { + Algorithm, + SubjectMismatch, + } + + let dir = tempfile::tempdir().expect("tempdir"); + let pki = test_pki(MACHINE_PATH); + let validator = validator_for(&dir, &pki.ca_pem); + let now = unix_now(); + let valid_times = serde_json::json!({ + "aud": "nico-api", "iat": now, "exp": now + 60, + }); + let subject_mismatch = leaf_signed_token( + &pki, + serde_json::json!({ + "sub": format!("spiffe://{TRUST_DOMAIN}/forge-system/machine/other"), + "aud": valid_times["aud"], + "iat": valid_times["iat"], + "exp": valid_times["exp"], + }), + ); + let other_algorithm = jsonwebtoken::encode( + &header_with_certificate_chain(&pki, Algorithm::HS256), + &serde_json::json!({ + "sub": format!("spiffe://{TRUST_DOMAIN}{MACHINE_PATH}"), + "aud": valid_times["aud"], + "iat": valid_times["iat"], + "exp": valid_times["exp"], + }), + &jsonwebtoken::EncodingKey::from_secret(b"test signing key"), + ) + .expect("test token encodes"); + + check_values( + [ + Check { + scenario: "subject does not match the certificate's SPIFFE URI", + input: subject_mismatch, + expect: Rejection::SubjectMismatch, + }, + Check { + scenario: "a non-ES256 signature still carries a valid certificate chain", + input: other_algorithm, + expect: Rejection::Algorithm, + }, + ], + |token| match validator.validate(&token) { + Err(RejectReason::Algorithm(_)) => Rejection::Algorithm, + Err(RejectReason::SubjectMismatch) => Rejection::SubjectMismatch, + result => panic!("unexpected node-token validation result: {result:?}"), + }, + ); + } + + #[test] + fn future_or_reversed_issue_times_are_rejected() { + let dir = tempfile::tempdir().expect("tempdir"); + let pki = test_pki(MACHINE_PATH); + let validator = validator_for(&dir, &pki.ca_pem); + let now = unix_now(); + let subject = format!("spiffe://{TRUST_DOMAIN}{MACHINE_PATH}"); + + check_values( + [ + Check { + scenario: "issue time is beyond the allowed clock skew", + input: leaf_signed_token( + &pki, + serde_json::json!({ + "sub": subject, + "aud": "nico-api", + "iat": now + 120, + "exp": now + 180, + }), + ), + expect: true, + }, + Check { + scenario: "issue time is after expiration", + input: leaf_signed_token( + &pki, + serde_json::json!({ + "sub": format!("spiffe://{TRUST_DOMAIN}{MACHINE_PATH}"), + "aud": "nico-api", + "iat": now + 240, + "exp": now + 180, + }), + ), + expect: true, + }, + ], + |token| matches!(validator.validate(&token), Err(RejectReason::Lifetime)), + ); + } + + #[test] + fn overlong_lifetime_is_rejected() { + let dir = tempfile::tempdir().expect("tempdir"); + let pki = test_pki(MACHINE_PATH); + let ca_path = write_temp(&dir, "ca.pem", &pki.ca_pem); + // A validator whose lifetime cap is below what the client mints. + let strict = NodeJwtValidator::from_root_ca_file( + &ca_path, + &NodeAuthConfig { + max_token_ttl_sec: 1, + ..NodeAuthConfig::default() + }, + ) + .expect("validator builds"); + + let token = mint_with(&dir, &pki); + assert!(strict.spiffe_id_from_bearer(&token).is_none()); + } + + /// The smallest cap startup accepts is exactly the client's lifetime, so + /// that value has to work in practice -- including when the API's clock + /// trails the node's, which is ordinary between hosts. Without a skew + /// allowance on the wall-clock arm, one second of lag rejects every token. + #[test] + fn the_exact_client_lifetime_cap_survives_a_lagging_api_clock() { + let dir = tempfile::tempdir().expect("tempdir"); + let pki = test_pki(MACHINE_PATH); + let ca_path = write_temp(&dir, "ca.pem", &pki.ca_pem); + let exact = NodeJwtValidator::from_root_ca_file( + &ca_path, + &NodeAuthConfig { + max_token_ttl_sec: u32::try_from(::rpc::node_jwt::NODE_JWT_TTL_SECS) + .expect("client TTL fits"), + ..NodeAuthConfig::default() + }, + ) + .expect("validator builds"); + + let token = mint_with(&dir, &pki); + assert_eq!( + exact.spiffe_id_from_bearer(&token).as_deref(), + Some(format!("spiffe://{TRUST_DOMAIN}{MACHINE_PATH}").as_str()), + "a token minted at the cap must validate, with room for clock skew" + ); + } +} diff --git a/crates/api-core/src/setup.rs b/crates/api-core/src/setup.rs index aeca59e675..2cc2a97ffb 100644 --- a/crates/api-core/src/setup.rs +++ b/crates/api-core/src/setup.rs @@ -423,6 +423,36 @@ pub(crate) async fn start_runtime( .map_err(|e| eyre::eyre!("failed to build NMX-C client pool: {e}"))?; let shared_nmxc_pool: Arc = Arc::new(nmxc_client_pool); + // Node-auth (Scout / DPU-agent bearer JWT, #355) preflight. Run before any + // DPF resource creation below, so a misconfiguration (the + // enabled=false + mtls_enabled=false lockout, bearer-over-plaintext, or an + // unreadable trust anchor) fails startup before it mutates cluster state. + // Validate unconditionally; when explicitly enabled, missing prerequisites + // fail rather than silently degrading. + carbide_config.node_auth.validate()?; + let node_jwt_validator = if carbide_config.node_auth.enabled { + // Bearer tokens must never be accepted over plaintext, and the + // validator trusts the same roots the TLS listener uses for client + // certificates — so a TLS listener is required on both counts. + if !matches!(carbide_config.listen_mode, ListenMode::Tls) { + return Err(eyre::eyre!( + "[node_auth] is enabled but listen_mode is not \"tls\"; bearer tokens must not be accepted over plaintext" + )); + } + let tls_ref = carbide_config + .tls + .as_ref() + .ok_or_else(|| eyre::eyre!("[node_auth] is enabled but [tls] is unset"))?; + Some(Arc::new( + crate::node_auth::NodeJwtValidator::from_root_ca_file( + &tls_ref.root_cafile_path, + &carbide_config.node_auth, + )?, + )) + } else { + None + }; + let dpf_sdk = initialize_dpf_sdk( &carbide_config, credential_manager.clone(), @@ -482,6 +512,7 @@ pub(crate) async fn start_runtime( certificate_provider, common_pools, credential_manager, + node_jwt_validator, database_connection: db_pool.clone(), dpu_health_log_limiter: LogLimiter::default(), dynamic_settings, @@ -708,6 +739,7 @@ async fn initialize_dpf_sdk( &services, &carbide_config.dpf.dpu_agent_bootstrap_ca, interfaces, + &carbide_config.node_auth, ), num_of_vfs: carbide_config.dpu_config.num_of_vfs, pf_total_sf_reserved: carbide_config.dpf.pf_total_sf_reserved, diff --git a/crates/api-core/src/test_support/builder.rs b/crates/api-core/src/test_support/builder.rs index d4262a051c..e9893802a4 100644 --- a/crates/api-core/src/test_support/builder.rs +++ b/crates/api-core/src/test_support/builder.rs @@ -290,6 +290,7 @@ impl TestApiBuilder { bmc_session_manager, bms_client: std::sync::OnceLock::new(), secrets_context: self.secrets_context, + node_jwt_validator: None, } } } diff --git a/crates/api-core/src/test_support/default_config.rs b/crates/api-core/src/test_support/default_config.rs index 1bd189107d..f1e8f7559b 100644 --- a/crates/api-core/src/test_support/default_config.rs +++ b/crates/api-core/src/test_support/default_config.rs @@ -43,7 +43,7 @@ use crate::cfg::file::{ DpuConfig as InitialDpuConfig, DsxExchangeEventBusConfig, FnnConfig, IbPartitionStateControllerConfig, KmsConfig, ListenMode, MachineUpdater, MeasuredBootMetricsCollectorConfig, MqttAuthConfig, NetworkSecurityGroupConfig, - NetworkSegmentStateControllerConfig, PowerShelfStateControllerConfig, + NetworkSegmentStateControllerConfig, NodeAuthConfig, PowerShelfStateControllerConfig, RackStateControllerConfig, SecretsConfig, SpdmConfig, SpdmStateControllerConfig, SwitchStateControllerConfig, TracingConfig, VmaasConfig, VpcPeeringPolicy, VpcPrefixStateControllerConfig, default_bmc_session_lockout_threshold, @@ -103,9 +103,14 @@ pub(crate) fn with_dpf_intercept_topology(selected_vfs: &[u8]) -> CarbideConfig /// [`get`] with every `Option` config section populated. Used by tests that /// walk the *serialized* config shape — e.g. the admin-UI documentation -/// guards, which can only verify sections that actually serialize. When a -/// new `Option` section is added to [`CarbideConfig`] (the compiler forces -/// it into [`get`]), populate it here too so those guards can see inside it. +/// guards, which can only verify sections and fields that actually serialize. +/// When a new `Option` section is added to [`CarbideConfig`] (the compiler +/// forces it into [`get`]), populate it here too. The +/// `fmds_use_node_tokens` inner `Option` below is populated so its documented +/// row is covered. New `skip_serializing_if` fields need the same explicit +/// treatment when they should be checked; this is not compiler-enforced. +/// `mlxconfig_profiles` is intentionally exempted by `SKIP_SERIALIZING` in +/// `crates/api-web/src/configuration.rs`. pub fn fully_populated() -> CarbideConfig { CarbideConfig { auth: Some(AuthConfig { @@ -134,6 +139,10 @@ pub fn fully_populated() -> CarbideConfig { import_from: None, import_approach: Default::default(), }), + node_auth: NodeAuthConfig { + fmds_use_node_tokens: Some(false), + ..Default::default() + }, ..get() } } @@ -145,6 +154,7 @@ pub fn get() -> CarbideConfig { web_ui_sidebar_tools: vec![], web_ui_logs_link_template: String::new(), log_history: Default::default(), + node_auth: Default::default(), observability: Default::default(), bgp_leaf_session_password: None, rack_validation_config: RackValidationConfig { diff --git a/crates/authn/Cargo.toml b/crates/authn/Cargo.toml index f7784741b7..1e86d09c9a 100644 --- a/crates/authn/Cargo.toml +++ b/crates/authn/Cargo.toml @@ -42,6 +42,7 @@ carbide-instrument = { path = "../instrument", features = ["test-support"] } carbide-rpc = { path = "../rpc" } carbide-test-support = { path = "../test-support" } rcgen = { workspace = true } +tokio = { workspace = true } [lints] workspace = true diff --git a/crates/authn/src/middleware.rs b/crates/authn/src/middleware.rs index 930b7c09c5..c52c0bf903 100644 --- a/crates/authn/src/middleware.rs +++ b/crates/authn/src/middleware.rs @@ -36,10 +36,33 @@ use crate::{SpiffeContext, SpiffeError}; // This middleware is not expected to enforce anything on its own, so anything // that an access control policy might need to do its work should be passed // along in the request extensions. +/// Verifies a bearer token (a node-auth JWT) and, on success, returns the +/// validated SPIFFE ID URI string from the token's `sub` claim. The +/// implementation lives outside this crate (api-core's `node_auth`) so the +/// authn layer stays free of JWT signing/key-management concerns. +/// +/// Returning the raw SPIFFE URI — rather than a `Principal` — lets the +/// middleware map it through the SAME [`SpiffeContext`] used for client certs, +/// so a JWT and an mTLS cert for the same machine yield identical principals. +pub trait BearerTokenAuthenticator: Send + Sync { + /// Returns the validated SPIFFE ID URI if `token` is a valid node-auth JWT, + /// or `None` if it is invalid/expired/unrecognized. + fn spiffe_id_from_bearer(&self, token: &str) -> Option; +} + #[derive(Clone)] pub struct CertDescriptionMiddleware { pub spiffe_context: Arc, pub extra_allowed_certs: Option, + /// Optional bearer-token (JWT) authenticator. When set, requests carrying an + /// `Authorization: Bearer ` header gain principals alongside any from + /// mTLS client certs (dual-support during the mTLS→JWT migration). + pub bearer_authenticator: Option>, + /// Whether machine mTLS client certificates mint machine principals. On by + /// default; disabled via `[node_auth] mtls_enabled = false` once a fleet has + /// migrated to bearer tokens. Scoped to machine certs — service and admin + /// client certs are unaffected. + pub machine_certs_enabled: bool, _authorization: std::marker::PhantomData, } @@ -51,9 +74,29 @@ impl CertDescriptionMiddleware { CertDescriptionMiddleware { spiffe_context: Arc::new(spiffe_context), extra_allowed_certs, + bearer_authenticator: None, + machine_certs_enabled: true, _authorization: std::marker::PhantomData, } } + + /// Enables bearer-token (JWT) authentication using the given authenticator. + #[must_use] + pub fn with_bearer_authenticator( + mut self, + authenticator: Arc, + ) -> Self { + self.bearer_authenticator = Some(authenticator); + self + } + + /// Controls whether machine mTLS client certificates are accepted as node + /// identity (`[node_auth] mtls_enabled`). + #[must_use] + pub fn with_machine_certs_enabled(mut self, enabled: bool) -> Self { + self.machine_certs_enabled = enabled; + self + } } impl Layer for CertDescriptionMiddleware { @@ -218,6 +261,36 @@ impl Principal { } } +/// Extracts the token from an `Authorization: Bearer ` header, if present. +/// +/// RFC 6750 spells the scheme with an ABNF quoted literal, which RFC 5234 +/// makes case-insensitive, and allows `1*SP` before the token — so `bearer` +/// and extra spaces are both legal and must be accepted. Our own clients only +/// ever send `Bearer `, but a third-party one that doesn't would +/// otherwise be turned away with nothing written anywhere, so the reject path +/// logs. Neither branch logs the header value: it is a live credential. +fn bearer_token_from_headers(headers: &hyper::HeaderMap) -> Option<&str> { + let header = headers.get(hyper::header::AUTHORIZATION)?; + let Ok(value) = header.to_str() else { + tracing::debug!( + target: "node_auth", + "node-auth: ignoring an Authorization header that is not valid UTF-8" + ); + return None; + }; + + match value.split_once(' ') { + Some((scheme, token)) if scheme.eq_ignore_ascii_case("Bearer") => Some(token.trim()), + _ => { + tracing::debug!( + target: "node_auth", + "node-auth: ignoring an Authorization header that is not a Bearer credential" + ); + None + } + } +} + // try_external_cert will return a Pricipal::ExternalUser if this looks like some external cert fn try_external_cert( der_certificate: &[u8], @@ -540,12 +613,55 @@ where } fn call(&mut self, mut request: Request) -> Self::Future { - if let Some(_req_auth_header) = request.headers().get(hyper::header::AUTHORIZATION) { - // If we want to extract additional principals from the request's - // Authorization header, we can do it here. + let mut auth_context = AuthContext::::default(); + + // Bearer-token (node-auth JWT) authentication. Validated tokens map + // through the same SpiffeContext as client certs. A recognized bearer + // credential also represents a trusted identity for authorization: it + // was signed by the node's certificate and validated by the configured + // authenticator, so it needs the same Casbin grant as mTLS. This runs + // in addition to (not instead of) cert auth to support the migration. + if let Some(authenticator) = &self.authorization_context.bearer_authenticator + && let Some(token) = bearer_token_from_headers(request.headers()) + && let Some(spiffe_uri) = authenticator.spiffe_id_from_bearer(token) + { + let principal = match crate::spiffe_id::SpiffeId::new(&spiffe_uri) { + Ok(spiffe_id) => match self + .authorization_context + .spiffe_context + .extract_service_identifier(&spiffe_id) + { + Ok(crate::SpiffeIdClass::Service(id)) => { + Some(Principal::SpiffeServiceIdentifier(id)) + } + Ok(crate::SpiffeIdClass::Machine(id)) => { + Some(Principal::SpiffeMachineIdentifier(id)) + } + Err(e) => { + tracing::debug!( + target: "node_auth", + error = %e, + "node-auth: bearer token SPIFFE id not recognized" + ); + None + } + }, + Err(e) => { + tracing::debug!( + target: "node_auth", + error = %e, + "node-auth: bearer token contained an unparsable SPIFFE URI" + ); + None + } + }; + if let Some(principal) = principal { + auth_context.principals.push(principal); + auth_context.principals.push(Principal::TrustedCertificate); + } } + let extensions = request.extensions_mut(); - let mut auth_context = AuthContext::::default(); if let Some(conn_attrs) = extensions.get::>() { let peer_certs = &conn_attrs.peer_certificates; // rustls presents the end-entity certificate first, intermediates @@ -556,16 +672,28 @@ where // per request, only when the whole chain minted no principal. let mut rejections = Vec::new(); let minted_before = auth_context.principals.len(); - let peer_cert_principals = peer_certs.iter().filter_map(|cert| { + // Tracked so the `TrustedCertificate` decision below can tell a + // machine cert we deliberately refused from one that simply minted + // nothing. + let mut refused_machine_cert = false; + for cert in peer_certs { match Principal::try_from_client_certificate(cert, &self.authorization_context) { - Ok(x) => Some(x), - Err(e) => { - rejections.push(e); - None + // `[node_auth] mtls_enabled = false`: machine certs no + // longer grant node identity (bearer JWTs are the only node + // auth path); service/admin cert principals pass through. + Ok(Principal::SpiffeMachineIdentifier(_)) + if !self.authorization_context.machine_certs_enabled => + { + refused_machine_cert = true; + tracing::debug!( + target: "node_auth", + "node-auth: machine mTLS authentication disabled; ignoring machine client certificate" + ); } + Ok(principal) => auth_context.principals.push(principal), + Err(e) => rejections.push(e), } - }); - auth_context.principals.extend(peer_cert_principals); + } if auth_context.principals.len() == minted_before && let Some(leaf_error) = rejections.first() { @@ -578,7 +706,19 @@ where // Regardless of whether we were able to get a specific Principal // flavor out of the certificate, having a trusted certificate // presented by the client is worth recording on its own. - if !peer_certs.is_empty() { + // + // Except when the only thing presented was a machine certificate we + // just refused. `TrustedCertificate` is not a bookkeeping marker — + // the shipped Casbin policy grants it `forge/*` and `nico/*`, so + // handing it out here would re-authorize the very request the + // machine-cert gate above declined, and `mtls_enabled = false` + // would filter the machine principal while leaving the caller fully + // authorized under another name. Scoped to that case, so service and + // admin-CLI certs keep it, as does a cert that failed to mint a + // principal for unrelated reasons. + let refused_the_only_credential = + refused_machine_cert && auth_context.principals.len() == minted_before; + if !peer_certs.is_empty() && !refused_the_only_credential { auth_context.principals.push(Principal::TrustedCertificate); } } else { @@ -593,10 +733,13 @@ where #[cfg(test)] mod tests { use std::collections::HashSet; + use std::convert::Infallible; use std::sync::Mutex; use std::task::{Context, Poll}; use carbide_instrument::testing::{MetricsCapture, capture_logs}; + use hyper::header::AUTHORIZATION; + use tower::{Layer, ServiceExt}; use super::*; use crate::spiffe_id::TrustDomain; @@ -661,6 +804,14 @@ mod tests { } } + /// Test authenticator: maps the literal token "good" to a fixed SPIFFE URI. + struct FakeAuth(String); + impl BearerTokenAuthenticator for FakeAuth { + fn spiffe_id_from_bearer(&self, token: &str) -> Option { + (token == "good").then(|| self.0.clone()) + } + } + fn spiffe_context() -> SpiffeContext { SpiffeContext { trust_domain: TrustDomain::new("example.test").expect("trust domain"), @@ -840,4 +991,294 @@ mod tests { "recognition" ); } + + /// Runs a request through the middleware and returns the principals the authn + /// layer attached (captured from the request extensions by an inner service). + /// + /// The request carries `ConnectionAttributes` with no peer certificates — + /// a bearer-token client over plain TLS. Omitting them entirely is the + /// wiring-error path, which logs `authentication_connection_attributes_missing` + /// and would poison that callsite's cached tracing interest for the + /// capture-based test of that warning. + async fn principals_for( + middleware: CertDescriptionMiddleware, + auth_header: Option<&str>, + ) -> Vec { + principals_for_certs(middleware, auth_header, Vec::new()).await + } + + #[tokio::test] + async fn valid_bearer_token_yields_machine_principal() { + let middleware = CertDescriptionMiddleware::::new(None, spiffe_context()) + .with_bearer_authenticator(Arc::new(FakeAuth( + "spiffe://example.test/carbide-system/machine/m1".to_string(), + ))); + let principals = principals_for(middleware, Some("Bearer good")).await; + assert!( + principals.contains(&Principal::SpiffeMachineIdentifier("m1".to_string())), + "expected machine principal, got {principals:?}" + ); + assert!( + principals.contains(&Principal::TrustedCertificate), + "a validated bearer token must authorize as a trusted identity, got {principals:?}" + ); + } + + /// RFC 6750's scheme is case-insensitive and allows more than one space + /// before the token. Our own clients send exactly `Bearer `, but a + /// conforming third-party client must not be turned away. + #[tokio::test] + async fn bearer_scheme_is_matched_case_insensitively() { + for header in ["bearer good", "BEARER good", "BeArEr good", "Bearer good"] { + let middleware = + CertDescriptionMiddleware::::new(None, spiffe_context()) + .with_bearer_authenticator(Arc::new(FakeAuth( + "spiffe://example.test/carbide-system/machine/m1".to_string(), + ))); + let principals = principals_for(middleware, Some(header)).await; + assert!( + principals.contains(&Principal::SpiffeMachineIdentifier("m1".to_string())), + "{header:?} is a valid bearer credential, got {principals:?}" + ); + } + } + + /// A different scheme must not be read as a bearer token, and neither must + /// a bare value with no scheme at all. + #[tokio::test] + async fn a_non_bearer_authorization_header_is_ignored() { + for header in ["Basic good", "good", "Bearer"] { + let middleware = + CertDescriptionMiddleware::::new(None, spiffe_context()) + .with_bearer_authenticator(Arc::new(FakeAuth( + "spiffe://example.test/carbide-system/machine/m1".to_string(), + ))); + let principals = principals_for(middleware, Some(header)).await; + assert!( + !principals + .iter() + .any(|p| matches!(p, Principal::SpiffeMachineIdentifier(_))), + "{header:?} must not authenticate, got {principals:?}" + ); + } + } + + #[tokio::test] + async fn invalid_bearer_token_yields_no_machine_principal() { + let middleware = CertDescriptionMiddleware::::new(None, spiffe_context()) + .with_bearer_authenticator(Arc::new(FakeAuth( + "spiffe://example.test/carbide-system/machine/m1".to_string(), + ))); + let principals = principals_for(middleware, Some("Bearer bogus")).await; + assert!( + !principals + .iter() + .any(|p| matches!(p, Principal::SpiffeMachineIdentifier(_))), + "rejected token must not yield a machine principal, got {principals:?}" + ); + assert!( + !principals.contains(&Principal::TrustedCertificate), + "rejected token must not earn trusted-certificate, got {principals:?}" + ); + } + + /// Like [`principals_for`], but the request also presents client + /// certificates via [`ConnectionAttributes`]. + async fn principals_for_certs( + middleware: CertDescriptionMiddleware, + auth_header: Option<&str>, + peer_certificates: Vec>, + ) -> Vec { + let inner = tower::service_fn(|req: Request| async move { + let principals = req + .extensions() + .get::>() + .map(|ctx| ctx.principals.clone()) + .unwrap_or_default(); + Ok::<_, Infallible>(principals) + }); + let svc = middleware.layer(inner); + let mut builder = Request::builder(); + if let Some(value) = auth_header { + builder = builder.header(AUTHORIZATION, value); + } + let mut request = builder.body(tonic::body::Body::empty()).unwrap(); + request + .extensions_mut() + .insert(Arc::new(ConnectionAttributes { + peer_address: "192.0.2.9:4433".parse().expect("socket address"), + peer_certificates, + })); + svc.oneshot(request).await.unwrap() + } + + #[tokio::test] + async fn machine_cert_yields_machine_principal_by_default() { + let middleware = CertDescriptionMiddleware::::new(None, spiffe_context()); + let principals = principals_for_certs( + middleware, + None, + vec![spiffe_leaf_certificate("/carbide-system/machine/m1")], + ) + .await; + assert!( + principals.contains(&Principal::SpiffeMachineIdentifier("m1".to_string())), + "expected cert-derived machine principal, got {principals:?}" + ); + } + + #[tokio::test] + async fn machine_cert_ignored_when_machine_certs_disabled() { + // `[node_auth] mtls_enabled = false`: a machine client cert mints no + // machine principal, while a valid bearer token still does — the JWT + // becomes the only node auth path. + let middleware = CertDescriptionMiddleware::::new(None, spiffe_context()) + .with_machine_certs_enabled(false) + .with_bearer_authenticator(Arc::new(FakeAuth( + "spiffe://example.test/carbide-system/machine/m2".to_string(), + ))); + let principals = principals_for_certs( + middleware, + Some("Bearer good"), + vec![spiffe_leaf_certificate("/carbide-system/machine/m1")], + ) + .await; + assert!( + !principals.contains(&Principal::SpiffeMachineIdentifier("m1".to_string())), + "cert-derived machine principal must be dropped, got {principals:?}" + ); + assert!( + principals.contains(&Principal::SpiffeMachineIdentifier("m2".to_string())), + "bearer-derived machine principal must survive, got {principals:?}" + ); + } + + #[tokio::test] + async fn service_cert_unaffected_when_machine_certs_disabled() { + // The gate is scoped to machine certs: service identities on the same + // listener keep authenticating via mTLS. + let middleware = CertDescriptionMiddleware::::new(None, spiffe_context()) + .with_machine_certs_enabled(false); + let principals = principals_for_certs( + middleware, + None, + vec![spiffe_leaf_certificate("/carbide-system/sa/test-service")], + ) + .await; + assert!( + principals + .iter() + .any(|p| matches!(p, Principal::SpiffeServiceIdentifier(_))), + "service cert principal must survive, got {principals:?}" + ); + } + + /// The point of `mtls_enabled = false` is that a machine certificate stops + /// authenticating the node. Dropping only the machine principal does not + /// achieve that: `TrustedCertificate` is granted `forge/*` and `nico/*` by + /// the shipped Casbin policy, so leaving it in place would keep the caller + /// fully authorized under a different name and the switch would be + /// cosmetic. Assert the whole principal set, not the absence of one member. + #[tokio::test] + async fn a_machine_cert_alone_authorizes_nothing_when_machine_certs_are_disabled() { + let middleware = CertDescriptionMiddleware::::new(None, spiffe_context()) + .with_machine_certs_enabled(false); + let principals = principals_for_certs( + middleware, + None, + vec![spiffe_leaf_certificate("/carbide-system/machine/m1")], + ) + .await; + assert!( + principals.is_empty(), + "a refused machine cert must leave no authorizing principal, got {principals:?}" + ); + } + + /// The realistic migration state: the agent presents its machine cert *and* + /// a bearer token. Refusing the cert does not affect the independently + /// validated bearer credential, which is the sole authorization path when + /// machine mTLS is disabled. + #[tokio::test] + async fn a_bearer_token_authorizes_when_the_machine_cert_is_refused() { + let middleware = CertDescriptionMiddleware::::new(None, spiffe_context()) + .with_bearer_authenticator(Arc::new(FakeAuth( + "spiffe://example.test/carbide-system/machine/m1".to_string(), + ))) + .with_machine_certs_enabled(false); + let principals = principals_for_certs( + middleware, + Some("Bearer good"), + vec![spiffe_leaf_certificate("/carbide-system/machine/m1")], + ) + .await; + + assert!( + principals.contains(&Principal::SpiffeMachineIdentifier("m1".to_string())), + "the bearer token is the credential now, and must still authenticate: {principals:?}" + ); + assert!( + principals.contains(&Principal::TrustedCertificate), + "a validated bearer token must authorize even when its mTLS cert is refused: {principals:?}" + ); + } + + /// The suppression above is scoped to machine certs: a service cert still + /// earns `TrustedCertificate` alongside its own principal. + #[tokio::test] + async fn a_service_cert_keeps_trusted_certificate_when_machine_certs_are_disabled() { + let middleware = CertDescriptionMiddleware::::new(None, spiffe_context()) + .with_machine_certs_enabled(false); + let principals = principals_for_certs( + middleware, + None, + vec![spiffe_leaf_certificate("/carbide-system/sa/test-service")], + ) + .await; + assert!( + principals.contains(&Principal::TrustedCertificate), + "service certs are unaffected by the machine gate, got {principals:?}" + ); + } + + /// With machine certs enabled the old behavior is unchanged — the gate must + /// not cost a normal mTLS node its trusted-certificate grant. + #[tokio::test] + async fn a_machine_cert_keeps_trusted_certificate_when_machine_certs_are_enabled() { + let middleware = CertDescriptionMiddleware::::new(None, spiffe_context()); + let principals = principals_for_certs( + middleware, + None, + vec![spiffe_leaf_certificate("/carbide-system/machine/m1")], + ) + .await; + assert!( + principals.contains(&Principal::TrustedCertificate), + "enabled machine certs keep their trusted-certificate grant, got {principals:?}" + ); + } + + #[tokio::test] + async fn bearer_ignored_when_no_authenticator_configured() { + // Without an authenticator installed, an Authorization header is ignored + // (nodes keep using mTLS) — no machine principal appears. + let middleware = CertDescriptionMiddleware::::new(None, spiffe_context()); + let principals = principals_for(middleware, Some("Bearer good")).await; + assert!( + !principals + .iter() + .any(|p| matches!(p, Principal::SpiffeMachineIdentifier(_))), + "no authenticator => no bearer principals, got {principals:?}" + ); + } + + #[test] + fn bearer_token_from_headers_parses_scheme() { + let mut headers = hyper::HeaderMap::new(); + assert_eq!(bearer_token_from_headers(&headers), None); + headers.insert(AUTHORIZATION, "Bearer abc.def".parse().unwrap()); + assert_eq!(bearer_token_from_headers(&headers), Some("abc.def")); + headers.insert(AUTHORIZATION, "Basic xyz".parse().unwrap()); + assert_eq!(bearer_token_from_headers(&headers), None); + } } diff --git a/crates/fmds/src/cfg.rs b/crates/fmds/src/cfg.rs index 2bdd5049dd..3ca6e7b93f 100644 --- a/crates/fmds/src/cfg.rs +++ b/crates/fmds/src/cfg.rs @@ -59,6 +59,13 @@ pub struct Options { #[clap(long)] pub client_key: Option, + /// Path of the dpu-agent's local API socket. When set, fmds fetches + /// short-lived node-auth bearer JWTs from the agent and presents them to + /// carbide-api — instead of (or in addition to) the mTLS client cert — + /// so this pod no longer needs the machine private key mounted. + #[clap(long, env = "FMDS_NODE_TOKEN_SOCKET")] + pub node_token_socket: Option, + /// Name of the interface to assign the metadata-service address to. #[clap(long, env = "FMDS_INTERFACE_NAME", default_value = "f_pf0hpf_if")] pub interface_name: String, diff --git a/crates/fmds/src/main.rs b/crates/fmds/src/main.rs index 6715944fe4..7b001f9b59 100644 --- a/crates/fmds/src/main.rs +++ b/crates/fmds/src/main.rs @@ -27,6 +27,7 @@ use fmds::{http_request_metrics, nic_init}; use forge_tls::client_config::ClientCert; use rpc::fmds::fmds_config_service_server::FmdsConfigServiceServer; use rpc::forge_tls_client::ForgeClientConfig; +use rpc::node_token_socket::SocketTokenSource; use tracing::metadata::LevelFilter; use tracing_subscriber::EnvFilter; use tracing_subscriber::layer::SubscriberExt as _; @@ -80,16 +81,47 @@ async fn main() -> eyre::Result<()> { nic_init::assign_address(&options.interface_name, options.interface_cidr).await?; nic_init::setup_metadata_routing(&options.interface_name, options.interface_cidr).await?; - // Build ForgeClientConfig for phone_home if cert paths are provided - let forge_client_config = match (&options.root_ca, &options.client_cert, &options.client_key) { - (Some(root_ca), Some(client_cert), Some(client_key)) => { - Some(Arc::new(ForgeClientConfig::new( - root_ca.clone(), - Some(ClientCert { - cert_path: client_cert.clone(), - key_path: client_key.clone(), - }), - ))) + // Build ForgeClientConfig for phone_home. Either credential works alone: + // the shared mTLS client cert, and/or node-auth bearer tokens fetched from + // the dpu-agent's local API socket (#355) — the latter lets this pod run + // without the machine private key mounted at all. + let client_cert = match (&options.client_cert, &options.client_key) { + (Some(cert_path), Some(key_path)) => Some(ClientCert { + cert_path: cert_path.clone(), + key_path: key_path.clone(), + }), + (None, None) => None, + // Half a credential is a deployment bug; fail loudly instead of + // silently running without mTLS. + (Some(_), None) => { + eyre::bail!("--client-cert was provided without --client-key") + } + (None, Some(_)) => { + eyre::bail!("--client-key was provided without --client-cert") + } + }; + // Same class of deployment bug as half a client credential above, and it + // deserves the same treatment: a token socket without a trust anchor + // cannot authenticate the server, so the client would never be built and + // phone_home would go quietly missing behind a generic warning. + if options.root_ca.is_none() && options.node_token_socket.is_some() { + eyre::bail!("--node-token-socket was provided without --root-ca"); + } + let forge_client_config = match &options.root_ca { + Some(root_ca) if client_cert.is_some() || options.node_token_socket.is_some() => { + let mut config = ForgeClientConfig::new(root_ca.clone(), client_cert); + if let Some(socket) = &options.node_token_socket { + tracing::info!( + socket = %socket, + "fetching node-auth bearer tokens from the dpu-agent local API" + ); + // Token mode usually runs without a client cert, which would + // otherwise leave the channel on the dummy TLS verifier. The + // bearer token is the client credential; the server still has + // to prove itself against the root CA. + config = config.with_token_provider(SocketTokenSource::spawn(socket.clone())); + } + Some(Arc::new(config)) } _ => { tracing::warn!( diff --git a/crates/host-support/src/agent_config.rs b/crates/host-support/src/agent_config.rs index 411575bb37..01f3ab53c4 100644 --- a/crates/host-support/src/agent_config.rs +++ b/crates/host-support/src/agent_config.rs @@ -119,6 +119,12 @@ pub struct ForgeSystemConfig { pub client_cert: String, #[serde(default = "default_client_key")] pub client_key: String, + /// Unix socket where the agent serves its local API (node tokens for + /// co-located services, issue #355). Works the same containerized (DPF) + /// and as a plain service on DPU OS; override when `/opt/forge` is not + /// the shared credential directory in a deployment. + #[serde(default = "default_local_api_socket")] + pub local_api_socket: String, } // Called if no `[forge-system]` is provided at all. @@ -130,6 +136,7 @@ impl Default for ForgeSystemConfig { root_ca: default_root_ca(), client_cert: default_client_cert(), client_key: default_client_key(), + local_api_socket: default_local_api_socket(), } } } @@ -150,6 +157,10 @@ pub fn default_client_key() -> String { tls_default::default_client_key().to_string() } +pub fn default_local_api_socket() -> String { + ::rpc::node_token_socket::DEFAULT_AGENT_LOCAL_SOCKET.to_string() +} + #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub struct MachineConfig { diff --git a/crates/host-support/src/registration.rs b/crates/host-support/src/registration.rs index 4dfbff7de8..e1a9267ebc 100644 --- a/crates/host-support/src/registration.rs +++ b/crates/host-support/src/registration.rs @@ -312,7 +312,7 @@ pub async fn write_certs( ))?; tracing::info!(%client_cert, "Wrote new machine certificate PEM"); - tokio::fs::write(client_key, machine_certificate.private_key.as_slice()) + write_private_key(client_key, machine_certificate.private_key.as_slice()) .await .wrap_err(format!( "failed to write new machine certificate key to: {client_key}" @@ -323,3 +323,114 @@ pub async fn write_certs( Ok(()) } + +/// Writes the machine private key owner-readable only (0600). The key is +/// created with the restrictive mode from the first byte — not chmod'ed after +/// the fact — so there is no window where it sits world-readable; a pre-existing +/// key file from an older agent (written 0644 via plain `fs::write`) is +/// tightened as well. Everything that legitimately reads the key (dpu-agent, +/// scout, fmds, otelcol) runs as root, so 0600 root-owned loses nobody access. +#[cfg(unix)] +async fn write_private_key(path: &str, key: &[u8]) -> Result<(), std::io::Error> { + use std::os::unix::fs::PermissionsExt; + + use tokio::io::AsyncWriteExt; + + let mut file = tokio::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .mode(0o600) + .open(path) + .await?; + // `mode` only applies at creation; tighten files that already existed. + file.set_permissions(std::fs::Permissions::from_mode(0o600)) + .await?; + file.write_all(key).await?; + // `sync_all`, not `flush`: flushing only pushes the buffer into the kernel. + // The agent's very next act is to use this key, and a DPU losing power + // between the write and the writeback would leave a truncated or empty key + // behind a certificate that looks installed — a state the mismatch check in + // the minter reports as a failure but cannot repair without re-enrolment. + file.sync_all().await +} + +#[cfg(not(unix))] +async fn write_private_key(path: &str, key: &[u8]) -> Result<(), std::io::Error> { + tokio::fs::write(path, key).await +} + +#[cfg(all(test, unix))] +mod tests { + use std::os::unix::fs::PermissionsExt; + + use super::*; + + /// `write_certs` concatenates and writes bytes without parsing them, so + /// these are opaque sentinels rather than PEM. Real-looking `-----BEGIN EC + /// PRIVATE KEY-----` markers would buy nothing here and trip secret + /// scanners, which match the marker and not its contents. + fn test_certificate() -> MachineCertificate { + MachineCertificate { + public_key: b"test-leaf-certificate-bytes".to_vec(), + issuing_ca: b"test-issuing-ca-bytes".to_vec(), + private_key: b"test-machine-private-key-bytes".to_vec(), + } + } + + fn key_mode(path: &str) -> u32 { + std::fs::metadata(path) + .expect("key metadata") + .permissions() + .mode() + & 0o777 + } + + #[tokio::test] + async fn private_key_is_written_owner_only() { + let dir = tempfile::tempdir().expect("tempdir"); + let paths = ClientCert { + cert_path: dir.path().join("cert.pem").to_string_lossy().into_owned(), + key_path: dir.path().join("cert.key").to_string_lossy().into_owned(), + }; + + write_certs(Some(test_certificate()), Some(&paths)) + .await + .expect("write_certs succeeds"); + + assert_eq!(key_mode(&paths.key_path), 0o600, "fresh key must be 0600"); + let cert = std::fs::read_to_string(&paths.cert_path).expect("cert readable"); + assert!( + cert.contains("leaf") && cert.contains("ca"), + "cert file carries leaf + chain" + ); + } + + #[tokio::test] + async fn pre_existing_loose_key_is_tightened_on_rewrite() { + let dir = tempfile::tempdir().expect("tempdir"); + let paths = ClientCert { + cert_path: dir.path().join("cert.pem").to_string_lossy().into_owned(), + key_path: dir.path().join("cert.key").to_string_lossy().into_owned(), + }; + // A key written by an older agent via plain fs::write (umask default). + std::fs::write(&paths.key_path, b"old key").expect("seed old key"); + std::fs::set_permissions(&paths.key_path, std::fs::Permissions::from_mode(0o644)) + .expect("loosen"); + + write_certs(Some(test_certificate()), Some(&paths)) + .await + .expect("write_certs succeeds"); + + assert_eq!( + key_mode(&paths.key_path), + 0o600, + "renewal must tighten an old 0644 key" + ); + let key = std::fs::read_to_string(&paths.key_path).expect("key readable by owner"); + assert!( + key.contains("test-machine-private-key-bytes"), + "key content replaced" + ); + } +} diff --git a/crates/host-support/test/min_agent_config/output.toml b/crates/host-support/test/min_agent_config/output.toml index 3c5db0e597..3a1ea631a9 100644 --- a/crates/host-support/test/min_agent_config/output.toml +++ b/crates/host-support/test/min_agent_config/output.toml @@ -3,6 +3,7 @@ api-server = "https://127.0.0.1:8001" root-ca = "/opt/forge/forge_root.pem" client-cert = "/opt/forge/machine_cert.pem" client-key = "/opt/forge/machine_cert.key" +local-api-socket = "/opt/forge/run/agent.sock" [machine] interface-id = "91609f10-c91d-470d-a260-6293ea0c1234" diff --git a/crates/rpc/Cargo.toml b/crates/rpc/Cargo.toml index e5a5a0fc8e..85f4bd3cce 100644 --- a/crates/rpc/Cargo.toml +++ b/crates/rpc/Cargo.toml @@ -62,6 +62,8 @@ carbide-uuid = { path = "../uuid" } chrono = { features = ["serde"], workspace = true } const_format = { workspace = true } eyre = { workspace = true } +jsonwebtoken = { workspace = true, features = ["rust_crypto"] } +p256 = { workspace = true } hyper = { workspace = true, features = ["full"] } hyper-rustls = { workspace = true, features = ["http2"] } hyper-util = { features = [ @@ -127,6 +129,8 @@ carbide-api-model = { path = "../api-model", features = ["test-support"] } carbide-libmlx-model = { path = "../libmlx-model", features = ["test-support"] } carbide-test-support = { path = "../test-support" } criterion = { workspace = true } +rcgen = { workspace = true } +tempfile = { workspace = true } [[bench]] name = "machine_convert" diff --git a/crates/rpc/build.rs b/crates/rpc/build.rs index 19c17e326e..039ffd3dc3 100644 --- a/crates/rpc/build.rs +++ b/crates/rpc/build.rs @@ -45,6 +45,18 @@ fn main() -> Result<(), Box> { // so that different builds get a different binary file and concurrent builds don't collide let reflection = out_dir.join("forge.bin"); + // AgentLocal is the dpu-agent's local unix-socket service (issue #355). + // The agent owns the file and serves it; this crate provides the client + // for `SocketTokenSource` and the server bindings the agent uses. Keeping + // both sides here ensures there is one generated Rust representation. + // + // Compiled outside the schema below on purpose: that schema feeds the API's + // reflection descriptor, and nico-api does not serve AgentLocal. + tonic_prost_build::configure() + .build_server(true) + .build_client(true) + .compile_protos(&["../agent/proto/agent_local.proto"], &["../agent/proto"])?; + // Run protoc once as the schema frontend. The resulting Schema fans out to // every code-generation backend and to runtime reflection. let schema = compile(&CompilerConfig { diff --git a/crates/rpc/src/forge_tls_client.rs b/crates/rpc/src/forge_tls_client.rs index fd06f71245..2176236669 100644 --- a/crates/rpc/src/forge_tls_client.rs +++ b/crates/rpc/src/forge_tls_client.rs @@ -41,6 +41,7 @@ use x509_parser::prelude::{FromDer, X509Certificate}; use crate::forge::VersionRequest; use crate::forge_resolver::resolver::ResolverError; use crate::forge_tls_client::ConfigurationError::CouldNotReadRootCa; +use crate::node_jwt::{BearerAuthService, NodeJwtMinter, NodeTokenProvider}; use crate::protos::forge::forge_client::ForgeClient; use crate::protos::nmx_c::nmx_controller_client::NmxControllerClient; use crate::{forge_resolver, protos}; @@ -96,6 +97,12 @@ pub struct ForgeClientConfig { pub socks_proxy: Option, pub connect_retries_max: Option, pub connect_retries_interval: Option, + /// Optional node-auth token provider (issue #355). When set, each request + /// carries an `Authorization: Bearer ` — either self-signed with the + /// client certificate's own private key ([`NodeJwtMinter`]) or fetched + /// from the dpu-agent's local API (`SocketTokenSource`). Independent of + /// mTLS: the channel may present a client cert, a token, or both. + pub node_token_provider: Option>, } impl ForgeClientConfig { @@ -134,9 +141,59 @@ impl ForgeClientConfig { // MR though, I think. connect_retries_max: Some(3), connect_retries_interval: Some(Duration::from_secs(20)), + node_token_provider: None, } } + /// Restores server-certificate validation on a config built without a + /// client certificate. + /// + /// [`ForgeClientConfig::new`] disables TLS enforcement whenever + /// `client_cert` is `None`, which predates node-auth: back then "no client + /// cert" meant "no credentials at all", and such callers were not expected + /// to reach the API over verified TLS. A node-token client does hold + /// credentials — a bearer JWT brokered by the dpu-agent (issue #355) — and + /// must still authenticate the server against `root_ca_path` rather than + /// fall back to [`DummyTlsVerifier`]. Callers that never opt in keep the + /// previous behavior. + /// + /// `DISABLE_TLS_ENFORCEMENT` continues to win, so local-development + /// overrides work the same as they do for mTLS clients. + #[must_use] + pub fn require_tls_enforcement(mut self) -> Self { + self.enforce_tls = std::env::var("DISABLE_TLS_ENFORCEMENT").is_err(); + self + } + + /// Enables node-auth JWTs: requests built from this config mint and carry + /// short-lived bearer tokens signed with the configured client cert's key. + /// A no-op when no client cert is configured. + #[must_use] + pub fn with_node_jwt(mut self) -> Self { + self.node_token_provider = self.client_cert.as_ref().map(|client_cert| { + NodeJwtMinter::new(client_cert.cert_path.clone(), client_cert.key_path.clone()) + as Arc + }); + self + } + + /// Attaches an explicit node-auth token provider — e.g. a pre-built + /// [`NodeJwtMinter`] the caller also serves through the agent's local + /// API, or a `SocketTokenSource` in a process that holds no key at all. + /// + /// Implies [`require_tls_enforcement`](Self::require_tls_enforcement). + /// A bearer token IS the client's credential, so the server must always + /// be authenticated before one is handed over — a token-only config + /// (no client cert) would otherwise sit on [`DummyTlsVerifier`] and + /// present its token to whatever answered. Enforcing it here rather than + /// asking every caller to remember the pairing keeps the type hard to + /// misuse; `DISABLE_TLS_ENFORCEMENT` still wins for local development. + #[must_use] + pub fn with_token_provider(mut self, provider: Arc) -> Self { + self.node_token_provider = Some(provider); + self.require_tls_enforcement() + } + /// This is required when using `ForgeTlsConfig` on a DPU to communicate with site-controller. /// The mgmt interface exists in the mgmt VRF. `use_mgmt_vrf` sets the /// `SO_BINDTODEVICE` socket option on the client socket used when performing DNS queries @@ -431,6 +488,32 @@ impl<'a> ForgeTlsClient<'a> { error: e, })?; + // A bearer token is the client's credential, so it may only be sent to + // a server this client has authenticated. Both ways that can fail are + // checked here rather than in the builders: `enforce_tls` and + // `node_token_provider` are public fields, so a caller can clear + // enforcement after `with_token_provider` or build the struct + // literally, and the handshake only happens at all for an https:// URL + // — a plaintext one skips TLS setup entirely while `BearerAuthService` + // below still stamps the token. `DISABLE_TLS_ENFORCEMENT` is honored + // for parity with the rest of this config so local development works. + if self.forge_client_config.node_token_provider.is_some() + && std::env::var("DISABLE_TLS_ENFORCEMENT").is_err() + { + if uri.scheme() != Some(&tonic::codegen::http::uri::Scheme::HTTPS) { + return Err(ConfigurationError::BearerTokenOverPlaintext { + uri_string: url.as_ref().to_string(), + } + .into()); + } + if !self.forge_client_config.enforce_tls { + return Err(ConfigurationError::BearerTokenWithoutTlsEnforcement { + uri_string: url.as_ref().to_string(), + } + .into()); + } + } + let connector = self.build_https_client(url.as_ref()).await?; // ping interval + ping timeout should add up to less than tcp_user_timeout, @@ -448,6 +531,13 @@ impl<'a> ForgeTlsClient<'a> { .pool_max_idle_per_host(2) .timer(TokioTimer::new()) .build(connector); + // Stamp a freshly-minted node-auth bearer token onto each request when + // configured (issue #355). A `None` minter is a transparent pass-through, + // so the boxed service type is identical in both modes. + let hyper_client = BearerAuthService::new( + hyper_client, + self.forge_client_config.node_token_provider.clone(), + ); // Inject the issuing span's W3C trace context into every request this client sends // (issue #2438). Wrapping before `boxed_clone` keeps the erased `BoxCloneService` type. let hyper_client = trace_propagation::TraceInjectService::new(hyper_client).boxed_clone(); @@ -744,6 +834,17 @@ pub enum ConfigurationError { }, #[error("could not read root CA cert at {path}: {error}")] CouldNotReadRootCa { path: String, error: io::Error }, + #[error( + "refusing to send node-auth bearer tokens to non-HTTPS URL {uri_string}: \ + the token would travel in cleartext" + )] + BearerTokenOverPlaintext { uri_string: String }, + #[error( + "refusing to send node-auth bearer tokens to {uri_string} with TLS enforcement \ + disabled: the server's certificate would not be verified, so the token could \ + be handed to an impersonator" + )] + BearerTokenWithoutTlsEnforcement { uri_string: String }, #[error("invalid client cert: {0}")] InvalidClientCert(rustls::Error), #[error("error configuring resolver: {0}")] @@ -769,6 +870,131 @@ mod tests { use super::*; + /// A node-token client presents no client certificate, which by itself + /// drops the channel onto `DummyTlsVerifier`. `require_tls_enforcement` + /// puts it back on root-CA validation. + #[test] + fn require_tls_enforcement_restores_validation_without_a_client_cert() { + if std::env::var("DISABLE_TLS_ENFORCEMENT").is_ok() { + // The override wins by design; the assertions below would not hold. + return; + } + + let config = ForgeClientConfig::new("/etc/carbide/root-ca.pem".to_string(), None); + assert!( + !config.enforce_tls, + "a config without a client cert starts out unenforced" + ); + + let config = config.require_tls_enforcement(); + assert!( + config.enforce_tls, + "opting in must restore server certificate validation" + ); + } + + #[derive(Debug)] + struct FixedToken; + + impl NodeTokenProvider for FixedToken { + fn current(&self) -> Option { + Some("a.b.c".to_string()) + } + } + + /// The bearer token IS the credential, so the server must be authenticated + /// before one is sent. Callers must not have to remember to pair + /// `with_token_provider` with `require_tls_enforcement` — forgetting it + /// would hand tokens to whatever answered the connection. + #[test] + fn with_token_provider_enforces_tls_without_being_asked() { + if std::env::var("DISABLE_TLS_ENFORCEMENT").is_ok() { + // The override wins by design; the assertion below would not hold. + return; + } + + let config = ForgeClientConfig::new("/etc/carbide/root-ca.pem".to_string(), None) + .with_token_provider(Arc::new(FixedToken)); + + assert!( + config.enforce_tls, + "attaching a token provider must restore server validation by itself" + ); + } + + /// TLS setup only runs for an https:// URL, so `enforce_tls` alone does not + /// cover a plaintext one — the token would still be stamped and travel in + /// the clear. Building such a client has to fail. + #[tokio::test] + async fn bearer_tokens_are_refused_over_plaintext() { + if std::env::var("DISABLE_TLS_ENFORCEMENT").is_ok() { + // The override deliberately permits plaintext development. + return; + } + + let config = ForgeClientConfig::new("/etc/carbide/root-ca.pem".to_string(), None) + .with_token_provider(Arc::new(FixedToken)); + + let error = ForgeTlsClient::new(&config) + .build("http://carbide-api.local:8080") + .await + .expect_err("a token client must refuse a plaintext endpoint"); + + assert!( + error.to_string().contains("cleartext"), + "the error should explain the refusal, got: {error}" + ); + } + + /// `enforce_tls` and `node_token_provider` are both public, so the builder + /// pairing is not the last word: a caller can clear enforcement afterwards + /// or build the struct literally. Over https:// that still selects + /// `DummyTlsVerifier`, so the check has to live at client construction. + #[tokio::test] + async fn bearer_tokens_are_refused_when_tls_enforcement_is_cleared() { + if std::env::var("DISABLE_TLS_ENFORCEMENT").is_ok() { + // The override deliberately permits unverified development setups. + return; + } + + let mut config = ForgeClientConfig::new("/etc/carbide/root-ca.pem".to_string(), None) + .with_token_provider(Arc::new(FixedToken)); + // Exactly what a caller ordering the builders the other way round, or + // assigning the public field, would end up with. + config.enforce_tls = false; + + let error = ForgeTlsClient::new(&config) + .build("https://carbide-api.local:8080") + .await + .expect_err("an unverified token client must be refused"); + + assert!( + error.to_string().contains("TLS enforcement disabled"), + "the error should explain the refusal, got: {error}" + ); + } + + /// The same client over https:// must still build — the guard above is + /// scoped to the scheme, not to token clients in general. + #[tokio::test] + async fn bearer_tokens_are_allowed_over_https() { + let config = ForgeClientConfig::new("/etc/carbide/root-ca.pem".to_string(), None) + .with_token_provider(Arc::new(FixedToken)); + + // Reaching a root-CA read means the plaintext guard let it through; + // the file itself need not exist in a unit test. + let result = ForgeTlsClient::new(&config) + .build("https://carbide-api.local:8080") + .await; + + if let Err(error) = result { + assert!( + !error.to_string().contains("cleartext"), + "https must not trip the plaintext guard, got: {error}" + ); + } + } + #[tokio::test] // test_max_retries builds up an instance of hyper client using // the ForgeHttpConnector, which is the same configuration used diff --git a/crates/rpc/src/lib.rs b/crates/rpc/src/lib.rs index c35850a501..f5c0f60f7d 100644 --- a/crates/rpc/src/lib.rs +++ b/crates/rpc/src/lib.rs @@ -74,13 +74,15 @@ pub use crate::protos::machine_discovery::{ self, BlockDevice, Cpu, DiscoveryInfo, DmiData, NetworkInterface, NvmeDevice, PciDeviceProperties, }; -pub use crate::protos::{fmds, health, scout_firmware_upgrade, site_explorer}; +pub use crate::protos::{agent_local, fmds, health, scout_firmware_upgrade, site_explorer}; pub mod errors; pub mod forge_tls_client; pub mod libmlx; pub mod measured_boot; pub mod network; +pub mod node_jwt; +pub mod node_token_socket; pub mod protos; pub mod secrets; mod site_explorer_report; diff --git a/crates/rpc/src/node_jwt.rs b/crates/rpc/src/node_jwt.rs new file mode 100644 index 0000000000..532cd0a8e7 --- /dev/null +++ b/crates/rpc/src/node_jwt.rs @@ -0,0 +1,507 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Client-side node-auth JWT minting (issue NVIDIA/infra-controller#355). +//! +//! The node self-signs short-lived ES256 JWTs with the private key of its +//! EXISTING mTLS client certificate; the certificate chain rides along in the +//! token's `x5c` header (RFC 7515 §4.1.6) so the API can verify it against the +//! same root CA its TLS listener already trusts. No new key material, no key +//! storage, and no refresh RPC: a fresh token is minted locally whenever the +//! cached one nears expiry, and key "rotation" happens for free when the +//! client certificate renews. +//! +//! [`BearerAuthService`] is the tower middleware that stamps the current token +//! onto each outgoing request's `Authorization` header. Minting is best-effort: +//! if the cert/key files are missing or unreadable (e.g. before first +//! registration), requests simply carry no bearer header and the channel's +//! mTLS client cert remains the only credential. + +use std::io::Cursor; +use std::sync::{Arc, RwLock}; +use std::task::{Context, Poll}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use data_encoding::BASE64; +use jsonwebtoken::{Algorithm, EncodingKey, Header}; +use p256::pkcs8::{EncodePrivateKey, LineEnding}; +use serde::Serialize; +use tower::Service; +use x509_parser::prelude::{FromDer, GeneralName, X509Certificate}; + +/// `aud` claim stamped on all node-auth tokens. +pub const NODE_JWT_AUDIENCE: &str = "nico-api"; + +/// Lifetime of minted tokens. Deliberately short: tokens cost nothing to +/// re-mint locally, so a leaked one ages out in minutes. +pub const NODE_JWT_TTL_SECS: u64 = 300; + +/// A cached token is reused until it has less than this long left, then +/// re-minted. Comfortably above per-request latency, comfortably below TTL. +const REMINT_MARGIN_SECS: u64 = 60; + +#[derive(Debug, thiserror::Error)] +pub enum NodeJwtError { + #[error("could not read client certificate or key: {0}")] + Io(#[from] std::io::Error), + #[error("client certificate file contains no parsable certificate")] + NoCertificate, + #[error("client certificate is not usable for node JWTs: {0}")] + BadCertificate(String), + #[error("client private key is not a usable EC key: {0}")] + BadKey(String), + #[error("client private key does not match the certificate's public key")] + KeyCertMismatch, + #[error("JWT signing failed: {0}")] + Sign(#[from] jsonwebtoken::errors::Error), + #[error("system clock is before the UNIX epoch")] + Clock, +} + +#[derive(Clone)] +struct CachedToken { + token: String, + expires_at: u64, +} + +/// Mints and caches node-auth JWTs from the node's existing mTLS client +/// certificate and private key files. +/// +/// Cert and key are re-read from disk on every mint, so certificate renewal +/// (which rewrites both files in place) is picked up automatically on the +/// next re-mint without any coordination. +pub struct NodeJwtMinter { + cert_path: String, + key_path: String, + cached: RwLock>, +} + +/// Manual impl so the cached token (a live credential) never lands in debug +/// output of the client config. +impl std::fmt::Debug for NodeJwtMinter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("NodeJwtMinter") + .field("cert_path", &self.cert_path) + .field("key_path", &self.key_path) + .finish_non_exhaustive() + } +} + +/// Claims carried by a node-auth JWT. `sub` duplicates the certificate's +/// SPIFFE URI SAN; the server derives identity from the *verified certificate* +/// and only cross-checks `sub` against it, so a forged `sub` buys nothing. +#[derive(Debug, Serialize)] +struct NodeClaims<'a> { + sub: &'a str, + aud: &'a str, + iat: u64, + exp: u64, +} + +impl NodeJwtMinter { + /// Mints tokens for [`NODE_JWT_AUDIENCE`]. + #[must_use] + pub fn new(cert_path: String, key_path: String) -> Arc { + Arc::new(Self { + cert_path, + key_path, + cached: RwLock::new(None), + }) + } + + /// Returns a currently-valid token, re-minting if the cached one is + /// missing or close to expiry. Returns `None` (and logs at debug) when + /// minting is impossible — e.g. the cert/key files don't exist yet — so + /// callers degrade gracefully to mTLS-only. + pub fn current(&self) -> Option { + self.current_with_expiry().map(|(token, _)| token) + } + + /// Like [`current`](Self::current), but also returns the token's expiry + /// (unix seconds) — used by the agent's local API to tell co-located + /// consumers when to re-fetch. + pub fn current_with_expiry(&self) -> Option<(String, u64)> { + let now = unix_now().ok()?; + // Poisoning recovers rather than disabling the cache. Here it would + // only cost a re-mint per call, since minting doesn't depend on the + // cache — but silently doing that forever is worse than continuing to + // use a `CachedToken` no panicking reader could have torn. + if let Some(cached) = self + .cached + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .as_ref() + && cached.expires_at > now + REMINT_MARGIN_SECS + { + return Some((cached.token.clone(), cached.expires_at)); + } + match self.mint(now) { + Ok(minted) => { + let result = (minted.token.clone(), minted.expires_at); + *self + .cached + .write() + .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(minted); + Some(result) + } + Err(error) => { + tracing::debug!( + target: "node_auth", + cert_path = %self.cert_path, + %error, + "node-auth: could not mint node JWT; continuing with mTLS only" + ); + None + } + } + } + + fn mint(&self, now: u64) -> Result { + let cert_pem = std::fs::read(&self.cert_path)?; + let key_pem = std::fs::read_to_string(&self.key_path)?; + + let chain = rustls_pemfile::certs(&mut Cursor::new(&cert_pem[..])) + .collect::, _>>() + .map_err(|e| NodeJwtError::BadCertificate(e.to_string()))?; + let leaf = chain.first().ok_or(NodeJwtError::NoCertificate)?; + let sub = spiffe_uri_from_cert(leaf.as_ref())?; + let secret = parse_secret_key(&key_pem)?; + + // Certificate renewal rewrites the two files in sequence, so a mint + // landing in between can pair a new certificate with the old key. + // Signing would still succeed and the API would reject the result, + // and the bad token would sit in the cache for a re-mint margin. Catch + // the mismatch here instead: returning an error means no bearer header + // on this request and a fresh attempt on the next one. + if !key_matches_certificate(&secret, leaf.as_ref())? { + return Err(NodeJwtError::KeyCertMismatch); + } + + let mut header = Header::new(Algorithm::ES256); + header.typ = Some("JWT".to_string()); + header.x5c = Some(chain.iter().map(|c| BASE64.encode(c.as_ref())).collect()); + + let expires_at = now + NODE_JWT_TTL_SECS; + let claims = NodeClaims { + sub: &sub, + aud: NODE_JWT_AUDIENCE, + iat: now, + exp: expires_at, + }; + let token = jsonwebtoken::encode(&header, &claims, &ec_encoding_key(&secret)?)?; + Ok(CachedToken { token, expires_at }) + } +} + +fn unix_now() -> Result { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .map_err(|_| NodeJwtError::Clock) +} + +/// Extracts the single SPIFFE URI SAN from the certificate — the same field +/// the server's authn layer maps to a machine principal for mTLS. +fn spiffe_uri_from_cert(der: &[u8]) -> Result { + let (_, cert) = X509Certificate::from_der(der) + .map_err(|e| NodeJwtError::BadCertificate(format!("X.509 parse error: {e}")))?; + let san = cert + .subject_alternative_name() + .map_err(|e| NodeJwtError::BadCertificate(format!("bad SAN extension: {e}")))? + .ok_or_else(|| NodeJwtError::BadCertificate("no SAN extension".to_string()))?; + // Exactly one, not merely the first. The doc comment says "single" and the + // server enforces that (`carbide_authn::validate_x509_certificate` applies + // the single-URI-SAN rule to the verified leaf), so picking whichever came + // first would let a client stamp `sub` from one SAN while the server + // derived identity from another — a disagreement the server would reject + // as a subject mismatch, reported here as a signing failure rather than a + // malformed certificate. Refuse to guess. + let mut spiffe_uris = san + .value + .general_names + .iter() + .filter_map(|name| match name { + GeneralName::URI(uri) if uri.starts_with("spiffe://") => Some(uri.to_string()), + _ => None, + }); + let uri = spiffe_uris + .next() + .ok_or_else(|| NodeJwtError::BadCertificate("no SPIFFE URI SAN".to_string()))?; + if spiffe_uris.next().is_some() { + return Err(NodeJwtError::BadCertificate( + "certificate carries more than one SPIFFE URI SAN".to_string(), + )); + } + Ok(uri) +} + +/// Parses a Vault-issued SEC1 or PKCS#8 client key. +fn parse_secret_key(key_pem: &str) -> Result { + if key_pem.contains("BEGIN EC PRIVATE KEY") { + p256::SecretKey::from_sec1_pem(key_pem).map_err(|e| NodeJwtError::BadKey(e.to_string())) + } else { + ::from_pkcs8_pem(key_pem) + .map_err(|e| NodeJwtError::BadKey(e.to_string())) + } +} + +/// Whether `secret`'s public half is the one certified by `leaf_der`. +/// +/// Both encodings the node may hold are accepted: Vault issues SEC1 ("EC +/// PRIVATE KEY") and renewal may leave PKCS#8. The comparison is on the +/// uncompressed SEC1 point, which is exactly what an EC `SubjectPublicKeyInfo` +/// carries. +fn key_matches_certificate( + secret: &p256::SecretKey, + leaf_der: &[u8], +) -> Result { + let (_, cert) = X509Certificate::from_der(leaf_der) + .map_err(|e| NodeJwtError::BadCertificate(e.to_string()))?; + let certified = &cert.public_key().subject_public_key.data; + let from_key = { + use p256::elliptic_curve::sec1::ToSec1Point as _; + secret.public_key().to_sec1_point(false) + }; + + Ok(certified.as_ref() == from_key.as_bytes()) +} + +/// Builds an ES256 signing key from an already-validated client key. +fn ec_encoding_key(secret: &p256::SecretKey) -> Result { + let pkcs8 = secret + .to_pkcs8_pem(LineEnding::LF) + .map_err(|e| NodeJwtError::BadKey(e.to_string()))?; + EncodingKey::from_ec_pem(pkcs8.as_bytes()).map_err(|e| NodeJwtError::BadKey(e.to_string())) +} + +/// A source of node-auth bearer tokens for outgoing requests. Implemented by +/// [`NodeJwtMinter`] (holds the key, signs locally) and +/// [`SocketTokenSource`](crate::node_token_socket::SocketTokenSource) +/// (fetches from the dpu-agent's local API — the caller never sees the key). +/// +/// `current` runs on the request path, so an implementation must return +/// promptly and must never wait on the network or on a remote peer — return +/// `None` instead. Cheap local work is permitted and [`NodeJwtMinter`] does +/// it: on a cold or near-expiry cache it reads the cert and key from disk and +/// signs, roughly once per token lifetime. `SocketTokenSource` is the shape to +/// copy for anything costlier — its fetch runs in a background task and the +/// request path only ever reads the cache. +pub trait NodeTokenProvider: Send + Sync + std::fmt::Debug { + /// Returns a currently-valid token, or `None` if one isn't available + /// (the request then proceeds with whatever else the channel carries). + fn current(&self) -> Option; +} + +impl NodeTokenProvider for NodeJwtMinter { + fn current(&self) -> Option { + NodeJwtMinter::current(self) + } +} + +/// Tower middleware that injects `Authorization: Bearer ` onto each +/// request when a [`NodeTokenProvider`] is configured. A `None` provider is a +/// no-op, so the same client construction path serves both token and +/// mTLS-only modes. +#[derive(Clone)] +pub struct BearerAuthService { + inner: S, + minter: Option>, +} + +impl BearerAuthService { + pub fn new(inner: S, minter: Option>) -> Self { + Self { inner, minter } + } +} + +impl Service> for BearerAuthService +where + S: Service>, +{ + type Response = S::Response; + type Error = S::Error; + type Future = S::Future; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx) + } + + fn call(&mut self, mut req: hyper::Request) -> Self::Future { + if let Some(value) = self + .minter + .as_ref() + .and_then(|minter| minter.current()) + .and_then(|token| hyper::http::HeaderValue::from_str(&format!("Bearer {token}")).ok()) + { + req.headers_mut() + .insert(hyper::header::AUTHORIZATION, value); + } + self.inner.call(req) + } +} + +#[cfg(test)] +mod tests { + use jsonwebtoken::{DecodingKey, Validation}; + use serde::Deserialize; + + use super::*; + + #[derive(Debug, Deserialize)] + struct Claims { + sub: String, + exp: u64, + iat: u64, + } + + const SPIFFE_URI: &str = "spiffe://forge.local/forge-system/machine/fm100xtest"; + + /// Self-signed leaf with a SPIFFE URI SAN plus its PKCS#8 key PEM — + /// stand-ins for the Vault-issued client cert/key pair on a node. + fn cert_and_key() -> (String, String) { + let mut params = rcgen::CertificateParams::default(); + params.subject_alt_names = vec![rcgen::SanType::URI( + rcgen::string::Ia5String::try_from(SPIFFE_URI.to_string()).expect("uri"), + )]; + let key = rcgen::KeyPair::generate().expect("key pair"); + let cert = params.self_signed(&key).expect("certificate"); + (cert.pem(), key.serialize_pem()) + } + + fn write_temp(dir: &tempfile::TempDir, name: &str, contents: &str) -> String { + let path = dir.path().join(name); + std::fs::write(&path, contents).expect("write"); + path.to_string_lossy().into_owned() + } + + fn decode_against_own_cert(token: &str) -> Claims { + // Validate exactly as the server does: pull the leaf from x5c, take + // its SPKI EC point, verify the signature with it. + let header = jsonwebtoken::decode_header(token).expect("header"); + assert_eq!(header.alg, Algorithm::ES256); + let chain = header.x5c_der().expect("x5c decodes").expect("x5c present"); + let (_, cert) = X509Certificate::from_der(&chain[0]).expect("leaf parses"); + let decoding_key = DecodingKey::from_ec_der(&cert.public_key().subject_public_key.data); + let mut validation = Validation::new(Algorithm::ES256); + validation.set_audience(&[NODE_JWT_AUDIENCE]); + jsonwebtoken::decode::(token, &decoding_key, &validation) + .expect("token validates") + .claims + } + + /// Certificate renewal rewrites cert and key in sequence, so a mint can + /// land on a new certificate paired with the previous key. That signs + /// cleanly but the API cannot verify it, so it must never be minted or + /// cached — the next attempt, on a consistent pair, should succeed. + #[test] + fn a_certificate_from_a_different_key_is_never_minted_or_cached() { + let dir = tempfile::tempdir().expect("tempdir"); + let (cert_pem, _) = cert_and_key(); + let (_, other_key_pem) = cert_and_key(); + + let cert_path = write_temp(&dir, "cert.pem", &cert_pem); + let key_path = write_temp(&dir, "key.pem", &other_key_pem); + let minter = NodeJwtMinter::new(cert_path.clone(), key_path.clone()); + + assert!( + minter.current().is_none(), + "a token the API could not verify must not be minted" + ); + + // Renewal completes: the matching pair lands, in place. The *same* + // minter has to recover — a fresh one would only show that a good pair + // mints, not that the instance which just refused a mismatch stopped + // refusing. It can, because the mismatch path caches nothing. + let (matching_cert, matching_key) = cert_and_key(); + std::fs::write(&key_path, &matching_key).expect("write key"); + std::fs::write(&cert_path, &matching_cert).expect("write cert"); + assert!( + minter.current().is_some(), + "a consistent cert/key pair must mint normally" + ); + } + + #[test] + fn mints_a_token_signed_by_the_client_cert_key() { + let (cert_pem, key_pem) = cert_and_key(); + let dir = tempfile::tempdir().expect("tempdir"); + let minter = NodeJwtMinter::new( + write_temp(&dir, "cert.pem", &cert_pem), + write_temp(&dir, "key.pem", &key_pem), + ); + + let token = minter.current().expect("token minted"); + let claims = decode_against_own_cert(&token); + assert_eq!(claims.sub, SPIFFE_URI); + assert_eq!(claims.exp - claims.iat, NODE_JWT_TTL_SECS); + } + + #[test] + fn sec1_key_pem_is_accepted() { + // Vault PKI hands out SEC1-encoded EC keys; re-encode the test key the + // same way and make sure minting still works. + use p256::pkcs8::DecodePrivateKey; + let (cert_pem, key_pem) = cert_and_key(); + let secret = p256::SecretKey::from_pkcs8_pem(&key_pem).expect("pkcs8 parses"); + let sec1_pem = secret + .to_sec1_pem(LineEnding::LF) + .expect("sec1 encodes") + .to_string(); + assert!(sec1_pem.contains("BEGIN EC PRIVATE KEY")); + + let dir = tempfile::tempdir().expect("tempdir"); + let minter = NodeJwtMinter::new( + write_temp(&dir, "cert.pem", &cert_pem), + write_temp(&dir, "key.pem", &sec1_pem), + ); + let token = minter.current().expect("token minted from SEC1 key"); + assert_eq!(decode_against_own_cert(&token).sub, SPIFFE_URI); + } + + #[test] + fn token_is_cached_until_near_expiry() { + let (cert_pem, key_pem) = cert_and_key(); + let dir = tempfile::tempdir().expect("tempdir"); + let minter = NodeJwtMinter::new( + write_temp(&dir, "cert.pem", &cert_pem), + write_temp(&dir, "key.pem", &key_pem), + ); + let first = minter.current().expect("token"); + + // Remove the material a mint would need. Comparing two live mints + // proves nothing: `iat`/`exp` come from whole seconds, so two fresh + // mints in the same second are byte-identical and the assertion would + // hold with the cache removed entirely. Only a token served without + // readable files can have come from the cache. + std::fs::remove_file(dir.path().join("cert.pem")).expect("remove cert"); + std::fs::remove_file(dir.path().join("key.pem")).expect("remove key"); + + let second = minter + .current() + .expect("fresh token must be served from cache"); + assert_eq!(first, second); + } + + #[test] + fn missing_files_yield_none_not_panic() { + let minter = NodeJwtMinter::new( + "/nonexistent/cert.pem".to_string(), + "/nonexistent/key.pem".to_string(), + ); + assert!(minter.current().is_none()); + } +} diff --git a/crates/rpc/src/node_token_socket.rs b/crates/rpc/src/node_token_socket.rs new file mode 100644 index 0000000000..7dc313e9ef --- /dev/null +++ b/crates/rpc/src/node_token_socket.rs @@ -0,0 +1,400 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Node-auth tokens for co-located services that hold no key (issue #355). +//! +//! On a DPU, only the dpu-agent holds the machine's private key; other NICo +//! pods (fmds, …) obtain bearer tokens from the agent's local API — the +//! `AgentLocal` gRPC service on a unix socket in the shared `/opt/forge` +//! directory — instead of mounting the key to do their own mTLS/minting. +//! +//! [`SocketTokenSource`] keeps a cached token fresh with a background task and +//! serves it synchronously from [`NodeTokenProvider::current`] on the request +//! path. Until the first successful fetch (e.g. the agent hasn't started or +//! registered yet), requests simply go out without a bearer header. + +use std::sync::{Arc, RwLock, Weak}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use hyper_util::rt::TokioIo; +use tokio::net::UnixStream; +use tonic::transport::{Channel, Endpoint, Uri}; +use tower::service_fn; + +use crate::node_jwt::NodeTokenProvider; +use crate::protos::agent_local::GetNodeTokenRequest; +use crate::protos::agent_local::agent_local_client::AgentLocalClient; + +/// Default path of the dpu-agent's local API socket. It lives in a dedicated +/// `run/` subdirectory of the shared `/opt/forge` credentials directory so a +/// containerized consumer can be given the socket without being given the +/// credentials beside it: the pod mounts `run/` and never the directory that +/// holds the machine key. +/// +/// The mount can be read-only. Connecting does not write to the filesystem, +/// and Linux applies the read-only-mount check only to regular files, +/// directories and symlinks -- socket inodes are exempt -- so `connect(2)` +/// succeeds. Access is gated by the socket's own 0600 mode instead. +pub const DEFAULT_AGENT_LOCAL_SOCKET: &str = "/opt/forge/run/agent.sock"; + +/// The background task re-fetches when less than this long remains on the +/// cached token. The request path keeps serving the cached token down to +/// HALF this margin — the refresher normally replaces it well before that — +/// so a briefly-late refresh doesn't strip requests of their header. +const REFRESH_MARGIN_SECS: u64 = 60; + +/// Delay between fetch attempts while the agent socket is absent or erroring, +/// and the per-attempt deadline for a fetch (connect + RPC) so a stalled +/// handshake can't wedge the refresh loop. +const RETRY_DELAY: Duration = Duration::from_secs(5); + +pub struct SocketTokenSource { + socket_path: String, + cached: RwLock>, + /// Signalled when the last `Arc` is dropped. + /// + /// The weak reference alone is enough for the refresh loop to *finish*, + /// but only at the top of its next iteration — which, on a healthy token, + /// is minutes away. That leaves a task sleeping on a source nobody holds, + /// which shows up as a runtime that will not quiesce in tests and as + /// pointless wakeups in production. + shutdown: Arc, +} + +impl Drop for SocketTokenSource { + fn drop(&mut self) { + // `notify_one` leaves a permit behind when nothing is waiting yet, so + // a drop that lands between the loop's iterations is still observed on + // its next await rather than being lost. + self.shutdown.notify_one(); + } +} + +/// Manual impl so the cached bearer token (a live credential) never lands in +/// debug output of the client config, which is itself `Debug`-logged. Mirrors +/// the redaction on [`NodeJwtMinter`](crate::node_jwt::NodeJwtMinter). +impl std::fmt::Debug for SocketTokenSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SocketTokenSource") + .field("socket_path", &self.socket_path) + .finish_non_exhaustive() + } +} + +impl SocketTokenSource { + /// Creates the source and spawns its refresh loop on the current tokio + /// runtime. The loop holds only a weak reference, and is woken on drop, so + /// releasing the last `Arc` (and the clients built from it) stops it + /// within moments rather than at its next scheduled refresh. + #[must_use] + pub fn spawn(socket_path: String) -> Arc { + let shutdown = Arc::new(tokio::sync::Notify::new()); + let source = Arc::new(Self { + socket_path, + cached: RwLock::new(None), + shutdown: Arc::clone(&shutdown), + }); + tokio::spawn(refresh_loop(Arc::downgrade(&source), shutdown)); + source + } + + async fn fetch(socket_path: &str) -> Result<(String, u64), tonic::Status> { + // Deadline over connect + RPC together: a peer that accepts the + // connection but never answers must fail the attempt, not wedge the + // refresh loop forever. + tokio::time::timeout(RETRY_DELAY, async { + let channel = connect_uds(socket_path) + .await + .map_err(|e| tonic::Status::unavailable(format!("agent local socket: {e}")))?; + let response = AgentLocalClient::new(channel) + .get_node_token(GetNodeTokenRequest {}) + .await? + .into_inner(); + Ok((response.token, response.expires_at)) + }) + .await + .map_err(|_| tonic::Status::deadline_exceeded("agent local API fetch timed out"))? + } +} + +impl NodeTokenProvider for SocketTokenSource { + fn current(&self) -> Option { + let now = unix_now()?; + // Recover from poisoning rather than treating it as "no token". A + // fetched token is the only credential this source has — unlike the + // minter, it cannot fall back to producing one — so `.ok()?` here + // would strip the bearer header from every later request, with the + // refresher below equally unable to repopulate the cache. The guarded + // value is a plain `(String, u64)` that a panicking reader cannot have + // left inconsistent. Matches `NodeJwtValidator`'s handling. + self.cached + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .as_ref() + .filter(|(_, expires_at)| *expires_at > now + REFRESH_MARGIN_SECS / 2) + .map(|(token, _)| token.clone()) + } +} + +async fn connect_uds(socket_path: &str) -> Result { + let socket_path = socket_path.to_owned(); + // The URI is required by the Endpoint API but unused for UDS connections. + Endpoint::try_from("http://[::]:50051")? + .connect_with_connector(service_fn(move |_: Uri| { + let path = socket_path.clone(); + async move { + let stream = UnixStream::connect(path).await?; + Ok::<_, std::io::Error>(TokioIo::new(stream)) + } + })) + .await +} + +async fn refresh_loop(source: Weak, shutdown: Arc) { + // Sleeping is where this loop spends almost all of its life — up to a + // whole token lifetime — so that is where a drop has to be observable. + // The fetch itself is already bounded by its own deadline, so it is left + // to finish rather than being torn down mid-RPC. + let sleep_or_shutdown = async |duration| { + tokio::select! { + () = shutdown.notified() => false, + () = tokio::time::sleep(duration) => true, + } + }; + + loop { + // Re-upgrade each iteration so the loop exits once every user of the + // source is gone, instead of pinning it alive forever. + let Some(source) = source.upgrade() else { + return; + }; + let socket_path = source.socket_path.clone(); + match SocketTokenSource::fetch(&socket_path).await { + Ok((token, expires_at)) => { + *source + .cached + .write() + .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some((token, expires_at)); + drop(source); + let sleep_secs = unix_now() + .map(|now| { + expires_at + .saturating_sub(now) + .saturating_sub(REFRESH_MARGIN_SECS) + }) + .unwrap_or(0) + .max(RETRY_DELAY.as_secs()); + if !sleep_or_shutdown(Duration::from_secs(sleep_secs)).await { + return; + } + } + Err(status) => { + tracing::debug!( + target: "node_auth", + socket = %socket_path, + %status, + "node-auth: could not fetch node token from agent local API; will retry" + ); + drop(source); + if !sleep_or_shutdown(RETRY_DELAY).await { + return; + } + } + } + } +} + +fn unix_now() -> Option { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .ok() +} + +#[cfg(test)] +mod tests { + use tokio_stream::wrappers::UnixListenerStream; + use tonic::{Request, Response, Status}; + + use super::*; + use crate::protos::agent_local::GetNodeTokenResponse; + use crate::protos::agent_local::agent_local_server::{AgentLocal, AgentLocalServer}; + + struct FixedToken(String, u64); + + #[tonic::async_trait] + impl AgentLocal for FixedToken { + async fn get_node_token( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(GetNodeTokenResponse { + token: self.0.clone(), + expires_at: self.1, + })) + } + } + + fn serve(socket: &std::path::Path, token: &str, expires_at: u64) { + let listener = tokio::net::UnixListener::bind(socket).expect("bind uds"); + let service = AgentLocalServer::new(FixedToken(token.to_string(), expires_at)); + tokio::spawn( + tonic::transport::Server::builder() + .add_service(service) + .serve_with_incoming(UnixListenerStream::new(listener)), + ); + } + + async fn wait_for_token(source: &SocketTokenSource) -> Option { + for _ in 0..100 { + if let Some(token) = source.current() { + return Some(token); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + None + } + + #[tokio::test] + async fn fetches_token_from_agent_socket() { + let dir = tempfile::tempdir().expect("tempdir"); + let socket = dir.path().join("agent.sock"); + let expires_at = unix_now().expect("clock") + 300; + serve(&socket, "the.node.token", expires_at); + + let source = SocketTokenSource::spawn(socket.to_string_lossy().into_owned()); + assert_eq!( + wait_for_token(&source).await.as_deref(), + Some("the.node.token") + ); + } + + #[tokio::test] + async fn missing_socket_yields_none_and_no_panic() { + let source = SocketTokenSource::spawn("/nonexistent/agent.sock".to_string()); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!(source.current().is_none()); + } + + #[tokio::test] + async fn debug_output_redacts_the_cached_token() { + let dir = tempfile::tempdir().expect("tempdir"); + let socket = dir.path().join("agent.sock"); + serve( + &socket, + "super.secret.token", + unix_now().expect("clock") + 300, + ); + + let source = SocketTokenSource::spawn(socket.to_string_lossy().into_owned()); + wait_for_token(&source).await.expect("token cached"); + + // ForgeClientConfig is Debug-logged, so a cached token must never + // appear in the provider's debug output. + let rendered = format!("{source:?}"); + assert!( + !rendered.contains("super.secret.token"), + "cached token leaked into Debug: {rendered}" + ); + } + + #[tokio::test] + async fn expired_cached_token_is_not_served() { + let dir = tempfile::tempdir().expect("tempdir"); + let socket = dir.path().join("agent.sock"); + // The agent hands out a token that is already (nearly) expired. + serve(&socket, "stale.token", unix_now().expect("clock")); + + let source = SocketTokenSource::spawn(socket.to_string_lossy().into_owned()); + tokio::time::sleep(Duration::from_millis(200)).await; + assert!( + source.current().is_none(), + "a token inside the refresh margin must not be presented" + ); + } + + /// Fetching is the only way this source can ever hold a credential, so a + /// poisoned cache must not become a permanent "no token" — that would + /// strip the bearer header from every later request, with the refresher + /// equally unable to recover. + #[tokio::test] + async fn a_poisoned_cache_still_serves_its_token() { + let dir = tempfile::tempdir().expect("tempdir"); + let socket = dir.path().join("agent.sock"); + let expires_at = unix_now().expect("clock") + 3600; + serve(&socket, "token.after.poison", expires_at); + + let source = SocketTokenSource::spawn(socket.to_string_lossy().into_owned()); + assert_eq!( + wait_for_token(&source).await.as_deref(), + Some("token.after.poison"), + "precondition: the source must hold a token before it is poisoned" + ); + + // Poison the lock the only way it can happen: panic while holding the + // write guard. The panic hook is left alone — it is global, and + // swapping it here would suppress the output of any test panicking + // concurrently. The backtrace this prints is expected. + let poisoned = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _guard = source.cached.write().expect("not yet poisoned"); + panic!("deliberate panic to poison the cache (expected)"); + })); + assert!(poisoned.is_err(), "the panic should have been caught"); + assert!( + source.cached.read().is_err(), + "precondition: the lock must actually be poisoned" + ); + + assert_eq!( + source.current().as_deref(), + Some("token.after.poison"), + "a poisoned cache must still serve the token it holds" + ); + } + + /// Dropping the last `Arc` must stop the refresh task promptly, not at its + /// next scheduled wake. With nothing listening the loop parks on + /// `RETRY_DELAY`, so a loop that only re-checks its `Weak` at the top of + /// the iteration would outlive the source by that whole delay — this + /// asserts it exits in a fraction of it. + #[tokio::test] + async fn dropping_the_source_stops_the_refresh_task_promptly() { + let dir = tempfile::tempdir().expect("tempdir"); + // No server: the fetch fails fast and the loop settles into its retry + // sleep, which is the state a drop has to be able to interrupt. + let socket = dir.path().join("agent.sock"); + let shutdown = Arc::new(tokio::sync::Notify::new()); + let source = Arc::new(SocketTokenSource { + socket_path: socket.to_string_lossy().into_owned(), + cached: RwLock::new(None), + shutdown: Arc::clone(&shutdown), + }); + let task = tokio::spawn(refresh_loop(Arc::downgrade(&source), shutdown)); + + // Let it fail once and enter the retry sleep. + tokio::time::sleep(Duration::from_millis(100)).await; + drop(source); + + let stopped = tokio::time::timeout(Duration::from_secs(1), task).await; + assert!( + stopped.is_ok(), + "the refresh task must observe the drop rather than sleeping out RETRY_DELAY ({:?})", + RETRY_DELAY + ); + stopped.unwrap().expect("the refresh task must not panic"); + } +} diff --git a/crates/rpc/src/protos/mod.rs b/crates/rpc/src/protos/mod.rs index c187ab59bb..6c266dbfc2 100644 --- a/crates/rpc/src/protos/mod.rs +++ b/crates/rpc/src/protos/mod.rs @@ -106,6 +106,12 @@ pub mod fmds { include!(concat!(env!("OUT_DIR"), "/fmds.rs")); } +#[allow(non_snake_case, unknown_lints, clippy::all)] +#[rustfmt::skip] +pub mod agent_local { + include!(concat!(env!("OUT_DIR"), "/agent_local.rs")); +} + #[allow(clippy::all, deprecated)] #[rustfmt::skip] pub mod forge_api_client { diff --git a/crates/scout/src/client.rs b/crates/scout/src/client.rs index 66bf434290..21f73b9c8f 100644 --- a/crates/scout/src/client.rs +++ b/crates/scout/src/client.rs @@ -30,7 +30,10 @@ pub(super) async fn create_forge_client( cert_path: config.client_cert.clone(), key_path: config.client_key.clone(), }), - ); + ) + // Node-auth (#355): also present a self-signed bearer JWT minted from the + // client cert's key. Ignored by the API unless [node_auth] is enabled. + .with_node_jwt(); let api_config = ApiConfig::new(&config.api, &client_config); let client = forge_tls_client::ForgeTlsClient::retry_build(&api_config) diff --git a/deploy/nico-base/api/config-files/carbide-api-config.toml b/deploy/nico-base/api/config-files/carbide-api-config.toml index f31f080806..588332d016 100644 --- a/deploy/nico-base/api/config-files/carbide-api-config.toml +++ b/deploy/nico-base/api/config-files/carbide-api-config.toml @@ -43,6 +43,29 @@ hardware_health_reports = "MonitorOnly" enabled = true create_machines = true +[node_auth] +# Node (Scout / DPU-agent) authentication to the API. +# See docs/design/machine-identity/node-auth-jwt.md. +# Bearer JWTs: off by default. When enabled, the API accepts short-lived +# `Authorization: Bearer` tokens that nodes self-sign with their existing +# mTLS client-certificate key (the cert rides in the token's x5c header and +# is verified against the same root CA as the TLS listener). +enabled = false +# Machine mTLS client-certificate authentication: on by default. Disable only +# after the fleet presents bearer tokens (requires enabled = true; the API +# refuses to start with both mechanisms off). Service/admin-CLI certs are +# unaffected. +mtls_enabled = true +# `aud` claim is always "nico-api". +# Maximum accepted token lifetime in seconds (nodes mint 300 s tokens). +#max_token_ttl_sec = 900 +# Whether DPF-deployed fmds is rendered in token mode. Unset follows `enabled`, +# which is what you want unless you are staging a change: set it to false while +# `enabled` is still true to move fmds back to client certificates first, then +# turn `enabled` off once that roll has landed. true with enabled = false is +# rejected at startup. See docs/design/machine-identity/node-auth-jwt.md. +#fmds_use_node_tokens = false + [firmware_global] autoupdate = true diff --git a/docs/design/machine-identity/node-auth-jwt.md b/docs/design/machine-identity/node-auth-jwt.md new file mode 100644 index 0000000000..4c4765d0b0 --- /dev/null +++ b/docs/design/machine-identity/node-auth-jwt.md @@ -0,0 +1,688 @@ +# Node-Auth: Self-Signed Bearer JWTs for Scout and DPU-Agent + +Design for [#355](https://github.com/NVIDIA/infra-controller/issues/355) +(sub-issue of the Vault-elimination epic +[#195](https://github.com/NVIDIA/infra-controller/issues/195)): Scout and the +DPU-agent authenticate to the API with short-lived bearer JWTs alongside — +and eventually instead of — mTLS client certificates. + +Nodes sign their own tokens with the private key of their **existing** mTLS +client certificate. There is no new key material anywhere, no server-side +signing key or key storage, and no dependency on the `machine_identity` +(tenant JWT-SVID) subsystem. **No node-auth JWT is issued or refreshed by the +API**: nodes re-mint locally, so this design adds no token-issuance or refresh +RPC to the API. (The API does still issue the *certificate* those tokens are +signed with, through the existing discovery/attestation flow — that is the +credential this design reuses, not one it replaces.) The agent serves a local +`GetNodeToken` RPC to processes on its own DPU, which distributes tokens it +has already minted rather than acting as an issuance authority. + +On a DPU the dpu-agent is the only holder of that key *for the purpose of +authenticating to nico-api*: co-located NICo pods get finished tokens from it +over a local unix socket instead of mounting the key +(see [Key distribution on DPF](#key-distribution-on-dpf-the-agent-is-the-token-broker)). +The key has not left the node entirely — otelcol still mounts the credentials +directory, because it needs the certificate for TLS client auth to its OTLP +gateway, which a nico-api bearer token cannot replace. +Approaches that were weighed and rejected are recorded under +[Designs not used](#designs-not-used). + +It is distinct from the tenant-facing [SPIFFE JWT-SVID design](spiffe-svid-sdd.md): +that issues identity tokens *to tenant workloads* via IMDS; this design covers +how *NICo's own node agents* authenticate to the NICo API. + +## How it works + +```text +node (scout / dpu-agent) nico-api +------------------------ -------- +has /opt/forge/machine_cert.pem ── x5c ──► 1. verify x5c chain against the +and /opt/forge/machine_cert.key client-cert root CA (same roots + as the TLS listener) +mint ES256 JWT signed with the 2. verify JWT signature with the +cert's own key, 5-min TTL, verified leaf's public key +cert chain in the x5c header 3. enforce exp / iat / aud, bounded + lifetime +attach as Authorization: Bearer 4. SPIFFE-validate the leaf; map its +on every gRPC request URI SAN through the same + SpiffeContext as mTLS certs + ⇒ identical machine principal, RBAC + unchanged +``` + +Key-less co-located services (fmds) never touch the cert or key at all: they +ask the dpu-agent for its current token over a unix socket and attach that +same header. + +```text +fmds pod dpu-agent nico-api +-------- --------- -------- +no key, no cert ──socket──► holds the key, ── same bearer token ──► +GetNodeToken mints as above (identical +(cached, refreshed verification) + in the background) +``` + +## Auth flow: new DPU → first authorized gRPC call + +**Provisioning & bootstrap (no credentials yet)** + +- A new DPU is provisioned (BFB installed via DPF); the `forge-dpu-agent` DPF + service starts on it. Two config inputs are sufficient to get through + bootstrap: the API endpoint and the root CA bundle (`forge_system.root_ca`). + The cert/key files at `/opt/forge/machine_cert.pem` / `machine_cert.key` + don't exist yet. Node-auth has no per-node configuration: every node mints + the fixed `nico-api` audience once its certificate lands. +- The agent opens a TLS connection to `nico-api` that is **server-auth only** + — it verifies the API's cert against the root CA but presents no client + credential (the API listener uses `allow_unauthenticated()`, so the + connection is accepted with an anonymous principal). + +**First credential: the machine certificate** + +- The agent calls `DiscoverMachine` + (`host-support/registration.rs::register_machine`) carrying its hardware + enumeration (`DiscoveryInfo`); this RPC is reachable pre-credential by + design — it is how a machine with no credential obtains its first one, so it + cannot itself require one. +- **What authorizes it is the network, not a credential.** The handler + (`handlers/machine_discovery.rs`) resolves the caller from the connection's + source IP: `find_optional_for_update_by_ip` maps the peer address to a + registered `machine_interface`, and the machine that interface belongs to is + the one that gets a certificate. A caller cannot name the machine it wants + to be. An already-allocated host running scout may pass a + `machine_interface_id`, but it is checked against the source IP + (`find_for_update_if_matches_instance_ip`); a mismatch is refused as + `PermissionDenied` and logged as a potential impersonation attempt. +- The strength of that boundary is therefore the strength of the provisioning + network: anything that can source packets from a registered machine's IP, + before that machine has enrolled, can obtain its certificate — and hence + mint node-auth tokens as it. Node-auth inherits this; it neither weakens nor + strengthens it, because the JWT is signed by the very key this exchange + delivers. Sites that need more should enable attestation, where hosts take + the `AttestQuote` path and issuance is gated on a TPM challenge instead. +- `allow_insecure_discovery` bypasses the IP check entirely and trusts a + caller-supplied `machine_interface_id`. It is for integration tests, logs a + warning when it fires, and must stay off in production. +- The API registers/matches the machine and returns a `machine_certificate` + in the response: a Vault-PKI-issued EC P-256 leaf whose SAN is the machine's + SPIFFE URI (`spiffe:////machine/`), plus the + issuing CA and private key. (On attestation-enabled sites, hosts get this + via `AttestQuote` after a TPM challenge instead; DPUs take the discovery + path.) +- `write_certs` persists leaf+issuing-CA to `/opt/forge/machine_cert.pem` and + the key to `/opt/forge/machine_cert.key`. **This existing cert key is the + JWT signing key — nothing else is ever created.** + +**Minting the JWT (client side, `rpc::node_jwt`)** + +- The agent's gRPC client was built with + `ForgeClientConfig::new(root_ca, ClientCert{...}).with_node_jwt()`, so a + `NodeJwtMinter` watches those two file paths. +- On the next outgoing RPC, the minter reads cert+key from disk (re-encoding + Vault's SEC1 key PEM to PKCS#8) and signs an **ES256 JWT**: header + `{alg: ES256, x5c: [leaf, issuing CA]}`, claims + `{sub: , aud: "nico-api", + iat: now, exp: now+300s}`. The token is cached and re-minted when < 60 s + remain — no refresh RPC, and cert renewal is picked up automatically + because the files are re-read at each mint. +- Renewal writes the certificate and the key in two separate operations, so + the minter compares the key's public half against the one certified by the + leaf before signing. A mismatch (a mint landing between the two writes) + fails the mint rather than caching a token the API will reject. +- `BearerAuthService` stamps `Authorization: Bearer ` onto the request. + (The TLS channel may still present the client cert too — dual-support; each + is sufficient alone.) + +**Validation (server side, requires `[node_auth] enabled = true` + TLS listener)** + +- The authn middleware sees the Bearer header and hands it to + `NodeJwtValidator`, which checks in order: + - header `alg` is exactly ES256 (no algorithm substitution); + - the `x5c` chain verifies against the **same root CA file the TLS listener + uses for client certs** (`[tls] root_cafile_path`) — path building, + validity window, client-auth EKU; + - the JWT signature verifies with the *verified leaf's* public key; + - claims: `exp` and `aud` are validated by `jsonwebtoken`; `iat` is + required when `NodeClaims` deserializes and is independently checked to be + no later than `exp` or the API's clock-skew allowance. The validator also + bounds both `exp - iat` and how far `exp` may reach into the future by + `max_token_ttl_sec` (default 900 s), so a client can't stretch `exp`; + - the leaf passes SPIFFE validation (leaf-only, single URI SAN), and `sub` + must equal that SAN — identity comes from the verified cert, never from + the claim. + +**Authorization & the first call** + +- The validated SPIFFE URI is mapped through the same `SpiffeContext` as mTLS + certs → a `SpiffeMachineIdentifier` principal, byte-identical to what the + cert path would have minted. +- Casbin RBAC evaluates that principal exactly as before (machine class → + Agent/Scout role rules) — **no RBAC changes** — and the handler executes. + That is the first authorized gRPC call on bearer auth. + +Note: the very first *authorized* call after discovery could ride either +credential, since the agent holds both from the same moment — the JWT only +becomes load-bearing once `mtls_enabled = false`. + +## Q1 — How does the agent get a private key that is trusted to create the JWT? + +**It already has one.** The node's Vault-issued mTLS client certificate key +(`/opt/forge/machine_cert.key`, EC P-256 — Vault PKI role `key_type=ec, +key_bits=256`) signs the JWT, and the certificate itself rides along in the +token's `x5c` header (RFC 7515 §4.1.6). The key is trusted because the +certificate chains to the root CA the API already trusts for client certs — +the JWT is effectively "mTLS at the application layer". + +Bootstrapping is unchanged: a machine obtains its first certificate through +the existing discovery/attestation flow (`DiscoverMachine` / `AttestQuote` +respond with the machine certificate — see the auth-flow walkthrough above), +and from that moment it can mint tokens. Minting is best-effort: before the +cert exists, requests simply carry no bearer header. + +## Q2 — JWT side by side with mTLS + +The authn middleware (`CertDescriptionMiddleware` in `crates/authn`) mints +principals from **both** sources on every request: the TLS-layer client cert, +and the `Authorization: Bearer` token (validated by `NodeJwtValidator`). Both +paths converge on the same SPIFFE URI → `SpiffeMachineIdentifier` mapping +through the same `SpiffeContext`, so RBAC (Casbin policy, role mapping) is +completely unchanged. A node presenting both credentials gets the same +principal twice — harmless. Scout configures a bearer-token provider with +`ForgeClientConfig::with_node_jwt()` and DPU-agent with +`ForgeClientConfig::with_token_provider()`, regardless of whether node-auth is +enabled. The provider attaches the `Authorization` header only when it can +obtain a token; before the certificate exists, requests carry no bearer header. +A server with node-auth disabled ignores any header that is present. + +**Enabling** is therefore order-independent: node and API images can be rolled +in either order, because a token nobody validates is inert and a node that +cannot mint yet simply sends no header. + +**Disabling is not.** Once fmds is deployed in token mode, turning +`[node_auth] enabled` off stops the API accepting bearer tokens immediately +while fmds returns to cert mode only as DPF rolls the DaemonSet — see +[Known issues](#known-issues) for the resulting window and +[Disabling node-auth](#disabling-node-auth) for the order that avoids it. The +symmetry claim applies to the enable direction only. + +## Q3 — JWT off by default, configured in the API config + +```toml +[node_auth] +enabled = false # master switch for accepting bearer JWTs +max_token_ttl_sec = 900 # upper bound on client-chosen lifetimes (cap 86400) +# fmds_use_node_tokens # optional; unset follows `enabled`. Only for staging + # a change — see "Disabling node-auth" below. +``` + +All `[node_auth]` values are read at API startup; changing any of them requires +an API restart. The defaults and validation contract is: + +| Setting | Default and accepted values | Startup behavior | +| --- | --- | --- | +| `enabled` | `false` or `true`. | `false` installs no bearer validator. `true` requires a TLS listener and readable `[tls] root_cafile_path`; the API refuses startup rather than accepting bearer tokens over plaintext. | +| `max_token_ttl_sec` | `900` seconds; with bearer auth enabled, an integer from `300` (the node's fixed minted lifetime) through `86400`, inclusive. | `0`, `1`–`299`, and values greater than `86400` fail startup when `enabled = true`. When bearer auth is disabled, this inactive value is not otherwise validated. | +| `mtls_enabled` | `true` or `false`; it controls only machine mTLS identities. | Setting both `enabled` and `mtls_enabled` to `false` always fails startup because no node credential would remain. Service and admin client certificates are unaffected. | +| `fmds_use_node_tokens` | Unset; then follows `enabled`. An explicit boolean is a rollout override. | `true` with `enabled = false` always fails startup, because that deployment would make fmds present tokens to an API that rejects them. | + +The `aud` claim is always `"nico-api"`, defined once in +`rpc::node_jwt::NODE_JWT_AUDIENCE` and used by every minter and by the API +validator. It has no configuration, CLI flag, or Helm value. `[node_auth]` +rejects unknown keys, including a legacy `audience` setting, rather than +silently accepting a value that cannot change token validation. + +The entire preflight runs *before* DPF resource creation, so a +misconfiguration cannot mutate cluster state on its way to failing. + +## Q4 — mTLS on by default, disableable in the API config + +```toml +[node_auth] +mtls_enabled = true +``` + +When `mtls_enabled = false`, the middleware stops minting machine principals +from client certificates — bearer JWTs become the only node auth path. The +gate is scoped to **machine** certs: service and admin-CLI certs on the same +listener are unaffected. `enabled = false` + `mtls_enabled = false` is +rejected at startup (node lockout). + +Note the trust chain is still the certificate PKI: disabling mTLS here +disables the *transport-layer* cert authentication, not cert issuance. Nodes +must keep renewing certificates because the JWT is signed by the cert key. + +Both mechanisms depend on `listen_mode = "tls"`, though only one says so. +Bearer tokens are refused over plaintext at startup; machine mTLS just quietly +has nothing to work with, because the plaintext accept path hands the +middleware an empty peer-certificate list. So the both-off lockout check +guarantees a working path only on a TLS listener -- on plaintext, +`mtls_enabled = true` passes it while authenticating nobody. Nothing shipped +selects a plaintext mode, which is why this is documented rather than +enforced: requiring TLS whenever `mtls_enabled` is set would break local +plaintext development to prevent a misconfiguration that only matters if you +expected node auth to work in the first place. + +## Q5 — Key regeneration and public-key exchange + +**Regeneration** is the existing client-certificate renewal: when +`ClientCertRenewer` rotates the cert/key files, the minter picks the new pair +up on its next re-mint (it re-reads both files from disk each time). No +coordination, no state. + +**Public-key exchange: the x5c header.** Every token carries the certificate +chain that vouches for its signing key, and the API verifies that chain +against the root CA bundle it already holds (`[tls] root_cafile_path`). There +is no JWKS endpoint, no key registry, and no key distribution problem — CA +rotation is handled wherever the root bundle is handled today. + +**CA rotation** moves both consumers of that bundle together. The TLS +listener already re-reads the file every five minutes for cert-manager +rotations; the validator's trust anchors are held behind a lock and refreshed +on that same tick, so a token chaining to a freshly rotated CA is accepted +without an API restart. A failed reload keeps the previous anchors — a bundle +caught mid-write must not disarm node auth. + +**Compromise response** is likewise the PKI's: a stolen key/cert pair is the +same incident as a stolen mTLS cert today. Two cases, with very different +bounds — do not conflate them: + +- **A captured token** is bounded by expiry. It is replayable only until its + own `exp` (see the replay window in Design decisions), against the same + `aud`, over TLS. Waiting is a sufficient response. +- **A compromised private key** is not bounded by token expiry at all. The + holder mints fresh, valid tokens whenever it likes, for as long as the + *certificate* remains acceptable. The validator calls + `allow_unknown_revocation_status()`, so revoking the certificate does not + stop it either: the only things that do are the certificate's own expiry, + or removing the issuing CA from the bundle the API trusts — which cuts off + every machine under that CA, not just the compromised one. + +So key compromise requires incident response, not patience — and it is worth +being precise about what each response actually achieves: + +- **Re-issuing the machine's credential is recovery, not containment.** It + gets the legitimate node onto a fresh key. It does nothing to the attacker, + who keeps minting acceptable tokens from the old certificate until that + certificate expires, because the validator does not check revocation. +- **Rotating the issuing CA is the containment step**, and the only one + available today. It invalidates the stolen certificate immediately — along + with every other certificate under that CA, so every machine has to + re-enrol. That is the cost, and it is why this is a decision rather than a + runbook step. +- **Waiting for expiry** contains it eventually, bounded by the certificate's + lifetime rather than the token's. + +Short token lifetimes limit what a *captured token* is worth; they do nothing +to limit a stolen key. Cutting a compromised key off surgically — without +taking the whole CA with it — needs revocation checking we do not do yet. + +## Q6 — JWT best-practice checklist + +| Practice | How it's honored | +| --- | --- | +| Asymmetric signing, no `alg` confusion | ES256 only; both the header check and `Validation` pin the algorithm, so `none`/HS256 substitution is rejected. | +| Identity never comes from claims | The principal derives from the **chain-verified certificate's SPIFFE SAN**; `sub` is only cross-checked against it. A forged `sub` buys nothing. | +| Short-lived tokens | Clients mint 300 s tokens; the server enforces `exp - iat` and `exp - now` ≤ `max_token_ttl_sec` (default 900 s, hard cap 86400), so a client cannot stretch `exp`. | +| `exp` / `iat` / `aud` enforced | `jsonwebtoken` validates `exp` and `aud`; `NodeClaims` requires `iat`, and the validator rejects an `iat` after `exp` or more than its clock-skew allowance in the future. It also bounds `exp - iat` and `exp - now`. | +| Chain validation, not pinning | `x5c` verified with rustls `WebPkiClientVerifier` (path building, validity window, client-auth EKU) against the same roots as the TLS listener. | +| SPIFFE leaf constraints | `carbide_authn::validate_x509_certificate` re-checks leaf-ness, key usage, and the single-URI-SAN rule — same code path as mTLS certs. | +| No bearer tokens over plaintext | Enforced at both ends. Server: startup refuses `enabled = true` on a non-TLS listener, the middleware only installs the validator when the listener is TLS-terminated, and a failed acceptor rebuild keeps the previous acceptor rather than falling back to cleartext. Client: `with_token_provider` implies `require_tls_enforcement`, and building a token client against a non-HTTPS URL is an error. | +| No key material at rest beyond the PKI | The server holds no signing key; the client holds only what it already had, and `write_certs` persists the key file owner-only (0600). Credentials never in logs (both token caches have a redacting `Debug`). | +| Least exposure for the signing key | The dpu-agent is the only process that signs node-auth tokens: consumers get finished short-lived ones over a local socket, and token-mode fmds never mounts the credentials directory. Not yet absolute — otelcol still mounts it, since it needs the certificate for TLS client auth to its OTLP gateway. | + +## Component map + +| Piece | Where | +| --- | --- | +| Client mint + cache + header injection | `crates/rpc/src/node_jwt.rs` (`NodeJwtMinter`, `BearerAuthService`, `NodeTokenProvider`) | +| Client opt-in | `ForgeClientConfig::with_node_jwt()` / `with_token_provider()` (`crates/rpc/src/forge_tls_client.rs`); called in scout `client.rs` and dpu-agent `lib.rs` | +| Key-less token source | `crates/rpc/src/node_token_socket.rs` (`SocketTokenSource`), consumed by fmds via `--node-token-socket` | +| Broker service | `AgentLocal/GetNodeToken` in `crates/agent/proto/agent_local.proto`, with bindings generated once by `carbide-rpc` and served by `crates/agent/src/local_api.rs` | +| Server validation | `crates/api-core/src/node_auth.rs` (`NodeJwtValidator`) | +| Config | `NodeAuthConfig` in `crates/api-core/src/cfg/file.rs` (`[node_auth]`) | +| Middleware hook | `BearerTokenAuthenticator` trait + machine-cert gate in `crates/authn/src/middleware.rs` | +| Wiring | `crates/api-core/src/setup.rs` (preflight, validator construction), `listener.rs` (middleware install, trust-anchor reload), `dpf_services.rs` (fmds token mode) | +| Charts | `bluefield/charts/nico-fmds` (`useNodeTokens`), `bluefield/charts/nico-dpu-agent` | + +All of it logs under one target: `RUST_LOG=node_auth=debug` turns on the +whole feature's tracing, and every message carries a `node-auth:` prefix. + +## Key distribution on DPF: the agent is the token broker + +The machine cert/key live at `/opt/forge` on the DPU, a hostPath directory +several NICo pods mount. Rather than share the key with each of them, the +dpu-agent is the only holder and hands out finished tokens. + +### `AgentLocal/GetNodeToken` contract + +`GetNodeTokenRequest` has no fields: callers neither send a current token nor +an identity for the agent to inspect. Authorization is the local socket mount +and its mode, not a request field. On success, `GetNodeTokenResponse.token` is +an ES256 JWT for the agent's machine identity and `expires_at` is its UNIX-time +expiration in seconds. Callers must treat `expires_at` as the only freshness +signal; the same token string may be returned repeatedly. + +The agent returns a cached token only when it had more than 60 seconds left at +the agent's cache check. Otherwise it reads the current certificate and key and +mints a replacement with the normal 300-second lifetime. This is not a +single-flight operation: concurrent cache misses can mint more than one valid +token, and callers must not infer ordering or uniqueness from the response. + +| Cache and credential state | Result | State after the call | +| --- | --- | --- | +| Cached token has more than 60 seconds remaining | `OK` with that cached token and its original `expires_at`. | Cache is unchanged. | +| Cache is absent or has 60 seconds or less remaining, and minting succeeds | `OK` with a replacement token and its new `expires_at`. | The replacement becomes the cached token. | +| Cache is absent or too close to expiry, and the certificate/key cannot be read, parsed, or matched | `UNAVAILABLE`: `no node token available yet; machine certificate not present or unreadable`. | No usable token is returned; a stale cache entry, if any, is not served. | + +The method defines no application deadline and performs no server-side retry. +Raw gRPC callers choose their own deadline and retry policy. The bundled +`SocketTokenSource` bounds each connect-plus-RPC attempt to five seconds, +retries failures every five seconds in one background loop, refreshes at 60 +seconds remaining, and stops serving its own cache at 30 seconds remaining; +its request path never waits for the RPC. Apart from transport failures, the +only status returned by this handler is `UNAVAILABLE` for a token that cannot +be minted. Concurrent requests are safe but may independently mint on a cache +miss as described above. + +- **The socket.** The agent serves `AgentLocal/GetNodeToken` on a unix socket + at `/opt/forge/run/agent.sock`, mode 0600 — a dedicated `run/` subdirectory + so a consumer can be given the socket without the credentials beside it. The + mount is read-only: connecting does not write to the filesystem, and Linux + exempts socket inodes from the read-only-mount check, so `connect(2)` + succeeds. Access is gated by the socket's 0600 mode. `bind` creates + the socket at umask permissions and the 0600 chmod only lands afterward, so + the directory — not the socket — is what closes that window; the agent + therefore requires the socket's parent to be dedicated, creating it `0700` or + refusing a directory that holds anything else. (Without that rule, + `local-api-socket = /run/agent.sock` would take `/run` to `0700` and lock + every non-root service on the box out of its runtime files.) The path is + configurable via `[forge-system] local-api-socket`; the server retries + forever rather than dying once, since a bare-metal boot can start the agent + before its directory exists. This socket is the consolidation point for + future agent ↔ co-located-service traffic, in place of new sockets, ports, + or file drops. +- **The consumer.** `SocketTokenSource` implements the same + `NodeTokenProvider` trait as the minter, so a client built with + `with_token_provider()` is indistinguishable on the wire from one that + minted its own. A background task keeps the cached token fresh (refresh at + 60 s remaining, request-path cutoff at half that, per-attempt deadline + covering connect + RPC); the request path never blocks, and before the + first successful fetch requests simply carry no bearer header. +- **The trust anchor, without the key.** fmds still needs the root CA, so the + agent mirrors it to `/pub/`, a directory holding nothing else. + Token-mode fmds pods mount `pub/` plus `run/` and **do not reference the + credentials volume at all**, not even in the init container — the machine + key is absent from the pod rather than merely read-only, which matters + because the container runs as UID 0. It is a directory mount, not a + `subPath`, so the atomic-rename CA replacement still propagates into a + running pod. The agent publishes from the init container, again at every + start, and then reconciles the copy against the configured CA every five + minutes — the same cadence the API listener re-reads it on. That last part + is what makes rotation work: `pub/` is the *only* trust anchor a token-mode + consumer has, since it does not mount the credentials directory, so a + publish-once-at-startup mirror would pin fmds to the old issuer and break it + the moment that issuer was retired. The reconcile compares before writing, + so a steady state costs a read rather than churning the inode consumers are + watching. +- **The switch.** fmds helm values are rendered by the API, so `useNodeTokens` + *defaults* to the one setting that makes tokens meaningful: `[node_auth] + enabled`. With node-auth off, the chart renders exactly as before. + `fmds_use_node_tokens` overrides that default, which is what makes an + ordered transition possible — see [Disabling node-auth](#disabling-node-auth). + Rollout note: enabling it requires an agent image that serves the socket, so + agent and fmds images must be deployed together. + +Scope: this covers *authentication to nico-api*. otelcol also uses the machine +cert for TLS client auth to its OTLP gateway, which a nico-api bearer token +cannot replace, so the key only fully disappears from other pods once that +ingest path has its own credential story. + +## Design decisions (resolved questions) + +1. **How does the server learn the public key?** → from the token itself + (`x5c`), verified against the existing root CA — the one key-distribution + mechanism the system already operates. +2. **Why not drop mTLS immediately, since the JWT proves the same key?** → + dual-support de-risks rollout and keeps requirement 4 orthogonal; and the + JWT still depends on the cert PKI, so cert issuance must outlive transport + mTLS. +3. **Client-side enable knob?** → none. Nodes always mint when they hold a + cert; a disabled server ignores the header (verified by middleware test). + One switch (`[node_auth] enabled`) controls the feature. +4. **Replay window** → bounded by what the *server* accepts, not by what our + clients mint. A token captured from a stock scout or dpu-agent is replayable + for ≤ 5 minutes (`NODE_JWT_TTL_SECS`), but a holder of the signing key can + mint up to `max_token_ttl_sec` — 900 s by default, and the config permits up + to 86400 — so that setting, not the client constant, is the number to reason + about. Replay is against the same `aud` over TLS only. Accepted; `jti`/nonce + tracking or DPoP-style proof-of-possession is the hardening path if needed, + and lowering `max_token_ttl_sec` tightens it today. +5. **RSA machine certs** → not supported (Vault PKI role is EC P-256 + everywhere); the validator rejects non-EC leaves with a clear debug reason. + +## Disabling node-auth + +Enabling is order-independent (see Q2). Disabling is not, so it needs a +sequence rather than a single config change. The hazard is that the API stops +*accepting* bearer tokens the moment it restarts, while fmds keeps *presenting* +them until DPF has rolled every DaemonSet. + +The safe path is to take fmds out of token mode first, while the API still +accepts tokens, and only then stop accepting them: + +1. **Move fmds back to cert mode ahead of the API.** Leave `[node_auth] + enabled = true` and set: + + ```toml + [node_auth] + enabled = true + fmds_use_node_tokens = false + ``` + + Restart the API. It keeps accepting bearer tokens, but now renders fmds + with `useNodeTokens: false`, so DPF rolls the DaemonSet back onto the + machine client certificate with nothing depending on tokens in the + meantime. The reverse combination — `fmds_use_node_tokens = true` with + `enabled = false` — is refused at startup, since it deploys fmds to present + tokens the API does not accept. +2. **Confirm the roll landed on every node** before touching the API. A single + node still in token mode is a single node that loses `phone_home` in the + next step, so check the thing that actually distinguishes the two modes — + which volumes the pods carry — not just that they restarted: + + ```bash + #!/usr/bin/env bash + set -euo pipefail + NS=dpf-operator-system + SEL=app.kubernetes.io/name=nico-fmds + + # Exactly one DaemonSet, or stop: picking the first of several would verify + # the wrong one, and zero means the selector or namespace is wrong. + mapfile -t DS < <(kubectl get daemonset -n "$NS" -l "$SEL" -o name) + [ "${#DS[@]}" -eq 1 ] || { echo "expected 1 DaemonSet, found ${#DS[@]}" >&2; exit 1; } + + # The roll must be finished before its result means anything. This is a + # precondition, not a nicety: the pod list below is a single sample, so + # checking it mid-roll can show every pod already converted while others + # have not been recreated yet. + kubectl rollout status "${DS[0]}" -n "$NS" --timeout=10m + + want=$(kubectl get "${DS[0]}" -n "$NS" -o jsonpath='{.status.desiredNumberScheduled}') + ready=$(kubectl get "${DS[0]}" -n "$NS" -o jsonpath='{.status.numberReady}') + [ -n "$want" ] && [ "$want" -gt 0 ] && [ "$ready" = "$want" ] \ + || { echo "daemonset not fully ready: ready=$ready desired=$want" >&2; exit 1; } + + mounts=$(mktemp) + trap 'rm -f "$mounts"' EXIT + + # What each pod actually mounts -- not what it declares. A volume can be + # declared and never mounted, which would read as a completed roll. + kubectl get pods -n "$NS" -l "$SEL" \ + -o jsonpath='{range .items[*]}{.spec.nodeName}{"\t"}{.metadata.name}{"\t"}{range .spec.containers[*]}{range .volumeMounts[*]}{.name}{" "}{end}{end}{"| init: "}{range .spec.initContainers[*]}{range .volumeMounts[*]}{.name}{" "}{end}{end}{"\n"}{end}' \ + | tee "$mounts" + + got=$(wc -l < "$mounts") + echo "desired=$want ready=$ready observed=$got" + ``` + + **Do not proceed unless all of these hold:** + + - `kubectl rollout status` returned success and `ready` equals `desired`; + - `$got` equals `$want`. Zero rows is a *failure*, not a pass -- a wrong + namespace, a wrong label, or pods not yet recreated all produce an empty + list that otherwise reads exactly like "nothing is in token mode any + more"; + - every row mounts `nico-certs`; + - no row mounts `nico-certs-pub` or `nico-agent-run`. + + The rollout check and the mount check answer different questions -- whether + the roll finished, and which mode it finished *into* -- and you need both. + A single node still in token mode is a single node that loses `phone_home` + in the next step, so stopping here is much cheaper than discovering it + after. +3. **Set `[node_auth] enabled = false` and restart the API.** Confirm + `mtls_enabled = true` first — the pair being false is refused at startup, + but confirming beforehand turns a failed rollout into a caught typo. + `fmds_use_node_tokens = false` can stay: it now agrees with the derived + value, and leaving it costs nothing. +4. **Verify** that node-auth is quiet: `RUST_LOG=node_auth=debug` should show + no bearer validation, and `phone_home` should be succeeding from fmds. + +To roll forward again, reverse it: turn `enabled` on and confirm the API +accepts tokens, then clear `fmds_use_node_tokens` (or set it to `true`) to +move fmds across. + +Followed in this order there is no window where fmds presents a credential the +API refuses, so this is an ordinary rolling change rather than a maintenance +window. Skipping step 1 — flipping `enabled` off on its own — reopens the gap +described below. + +## Known issues + +**Disabling `[node_auth]` after fmds has been in token mode is unsequenced.** +The two halves of the switch take effect on different clocks: + +- The API stops accepting bearer tokens *immediately*, when it restarts with + `enabled = false`. +- fmds returns to cert mode only *eventually*. The re-apply is real — + `create_initialization_objects` upserts the DPUServiceConfiguration by + forced server-side apply, so the new `useNodeTokens: false` does land — but + DPF then has to re-render and roll the DaemonSet across the fleet. + +In between, fmds pods are still running token mode with no client cert and +tokens the API now rejects: `phone_home` and machine-identity signing take +401s until the roll reaches each node. Config serving and instance metadata +are unaffected, and it self-heals once the roll completes. + +Nothing is corrupted by the transition. The agent writes `machine_cert.pem` / +`.key` and the base-path root CA unconditionally, in both modes, so cert mode +works the moment a pod rolls; leftover `pub/` and `run/` directories are +inert. + +**This is avoidable, and [Disabling node-auth](#disabling-node-auth) is how.** +`[node_auth] fmds_use_node_tokens` overrides the derived value, so fmds can be +moved off tokens while the API still accepts them; sequenced that way there is +no window at all. What remains a known issue is that the *unsequenced* change +— flipping `enabled` off on its own — still produces the outage above, and +nothing stops an operator doing that. Setting the override the wrong way round +(`true` with `enabled = false`) is refused at startup, but the plain +single-switch disable is a legitimate-looking config change with a transient +cost. + +The enable direction has the mirror window but fails safe on its own: the init +container blocks waiting for the CA in `pub/` rather than starting into a +broken state. + +## Designs not used + +**Server-issued tokens.** A site-level signing key in the credential store, a +`RefreshNodeToken` RPC, and DPU device identity as the refresh anchor. It +brings back everything this design deletes: a server-side key to store, +rotate, and share across HA replicas; issuance and refresh RPCs to build, +version, and rate-limit; and a bootstrap question of its own (what +credential authorizes the *first* refresh). Reusing the node's existing +client-cert key gets the same authenticated principal with no new secret +anywhere. Worth revisiting only if per-node keys are removed from the +architecture, at which point nothing is left to self-sign with. + +**A JWKS endpoint or key registry.** The obvious way to publish per-node +public keys, and unnecessary: `x5c` carries the certificate with every token +and the API already holds the root CA that vouches for it. A registry would +add a distribution channel that can go stale, be unreachable, or disagree +with the PKI — and CA rotation would have to be handled in two places instead +of one. + +**Reusing the tenant JWT-SVID subsystem.** `machine_identity` issues SPIFFE +JWT-SVIDs to *tenant workloads* via IMDS. Node agents authenticating to the +NICo API is a different problem with a different trust root, and coupling +them would make NICo's own control plane depend on a tenant-facing service +being healthy. + +**Kubernetes Secret for the machine key.** Secret mounts are tmpfs, so the +key stops touching DPU flash, and sharing becomes explicit, per-pod, and +auditable through the K8s API. But in DPF the DPU-cluster control plane +(kamaji-hosted etcd) runs on the x86 management cluster, so the key gains a +durable copy — plus backups — *off the DPU*, requiring etcd encryption at +rest to be guaranteed; the agent needs new K8s API rights to create and +update the Secret; and scout (pre-DPF, live-image) can't use this path at +all, so the file mechanism would survive alongside it. The token broker +addresses the same concern — key exposure surface — without moving the key +anywhere, so this became moot rather than merely deferred. + +**Sharing the key by hostPath, read-only.** What token mode replaced. A +read-only mount of the credentials directory stops nothing: the container +runs as UID 0 and can read `machine_cert.key` regardless. Whether the mount +is read-only is not the control; whether the file is in the pod's namespace +at all is. + +**Mounting just the CA file by `subPath`.** A tempting one-line fix for the +above — mount the single file rather than the directory. `subPath` +bind-mounts an inode, and `install_bootstrap_ca` replaces the CA by atomic +rename, so a running pod would keep the old anchor forever and silently fail +after a CA rotation. Hence the dedicated `pub/` directory. + +**A network endpoint for the broker.** There is precedent for it — the agent +already dials `nico-dhcp-server` over gRPC through a k8s `Service` — but the +token broker is a different kind of service, and a port is the wrong shape +for it: + +- *The socket is the authorization check.* `GetNodeToken` ignores its request + entirely: there is no authentication in the handler, because anyone who can + connect is already on the node with the path mounted. On a TCP port that + same RPC hands a full machine identity to every pod in the DPU cluster, and + to anything that can route to the DPU. +- *TLS can't fix that.* Protecting the endpoint means mTLS with the machine + cert — which the consumer does not have, that being the entire premise. The + transport cannot be secured by the credential it exists to distribute, so a + port needs a second credential system to bootstrap the first. The SPIFFE + Workload API is a unix socket for this reason. +- *Locality.* Agent and consumer are DaemonSets on the same node; the call is + same-node by construction. A `Service` routes it through cluster DNS and a + VIP whose control plane (kamaji-hosted etcd) runs on the x86 management + cluster — making "can this node authenticate at all" depend on remote + infrastructure. A socket is a path. +- *No listening port* to firewall or expose by accident, which matters on a + DPU where `nico-otelcol` runs with `hostNetwork: true`. + +The cost of the choice is real, though smaller than it first appeared: the +socket sits in its own `run/` subdirectory so a consumer can mount it without +mounting the credentials next to it, but that mount can be read-only, because +socket inodes are exempt from the read-only-mount check. And it serves +co-located consumers only — anything off-DPU needing a node token is not +covered by this design. + +Note what mode 0600 does and does not buy. It excludes other UIDs, which is +real — but the NICo pods that mount this directory (`dpu-agent`, `fmds`, +`otelcol`) all run as UID 0, so it separates none of *them* from each other. +Among those, the control that matters is which pods get the mount at all. +Against anything else on the DPU running as another user, the mode does its +usual job. The gain over sharing the key is the blast radius of a leak — a +short-lived token instead of a long-lived private key — not a hard boundary +between the NICo pods themselves. + +**Certificate revocation checking.** Not implemented — see Q5. A compromised +key keeps minting valid tokens until the certificate expires, and revoking +the certificate does not stop that; if the incident model needs a faster +cutoff than certificate expiry or a CA rotation, revocation is the work, and +this design does not do it. diff --git a/helm/charts/nico-api/files/carbide-api-config.toml b/helm/charts/nico-api/files/carbide-api-config.toml index f577156089..3939dc9b0f 100644 --- a/helm/charts/nico-api/files/carbide-api-config.toml +++ b/helm/charts/nico-api/files/carbide-api-config.toml @@ -40,6 +40,29 @@ hardware_health_reports = "MonitorOnly" enabled = true create_machines = true +[node_auth] +# Node (Scout / DPU-agent) authentication to the API. +# See docs/design/machine-identity/node-auth-jwt.md. +# Bearer JWTs: off by default. When enabled, the API accepts short-lived +# `Authorization: Bearer` tokens that nodes self-sign with their existing +# mTLS client-certificate key (the cert rides in the token's x5c header and +# is verified against the same root CA as the TLS listener). +enabled = false +# Machine mTLS client-certificate authentication: on by default. Disable only +# after the fleet presents bearer tokens (requires enabled = true; the API +# refuses to start with both mechanisms off). Service/admin-CLI certs are +# unaffected. +mtls_enabled = true +# `aud` claim is always "nico-api". +# Maximum accepted token lifetime in seconds (nodes mint 300 s tokens). +#max_token_ttl_sec = 900 +# Whether DPF-deployed fmds is rendered in token mode. Unset follows `enabled`, +# which is what you want unless you are staging a change: set it to false while +# `enabled` is still true to move fmds back to client certificates first, then +# turn `enabled` off once that roll has landed. true with enabled = false is +# rejected at startup. See docs/design/machine-identity/node-auth-jwt.md. +#fmds_use_node_tokens = false + [firmware_global] autoupdate = true