Authenticate Scout and DPU-agent with self-signed bearer JWTs

bluefield/charts/nico-dpu-agent/templates/daemonset.yaml

@@ -131,6 +131,12 @@ spec:
             {{- toYaml .Values.securityContext | nindent 12 }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           args:
+            {{- with .Values.nodeAuth.audience }}
+            # DPF agents run with no config file, so the API's [node_auth]
+            # audience can only reach them as a flag. Both ends must agree or
+            # every token this agent mints is rejected.
+            - {{ printf "--node-auth-audience=%s" . | quote }}
+            {{- end }}
             - run
             - "--hbn-config-mode=nvue-rest"
             - "--agent-platform-type=containerized"

bluefield/charts/nico-dpu-agent/tests/node_auth_audience_test.yaml

@@ -0,0 +1,46 @@
+suite: node-auth audience
+templates:
+  - daemonset.yaml
+tests:
+  # DPF agents run with no config file, so the flag is the only way the API's
+  # [node_auth] audience reaches them. A mismatch means every token this agent
+  # mints -- and every token it brokers to co-located services -- is rejected.
+  - it: should pass the configured audience to the agent
+    set:
+      image:
+        repository: test
+        tag: test
+      nodeAuth:
+        audience: nico-api-eu
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].args
+          content: --node-auth-audience=nico-api-eu
+
+  # Empty means "say nothing and let the agent default apply", so the flag must
+  # not be rendered with an empty value -- that would fail the agent's
+  # non-blank parser at startup.
+  - it: should omit the flag entirely when no audience is set
+    set:
+      image:
+        repository: test
+        tag: test
+    asserts:
+      - notContains:
+          path: spec.template.spec.containers[0].args
+          content: --node-auth-audience=
+      - lengthEqual:
+          path: spec.template.spec.containers[0].args
+          count: 5
+
+  - it: should quote an audience containing shell-significant characters
+    set:
+      image:
+        repository: test
+        tag: test
+      nodeAuth:
+        audience: 'nico"api'
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].args
+          content: '--node-auth-audience=nico"api'

bluefield/charts/nico-dpu-agent/values.yaml

@@ -67,3 +67,12 @@ dhcp_server:
 fmds:
   # Will be updated in dpf_services.rs
   service_name: ""
+
+### Node-auth (issue #355).
+nodeAuth:
+  # `aud` the agent stamps on the bearer JWTs it mints, both for its own API
+  # calls and for the tokens it brokers to co-located services. Must match the
+  # API's [node_auth] audience, which is where dpf_services.rs templates this
+  # from — so a DPF deployment always sets it. Empty means "pass no flag" and
+  # falls back to the agent's own default.
+  audience: ""

bluefield/charts/nico-fmds/templates/daemonset.yaml

@@ -53,15 +53,32 @@ spec:
             - |
               {{- $dir := .Values.certsDir | default "/opt/nico" }}
               {{- $rootCa := .Values.rootCaFile | default "nico_root.pem" }}
+              {{- if .Values.useNodeTokens }}
+              echo "Waiting for root CA in {{ $dir }}/pub ..."
+              while [ ! -f {{ $dir }}/pub/{{ $rootCa }} ]; do
+                sleep 5
+              done
+              echo "Root CA found, starting nico-fmds (node tokens from the dpu-agent)."
+              {{- else }}
               echo "Waiting for certificates in {{ $dir }} ..."
               while [ ! -f {{ $dir }}/{{ $rootCa }} ] || [ ! -f {{ $dir }}/machine_cert.pem ] || [ ! -f {{ $dir }}/machine_cert.key ]; do
                 sleep 5
               done
               echo "Certificates found, starting nico-fmds."
+              {{- end }}
           volumeMounts:
+            {{- if .Values.useNodeTokens }}
+            # Token mode never needs the credentials directory, not even to
+            # wait: the agent publishes the trust anchor to pub/, which holds
+            # nothing else.
+            - name: nico-certs-pub
+              mountPath: {{ .Values.certsDir | default "/opt/nico" }}/pub
+              readOnly: true
+            {{- else }}
             - name: nico-certs
               mountPath: {{ .Values.certsDir | default "/opt/nico" }}
               readOnly: true
+            {{- end }}
       containers:
         - name: nico-fmds
           securityContext:
@@ -71,9 +88,14 @@ spec:
             {{- $dir := .Values.certsDir | default "/opt/nico" }}
             {{- $rootCa := .Values.rootCaFile | default "nico_root.pem" }}
             - "--grpc-address=$(POD_IP):50052"
+            {{- if .Values.useNodeTokens }}
+            - "--root-ca={{ $dir }}/pub/{{ $rootCa }}"
+            - "--node-token-socket={{ $dir }}/run/agent.sock"
+            {{- else }}
             - "--root-ca={{ $dir }}/{{ $rootCa }}"
             - "--client-cert={{ $dir }}/machine_cert.pem"
             - "--client-key={{ $dir }}/machine_cert.key"
+            {{- end }}
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           {{- with toYaml .Values.serviceDaemonSet.resources }}
           resources:
@@ -106,14 +128,45 @@ spec:
                 fieldRef:
                   fieldPath: metadata.namespace
           volumeMounts:
+            {{- $dir := .Values.certsDir | default "/opt/nico" }}
+            {{- $rootCa := .Values.rootCaFile | default "nico_root.pem" }}
+            {{- if .Values.useNodeTokens }}
+            # Token mode: the bearer JWT is the credential, so this container
+            # gets the trust anchor and nothing else. pub/ holds only the CA,
+            # so the machine key is absent from the pod — and because this is a
+            # directory mount rather than a subPath, the agent's atomic
+            # republish of the CA reaches a running pod.
+            - name: nico-certs-pub
+              mountPath: {{ $dir }}/pub
+              readOnly: true
+            # The agent's token socket gets its own writable mount — connect(2)
+            # needs write access to the socket inode, which a read-only mount
+            # (correctly) denies.
+            - name: nico-agent-run
+              mountPath: {{ $dir }}/run
+            {{- else }}
             - name: nico-certs
-              mountPath: {{ .Values.certsDir | default "/opt/nico" }}
+              mountPath: {{ $dir }}
               readOnly: true
+            {{- end }}
       volumes:
+        {{- if .Values.useNodeTokens }}
+        # No credentials-directory volume at all in token mode: nothing in this
+        # pod is entitled to the machine key.
+        - name: nico-certs-pub
+          hostPath:
+            path: {{ .Values.certsDir | default "/opt/nico" }}/pub
+            type: DirectoryOrCreate
+        - name: nico-agent-run
+          hostPath:
+            path: {{ .Values.certsDir | default "/opt/nico" }}/run
+            type: DirectoryOrCreate
+        {{- else }}
         - name: nico-certs
           hostPath:
             path: {{ .Values.certsDir | default "/opt/nico" }}
             type: DirectoryOrCreate
+        {{- end }}
       {{- with .Values.tolerations }}
       tolerations:
         {{- toYaml . | nindent 8 }}

bluefield/charts/nico-fmds/tests/node_tokens_test.yaml

@@ -0,0 +1,156 @@
+suite: node-auth token mode
+templates:
+  - daemonset.yaml
+tests:
+  # The point of token mode is that the machine private key is not reachable
+  # from this pod at all. The key sits beside the root CA in certsDir, and the
+  # container runs as UID 0, so a read-only mount of that directory would not
+  # be enough -- the volume must be absent entirely.
+  - it: should not mount the credentials directory anywhere in token mode
+    set:
+      image:
+        repository: test
+        tag: test
+      useNodeTokens: true
+    asserts:
+      - notContains:
+          path: spec.template.spec.volumes
+          content:
+            name: nico-certs
+            hostPath:
+              path: /opt/forge
+              type: DirectoryOrCreate
+      - notContains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: nico-certs
+            mountPath: /opt/forge
+            readOnly: true
+      - notContains:
+          path: spec.template.spec.initContainers[0].volumeMounts
+          content:
+            name: nico-certs
+            mountPath: /opt/forge
+            readOnly: true
+
+  # pub/ is mounted as a directory, not as a subPath of the CA file: the agent
+  # installs the CA by atomic rename, and a subPath mount would pin the old
+  # inode so a rotation never reached a running pod.
+  - it: should mount only the published CA directory and the agent socket
+    set:
+      image:
+        repository: test
+        tag: test
+      useNodeTokens: true
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: nico-certs-pub
+            mountPath: /opt/forge/pub
+            readOnly: true
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: nico-agent-run
+            mountPath: /opt/forge/run
+      - lengthEqual:
+          path: spec.template.spec.containers[0].volumeMounts
+          count: 2
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: nico-certs-pub
+            hostPath:
+              path: /opt/forge/pub
+              type: DirectoryOrCreate
+
+  - it: should read the CA from pub/ and authenticate with a token, not a cert
+    set:
+      image:
+        repository: test
+        tag: test
+      useNodeTokens: true
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].args
+          content: --root-ca=/opt/forge/pub/forge_root.pem
+      - contains:
+          path: spec.template.spec.containers[0].args
+          content: --node-token-socket=/opt/forge/run/agent.sock
+      - notContains:
+          path: spec.template.spec.containers[0].args
+          content: --client-cert=/opt/forge/machine_cert.pem
+      - notContains:
+          path: spec.template.spec.containers[0].args
+          content: --client-key=/opt/forge/machine_cert.key
+
+  # The init container gates the main one; in token mode it has no business
+  # waiting on the machine cert/key, which may never arrive in this pod.
+  - it: should wait only for the published CA in token mode
+    set:
+      image:
+        repository: test
+        tag: test
+      useNodeTokens: true
+    asserts:
+      - matchRegex:
+          path: spec.template.spec.initContainers[0].command[2]
+          pattern: '/opt/forge/pub/forge_root\.pem'
+      - notMatchRegex:
+          path: spec.template.spec.initContainers[0].command[2]
+          pattern: 'machine_cert\.key'
+      - contains:
+          path: spec.template.spec.initContainers[0].volumeMounts
+          content:
+            name: nico-certs-pub
+            mountPath: /opt/forge/pub
+            readOnly: true
+      - lengthEqual:
+          path: spec.template.spec.initContainers[0].volumeMounts
+          count: 1
+
+  - it: should follow certsDir into the published CA path
+    set:
+      image:
+        repository: test
+        tag: test
+      useNodeTokens: true
+      certsDir: /srv/creds
+      rootCaFile: site_root.pem
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].args
+          content: --root-ca=/srv/creds/pub/site_root.pem
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: nico-certs-pub
+            hostPath:
+              path: /srv/creds/pub
+              type: DirectoryOrCreate
+
+  # mTLS remains the default, and must not acquire the token-mode plumbing.
+  - it: should keep the credentials mount and no pub volume by default
+    set:
+      image:
+        repository: test
+        tag: test
+    asserts:
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: nico-certs
+            hostPath:
+              path: /opt/forge
+              type: DirectoryOrCreate
+      - notContains:
+          path: spec.template.spec.volumes
+          content:
+            name: nico-certs-pub
+            hostPath:
+              path: /opt/forge/pub
+              type: DirectoryOrCreate
+      - notContains:
+          path: spec.template.spec.containers[0].args
+          content: --node-token-socket=/opt/forge/run/agent.sock

bluefield/charts/nico-fmds/values.yaml

@@ -13,6 +13,18 @@ exposedPorts:
 certsDir: /opt/forge
 rootCaFile: forge_root.pem
 
+### Node-auth (issue #355).
+# When true, fmds authenticates to nico-api with short-lived bearer JWTs
+# fetched from the dpu-agent's local API socket (<certsDir>/run/agent.sock)
+# instead of the machine mTLS cert/key. The pod then mounts only
+# <certsDir>/pub — where the agent publishes the trust anchor and nothing
+# else — plus that socket, so the machine private key is absent from it
+# rather than merely read-only (the container runs as UID 0). Mounting pub/
+# as a directory also means the agent's atomic republish of a rotated CA
+# reaches a running pod. Requires a dpu-agent that serves the local API and
+# publishes the CA, and [node_auth] enabled on the API.
+useNodeTokens: false
+
 ### Service specific values ###
 # Prometheus /metrics listen port (must match nico-otelcol scrape target).
 metricsPort: 8888

crates/agent/Cargo.toml

@@ -140,6 +140,7 @@ tonic-prost-build = { workspace = true }
 carbide-instrument = { path = "../instrument", features = ["test-support"] }
 carbide-test-support = { path = "../test-support" }
 ctor = { workspace = true }
+rcgen = { workspace = true }
 prost = { workspace = true }
 rustls-pki-types = { workspace = true }