From 0fe42e8758e01b367335fbcc17a089d3016f0f66 Mon Sep 17 00:00:00 2001 From: Philippe Chepy Date: Tue, 11 Aug 2026 18:51:33 +0200 Subject: [PATCH 1/2] feat(oidc): let a provider authenticate its discovery fetches with a client certificate --- CHANGELOG.md | 1 + doc/how-to/configure-generic-oidc.md | 51 ++++++++++++++++++- doc/reference/configuration.md | 7 +++ src/auth/authenticator.rs | 73 ++++++++++++++++++++++++++-- src/auth/oidc/mod.rs | 8 +++ src/auth/oidc/validator/tests.rs | 4 ++ 6 files changed, 137 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a84e278..af385aa9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - An optional token service exchanges a client's credential for a registry-signed bearer token at `GET /token`, so a short-lived CI credential no longer has to outlive the push it starts. - `auth.oidc..required_claims` rejects a token that does not carry the claims listed, before any access policy runs. - `auth.oidc..server_ca_bundle` trusts a private CA for that provider's discovery and JWKS fetches, so an issuer such as a kube-apiserver needs no host-wide trust. +- `auth.oidc..client_certificate_bundle` and `client_private_key` present a client certificate on those fetches, so a cluster that keeps discovery closed to unauthenticated users can still back image pulls with projected service-account tokens. - The `has_repository_policy()` access-policy function lets a global rule admit only what a `[repository]` declaring its own `access_policy` will decide, instead of restating every repository rule globally. ### Changed diff --git a/doc/how-to/configure-generic-oidc.md b/doc/how-to/configure-generic-oidc.md index 9506ba09..4da0b8fb 100644 --- a/doc/how-to/configure-generic-oidc.md +++ b/doc/how-to/configure-generic-oidc.md @@ -42,6 +42,8 @@ jwks_refresh_interval = 3600 # Refresh keys hourly (default) clock_skew_tolerance = 60 # Allow 60s clock drift (default) allowed_algorithms = ["RS256"] # Restrict accepted JWT algorithms (default) server_ca_bundle = "/certs/ca.pem" # Trust a private CA for this issuer +client_certificate_bundle = "/certs/client.pem" # Authenticate to an issuer that +client_private_key = "/certs/client-key.pem" # refuses anonymous discovery ``` `required_claims` checks presence only. To test a claim's *value*, use an access @@ -103,8 +105,10 @@ required_audience = "api://your-app-id" ### Kubernetes API Server -The cluster CA signs the issuer, so point `server_ca_bundle` at it rather than -trusting that CA for every outbound connection. +A pod pulls with its projected service-account token, which angos validates +against the cluster's JWKS. The cluster CA signs the issuer, so point +`server_ca_bundle` at it rather than trusting that CA for every outbound +connection. ```toml [auth.oidc.kube] @@ -113,6 +117,49 @@ server_ca_bundle = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" required_audience = "angos" ``` +Project the token with that audience so the apiserver mints it for angos: + +```yaml +volumes: + - name: angos-token + projected: + sources: + - serviceAccountToken: + audience: angos + expirationSeconds: 3600 + path: token +``` + +Most clusters answer the `.well-known/openid-configuration` and JWKS fetches +with `401`, because reading discovery takes the +`system:service-account-issuer-discovery` role and no unauthenticated user holds +it. Give angos an identity of its own and bind that role to it alone, rather +than granting it to `system:unauthenticated`, which publishes the cluster's +signing keys and issuer metadata to everyone who can reach the apiserver. + +Issue a client certificate through the `kubernetes.io/kube-apiserver-client` +signer, or from any CA in the apiserver's `--client-ca-file`. Its subject `CN` +becomes the username the apiserver authenticates, and its `O` values the groups, +so the binding names the `CN` you signed: + +```bash +kubectl create clusterrolebinding angos-issuer-discovery \ + --clusterrole=system:service-account-issuer-discovery \ + --user=angos +``` + +```toml +[auth.oidc.kube] +issuer = "https://kubernetes.default.svc" +server_ca_bundle = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" +client_certificate_bundle = "/certs/angos-client.pem" # CN=angos +client_private_key = "/certs/angos-client-key.pem" +required_audience = "angos" +``` + +Configuring one of the two without the other fails startup, so a half-configured +pair cannot degrade into an anonymous fetch. + --- ## Multiple Providers diff --git a/doc/reference/configuration.md b/doc/reference/configuration.md index 24f4b407..a7b052e3 100644 --- a/doc/reference/configuration.md +++ b/doc/reference/configuration.md @@ -433,6 +433,8 @@ tokens are validated, so there is no provider type to select. | `issuer` | string | required | OIDC issuer URL | | `jwks_uri` | string | - | Custom JWKS URI (auto-discovered if not set) | | `server_ca_bundle` | string | - | PEM CA bundle trusted for this provider's HTTPS fetches | +| `client_certificate_bundle` | string | - | PEM client certificate presented on those fetches, requires `client_private_key` | +| `client_private_key` | string | - | PEM key for `client_certificate_bundle` | | `required_claims` | array | `[]` | Claims a token must carry; a missing or null one is rejected | | `jwks_refresh_interval` | u64 | `3600` | JWKS refresh interval (seconds) | | `required_audience` | string | - | Required audience claim | @@ -455,6 +457,11 @@ cover, such as a kube-apiserver signed by the cluster CA. It applies to the discovery and JWKS fetches for that provider alone; other providers keep the system roots. +Set `client_certificate_bundle` and `client_private_key` for an issuer that +refuses an anonymous caller on those endpoints, a kube-apiserver serving +discovery to authenticated users only being the usual case. Configuring one +without the other fails startup rather than fetching anonymously. + `required_claims` checks presence only. Predicates over claim *values* belong in the access policy, which sees the whole claim map. diff --git a/src/auth/authenticator.rs b/src/auth/authenticator.rs index c630da90..7a090f1a 100644 --- a/src/auth/authenticator.rs +++ b/src/auth/authenticator.rs @@ -264,8 +264,9 @@ impl Authenticator { } } -/// One client per provider: a CA bundle is baked into a client when it is built, -/// so a provider trusting its own issuer cannot share one with the others. +/// One client per provider: its TLS material is baked in when it is built, so a +/// provider trusting its own issuer, or authenticating to it, cannot share one +/// with the others. fn build_oidc_client(name: &str, config: &oidc::Config) -> Result, Error> { let initialization_error = |e: String| { Error::Initialization(format!( @@ -273,13 +274,22 @@ fn build_oidc_client(name: &str, config: &oidc::Config) -> Result, E )) }; + // A lone certificate or key would otherwise fetch anonymously and fail as an + // unauthorized issuer at runtime. + if config.client_certificate_bundle.is_some() != config.client_private_key.is_some() { + return Err(initialization_error( + "both client_certificate_bundle and client_private_key are required for mTLS" + .to_string(), + )); + } + // No client-level timeout: each fetch carries a per-request timeout from the // provider config (`http_request_timeout_secs`, `jwks_refresh_timeout_secs`). apply_tls_files( Client::builder(), config.server_ca_bundle.as_deref(), - None, - None, + config.client_certificate_bundle.as_deref(), + config.client_private_key.as_deref(), ) .map_err(initialization_error)? .build() @@ -329,7 +339,7 @@ mod tests { mtls::cert_der, oidc::KID, requests::{empty_parts, parts_with_authorization, parts_with_basic_auth}, - webhook::ca_bundle_pem, + webhook::{ca_bundle_pem, client_cert_pem, client_key_pem}, }, }; @@ -511,6 +521,59 @@ mod tests { assert!(Authenticator::build_oidc_validators(&config.auth, &cache).is_ok()); } + /// An issuer that refuses an anonymous caller, such as a kube-apiserver + /// serving discovery to authenticated users only, is reached with a client + /// certificate. + #[test] + fn a_provider_may_present_a_client_certificate() { + let material = tempdir().unwrap(); + let certificate_path = material.path().join("client.pem"); + let key_path = material.path().join("client-key.pem"); + fs::write(&certificate_path, client_cert_pem()).unwrap(); + fs::write(&key_path, client_key_pem()).unwrap(); + + let config = load_config(&format!( + r#" + [auth.oidc.kube] + issuer = "https://kubernetes.default.svc" + client_certificate_bundle = "{}" + client_private_key = "{}" + "#, + certificate_path.display(), + key_path.display() + )); + + let cache = cache::Config::Memory.to_backend().unwrap(); + + assert!(Authenticator::build_oidc_validators(&config.auth, &cache).is_ok()); + } + + #[test] + fn a_client_certificate_without_its_key_is_refused_at_startup() { + let material = tempdir().unwrap(); + let certificate_path = material.path().join("client.pem"); + fs::write(&certificate_path, client_cert_pem()).unwrap(); + + let config = load_config(&format!( + r#" + [auth.oidc.kube] + issuer = "https://kubernetes.default.svc" + client_certificate_bundle = "{}" + "#, + certificate_path.display() + )); + + let cache = cache::Config::Memory.to_backend().unwrap(); + + let Err(error) = Authenticator::build_oidc_validators(&config.auth, &cache) else { + panic!("half a client identity must be refused rather than fetch anonymously"); + }; + assert!( + matches!(&error, Error::Initialization(msg) if msg.contains("auth.oidc.kube")), + "got: {error:?}" + ); + } + #[test] fn a_ca_bundle_that_does_not_load_is_refused_at_startup() { let config = load_config( diff --git a/src/auth/oidc/mod.rs b/src/auth/oidc/mod.rs index 7a6dafad..124ffa25 100644 --- a/src/auth/oidc/mod.rs +++ b/src/auth/oidc/mod.rs @@ -32,6 +32,14 @@ pub struct Config { /// certificate the system roots do not cover, such as a kube-apiserver. #[serde(default)] pub server_ca_bundle: Option, + /// Client certificate and key presented on those same fetches, for an issuer + /// that refuses an anonymous caller, such as a kube-apiserver whose + /// `system:service-account-issuer-discovery` role no unauthenticated user + /// holds. Both or neither; a lone one is refused at startup. + #[serde(default)] + pub client_certificate_bundle: Option, + #[serde(default)] + pub client_private_key: Option, /// Discovered from the issuer's `.well-known/openid-configuration` when omitted. #[serde(default)] pub jwks_uri: Option, diff --git a/src/auth/oidc/validator/tests.rs b/src/auth/oidc/validator/tests.rs index 6c993034..f07cd3c4 100644 --- a/src/auth/oidc/validator/tests.rs +++ b/src/auth/oidc/validator/tests.rs @@ -29,6 +29,8 @@ use crate::{ pub fn build_test_provider_config(uri: &str) -> Config { Config { server_ca_bundle: None, + client_certificate_bundle: None, + client_private_key: None, issuer: uri.to_string(), jwks_uri: Some(format!("{uri}/.well-known/jwks")), required_claims: Vec::new(), @@ -768,6 +770,8 @@ pub fn valid_claims(issuer: &str, audience: &str) -> HashMap) -> Config { Config { server_ca_bundle: None, + client_certificate_bundle: None, + client_private_key: None, issuer: issuer.to_string(), jwks_uri: None, required_claims: Vec::new(), From 373842b4e4cfe419432a01f883cff1cd878eee4f Mon Sep 17 00:00:00 2001 From: Philippe Chepy Date: Tue, 11 Aug 2026 21:05:01 +0200 Subject: [PATCH 2/2] feat(kubelet): add an experimental credential provider --- .github/workflows/build.yaml | 33 ++ CHANGELOG.md | 1 + Cargo.lock | 10 + Cargo.toml | 1 + Dockerfile | 1 + .../kubelet-credential-provider/Cargo.toml | 17 + contrib/kubelet-credential-provider/README.md | 47 +++ .../daemonset.yaml | 132 +++++++ .../kubelet-credential-provider/src/main.rs | 361 ++++++++++++++++++ doc/how-to/configure-generic-oidc.md | 58 +-- doc/how-to/configure-kubernetes-oidc.md | 176 +++++++++ src/auth/oidc/mod.rs | 9 +- website/sidebars.ts | 1 + 13 files changed, 787 insertions(+), 60 deletions(-) create mode 100644 contrib/kubelet-credential-provider/Cargo.toml create mode 100644 contrib/kubelet-credential-provider/README.md create mode 100644 contrib/kubelet-credential-provider/daemonset.yaml create mode 100644 contrib/kubelet-credential-provider/src/main.rs create mode 100644 doc/how-to/configure-kubernetes-oidc.md diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index dfd6b5b6..fd6a7b86 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -593,9 +593,11 @@ jobs: - os: ubuntu-latest target: x86_64-unknown-linux-musl binary_name: angos-linux-amd64 + provider_binary_name: angos-credential-provider-linux-amd64 - os: ubuntu-24.04-arm target: aarch64-unknown-linux-musl binary_name: angos-linux-arm64 + provider_binary_name: angos-credential-provider-linux-arm64 runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v7 @@ -630,6 +632,26 @@ jobs: if-no-files-found: error retention-days: 1 + # The kubelet credential provider is installed on nodes, it ships for + # Linux alone. + - name: Build kubelet credential provider + if: contains(matrix.target, 'linux') + env: + CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_RUSTFLAGS: "-Clink-self-contained=yes -Clinker=rust-lld" + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS: "-Clink-self-contained=yes -Clinker=rust-lld" + run: | + cargo build --release --target ${{ matrix.target }} -p kubelet-credential-provider + mv target/${{ matrix.target }}/release/angos-credential-provider ${{ matrix.provider_binary_name }} + + - name: Upload kubelet credential provider + if: contains(matrix.target, 'linux') + uses: actions/upload-artifact@v7 + with: + name: binary-${{ matrix.provider_binary_name }} + path: ${{ matrix.provider_binary_name }} + if-no-files-found: error + retention-days: 1 + build-container: runs-on: ubuntu-latest strategy: @@ -793,6 +815,17 @@ jobs: | Linux amd64 | `angos-linux-amd64` | | Linux arm64 | `angos-linux-arm64` | + ## Kubelet Credential Provider + + Installed on nodes so image pulls authenticate with the pulling pod's + service-account token. `contrib/kubelet-credential-provider` carries a + DaemonSet that installs it from this release. + + | Platform | Binary | + |-------------|-------------------------------------------| + | Linux amd64 | `angos-credential-provider-linux-amd64` | + | Linux arm64 | `angos-credential-provider-linux-arm64` | + ## Verification with Cosign All release artifacts are signed with [Sigstore Cosign](https://docs.sigstore.dev/) diff --git a/CHANGELOG.md b/CHANGELOG.md index af385aa9..578529a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - `auth.oidc..required_claims` rejects a token that does not carry the claims listed, before any access policy runs. - `auth.oidc..server_ca_bundle` trusts a private CA for that provider's discovery and JWKS fetches, so an issuer such as a kube-apiserver needs no host-wide trust. - `auth.oidc..client_certificate_bundle` and `client_private_key` present a client certificate on those fetches, so a cluster that keeps discovery closed to unauthenticated users can still back image pulls with projected service-account tokens. +- EXPERIMENTAL: `contrib/kubelet-credential-provider` hands the kubelet the pulling pod's service-account token as its registry credential, so an image pull authenticates as the workload instead of a shared `imagePullSecret`. It ships as a released Linux binary, with a DaemonSet that installs it on every node and restarts the kubelet only when the binary changes. - The `has_repository_policy()` access-policy function lets a global rule admit only what a `[repository]` declaring its own `access_policy` will decide, instead of restating every repository rule globally. ### Changed diff --git a/Cargo.lock b/Cargo.lock index a863ff54..1489d0a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1521,6 +1521,16 @@ dependencies = [ "libc", ] +[[package]] +name = "kubelet-credential-provider" +version = "0.1.0" +dependencies = [ + "argh", + "base64", + "serde", + "serde_json", +] + [[package]] name = "lazy_static" version = "1.5.0" diff --git a/Cargo.toml b/Cargo.toml index eac7bf81..0a37471b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = [ + "contrib/kubelet-credential-provider", "crates/backoff", "crates/conformance-gates", "crates/s3-client", diff --git a/Dockerfile b/Dockerfile index 248eba5c..8d6fc14b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,6 +24,7 @@ RUN if [ "$TARGETARCH" = "amd64" ] ; then export TOOLCHAIN="x86_64-unknown-linux COPY Cargo.toml Cargo.lock build.rs ./ COPY crates ./crates +COPY contrib/kubelet-credential-provider ./contrib/kubelet-credential-provider COPY src ./src COPY ui ./ui diff --git a/contrib/kubelet-credential-provider/Cargo.toml b/contrib/kubelet-credential-provider/Cargo.toml new file mode 100644 index 00000000..f9f0c509 --- /dev/null +++ b/contrib/kubelet-credential-provider/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "kubelet-credential-provider" +version = "0.1.0" +edition = "2024" + +[lints.clippy] +pedantic = { level = "warn", priority = -1 } + +[[bin]] +name = "angos-credential-provider" +path = "src/main.rs" + +[dependencies] +argh = { workspace = true } +base64 = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } diff --git a/contrib/kubelet-credential-provider/README.md b/contrib/kubelet-credential-provider/README.md new file mode 100644 index 00000000..22a4648e --- /dev/null +++ b/contrib/kubelet-credential-provider/README.md @@ -0,0 +1,47 @@ +# angos kubelet credential provider + +Lets the kubelet pull images with the service-account token of the pod it is +starting, instead of a static `imagePullSecret`. + +The kubelet mints the token, execs this plugin with it on stdin, and gets back +the registry credential angos accepts: the `[auth.oidc.]` section name as +the username, the token as the password. + +It builds under the workspace release profile the registry itself uses, so +`cargo build --release -p kubelet-credential-provider` already applies fat LTO +and `opt-level = 3`; add `--target x86_64-unknown-linux-musl` for the static +binary the nodes want. Install and configure it as described in +[Configure Kubernetes OIDC](../../doc/how-to/configure-kubernetes-oidc.md#pulling-images-with-the-kubelet). + +## Installing on every node + +`daemonset.yaml` installs the released binary on each node and restarts the +kubelet, which resolves its configured plugins once at startup and so ignores a +binary that appeared later. Set `VERSION` and both checksums first: + +```bash +VERSION=v1.5.0 +for arch in amd64 arm64; do + curl -sL "https://github.com/project-angos/angos/releases/download/${VERSION}/angos-credential-provider-linux-${arch}" \ + | sha256sum +done +kubectl apply -f daemonset.yaml +``` + +Re-running costs one checksum: the installer compares what the node already has +against the pinned value and exits without downloading or restarting anything +when they match. A download that fails verification leaves the node's existing +binary in place, and the kubelet is restarted only after a new binary lands. + +The installer does not touch kubelet flags. Point the kubelet at the plugin with +`--image-credential-provider-config` and `--image-credential-provider-bin-dir` +through whatever bootstraps your nodes: rewriting a unit file from a DaemonSet +would fight the distribution that owns it. Note also that a first rollout +restarts the kubelet on every node at once, so apply it to a `nodeSelector` +subset first if that matters to you. + +The exchange follows `k8s.io/kubelet/pkg/apis/credentialprovider/v1`, and the +`tokenAttributes` that make the kubelet mint the token are defined by +`CredentialProvider` in `k8s.io/kubelet/config/v1`. Both are published from +`staging/src/k8s.io/kubelet` in kubernetes/kubernetes and rendered at +[kubernetes.io/docs/reference/config-api/kubelet-credentialprovider.v1](https://kubernetes.io/docs/reference/config-api/kubelet-credentialprovider.v1/). diff --git a/contrib/kubelet-credential-provider/daemonset.yaml b/contrib/kubelet-credential-provider/daemonset.yaml new file mode 100644 index 00000000..1c562edd --- /dev/null +++ b/contrib/kubelet-credential-provider/daemonset.yaml @@ -0,0 +1,132 @@ +# Installs the angos kubelet credential provider on every node from a GitHub +# release, then restarts the kubelet so it picks the plugin up. +# +# Idempotent: the installer compares the checksum already on the node with the +# one configured here and exits without downloading or restarting anything when +# they match, so a rescheduled pod or a node reboot costs one checksum. +# +# Set VERSION and both checksums before applying. The kubelet flags +# (--image-credential-provider-config, --image-credential-provider-bin-dir) stay +# a node-bootstrap concern; this only keeps the binary current. +apiVersion: v1 +kind: ConfigMap +metadata: + name: angos-credential-provider-installer + namespace: kube-system +data: + install.sh: | + #!/bin/sh + set -eu + + case "$(uname -m)" in + x86_64) asset="angos-credential-provider-linux-amd64"; expected="$SHA256_AMD64" ;; + aarch64) asset="angos-credential-provider-linux-arm64"; expected="$SHA256_ARM64" ;; + *) echo "unsupported architecture $(uname -m)" >&2; exit 1 ;; + esac + + if [ -z "$expected" ]; then + echo "no checksum configured for $asset; refusing to install an unverified binary" >&2 + exit 1 + fi + + target="/host${BIN_DIR}/angos-credential-provider" + installed="" + if [ -f "$target" ]; then + installed="$(sha256sum "$target" | cut -d' ' -f1)" + fi + + if [ "$installed" = "$expected" ]; then + echo "$target is already ${VERSION}; leaving the kubelet alone" + exit 0 + fi + + work="$(mktemp -d)" + trap 'rm -rf "$work"' EXIT + wget -qO "$work/$asset" \ + "https://github.com/project-angos/angos/releases/download/${VERSION}/${asset}" + echo "$expected $asset" | (cd "$work" && sha256sum -c -) + + # Install beside the target and rename: a pull racing the write never sees a + # half-written plugin. + mkdir -p "/host${BIN_DIR}" + install -m 0755 "$work/$asset" "$target.new" + mv "$target.new" "$target" + + # The kubelet resolves the configured plugins once at startup, so a binary + # that appeared or changed since then is only used after a restart. + chroot /host systemctl restart kubelet + echo "installed ${VERSION} and restarted the kubelet" +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: angos-credential-provider-installer + namespace: kube-system + labels: + app.kubernetes.io/name: angos-credential-provider +spec: + selector: + matchLabels: + app.kubernetes.io/name: angos-credential-provider + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + app.kubernetes.io/name: angos-credential-provider + spec: + priorityClassName: system-node-critical + automountServiceAccountToken: false + tolerations: + - operator: Exists + initContainers: + - name: install + image: alpine:3.22 + command: ["/bin/sh", "/script/install.sh"] + env: + - name: VERSION + value: v1.5.0 + - name: SHA256_AMD64 + value: "" + - name: SHA256_ARM64 + value: "" + - name: BIN_DIR + value: /var/lib/kubelet/credential-providers + securityContext: + # Writes into the kubelet's bin dir and restarts the host unit. + privileged: true + volumeMounts: + - name: host + mountPath: /host + - name: script + mountPath: /script + readOnly: true + containers: + # The install runs to completion in the init container; this only keeps + # the pod scheduled, so it holds no host access at all. + - name: idle + image: alpine:3.22 + command: ["/bin/sh", "-c", "trap 'exit 0' TERM; sleep infinity & wait"] + resources: + requests: + cpu: 1m + memory: 8Mi + limits: + memory: 16Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65534 + capabilities: + drop: ["ALL"] + volumes: + - name: host + hostPath: + path: / + - name: script + configMap: + name: angos-credential-provider-installer + defaultMode: 0755 diff --git a/contrib/kubelet-credential-provider/src/main.rs b/contrib/kubelet-credential-provider/src/main.rs new file mode 100644 index 00000000..50881242 --- /dev/null +++ b/contrib/kubelet-credential-provider/src/main.rs @@ -0,0 +1,361 @@ +//! Kubelet credential provider for angos. +//! +//! The kubelet mints a service-account token for the pod being started and +//! hands it to this plugin on stdin; the plugin returns it as the registry +//! password under the `[auth.oidc.]` section name, which is how angos +//! reads a Basic credential as that provider's token. +//! +//! The exchange below mirrors `k8s.io/kubelet/pkg/apis/credentialprovider/v1`, +//! and the `tokenAttributes` that make the kubelet send a token belong to +//! `CredentialProvider` in `k8s.io/kubelet/config/v1`. + +use std::{ + collections::HashMap, + io::{Read, Write, stdin, stdout}, + process::ExitCode, + time::{SystemTime, UNIX_EPOCH}, +}; + +use argh::FromArgs; +use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; +use serde::{Deserialize, Serialize}; + +/// Cut off the cached credential before the token it carries expires, so a pull +/// starting on the last cache hit still has a valid token to present. +const EXPIRY_MARGIN_SECS: u64 = 60; + +/// Answer one kubelet credential-provider exchange on stdin and stdout. +#[derive(FromArgs)] +struct Arguments { + /// name of the `[auth.oidc.]` section that validates the token, sent + /// as the registry username + #[argh(option)] + provider: String, + + /// registry host the token may be handed to, refusing any other one a + /// broader `matchImages` would otherwise route here. Repeat it for every + /// host angos serves, including one a containerd mirror redirects here + /// under another name; given none, `matchImages` alone decides + #[argh(option)] + registry: Vec, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct CredentialProviderRequest { + api_version: String, + image: String, + /// Present only when the provider entry sets `tokenAttributes`. + #[serde(default)] + service_account_token: Option, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct CredentialProviderResponse { + api_version: String, + kind: &'static str, + cache_key_type: &'static str, + cache_duration: String, + auth: HashMap, +} + +#[derive(Serialize)] +struct AuthConfig { + username: String, + password: String, +} + +/// The one claim this plugin reads; the registry checks the rest. +#[derive(Deserialize)] +struct Claims { + exp: u64, +} + +fn main() -> ExitCode { + let arguments: Arguments = argh::from_env(); + + match run(&arguments) { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + eprintln!("angos-credential-provider: {error}"); + ExitCode::FAILURE + } + } +} + +fn run(arguments: &Arguments) -> Result<(), String> { + let mut body = String::new(); + stdin() + .read_to_string(&mut body) + .map_err(|e| format!("failed to read the request: {e}"))?; + + let request: CredentialProviderRequest = + serde_json::from_str(&body).map_err(|e| format!("failed to parse the request: {e}"))?; + let response = respond(arguments, &request, now_seconds())?; + let response = serde_json::to_vec(&response) + .map_err(|e| format!("failed to serialize the response: {e}"))?; + + stdout() + .write_all(&response) + .map_err(|e| format!("failed to write the response: {e}")) +} + +fn respond( + arguments: &Arguments, + request: &CredentialProviderRequest, + now: u64, +) -> Result { + // The token is a bearer credential the holder can replay against the + // registry, so a listed host is the one place it may go and a `matchImages` + // wider than that fails the pull instead. Listing none leaves that call to + // `matchImages`. + let registry = registry_of(&request.image); + if !arguments.registry.is_empty() + && !arguments + .registry + .iter() + .any(|served| served.eq_ignore_ascii_case(registry)) + { + return Err(format!( + "refusing to hand the token to '{registry}': this provider serves {}", + arguments.registry.join(", ") + )); + } + + let Some(token) = &request.service_account_token else { + return Err(format!( + "the kubelet sent no service-account token for '{}': set \ + tokenAttributes.serviceAccountTokenAudience on this provider entry", + request.image + )); + }; + + let credential = AuthConfig { + username: arguments.provider.clone(), + password: token.clone(), + }; + + Ok(CredentialProviderResponse { + api_version: request.api_version.clone(), + kind: "CredentialProviderResponse", + cache_key_type: "Registry", + cache_duration: cache_duration(token, now), + auth: HashMap::from([(registry.to_string(), credential)]), + }) +} + +/// How long the kubelet may reuse the credential, which is the token's own +/// remaining life less [`EXPIRY_MARGIN_SECS`]: the pass-through credential is +/// worth nothing past its `exp`. A token whose expiry cannot be read caches for +/// no time at all, the zero KEP-4412 defines for plugins returning a token as-is. +fn cache_duration(token: &str, now: u64) -> String { + let seconds = expiry_of(token) + .unwrap_or(0) + .saturating_sub(now.saturating_add(EXPIRY_MARGIN_SECS)); + format!("{seconds}s") +} + +/// Read from the token's payload without verifying its signature: the registry +/// is what validates the token, and a forged `exp` only shortens or forfeits +/// this cache entry. +fn expiry_of(token: &str) -> Option { + let payload = token.split('.').nth(1)?; + let payload = URL_SAFE_NO_PAD.decode(payload).ok()?; + let claims: Claims = serde_json::from_slice(&payload).ok()?; + Some(claims.exp) +} + +/// Seconds since the epoch, or `u64::MAX` when the clock is unreadable, so a +/// clock angos cannot trust disables caching instead of extending it. +fn now_seconds() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(u64::MAX, |since_epoch| since_epoch.as_secs()) +} + +/// The registry host the kubelet matched, which keys the returned credential. +fn registry_of(image: &str) -> &str { + match image.split_once('/') { + Some((registry, _)) => registry, + None => image, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const NOW: u64 = 1_800_000_000; + + fn arguments() -> Arguments { + Arguments { + provider: "kube".to_string(), + registry: vec!["registry.example.com:5000".to_string()], + } + } + + fn request(token: Option<&str>) -> CredentialProviderRequest { + CredentialProviderRequest { + api_version: "credentialprovider.kubelet.k8s.io/v1".to_string(), + image: "registry.example.com:5000/team/app:1.0".to_string(), + service_account_token: token.map(str::to_string), + } + } + + fn token_expiring_at(exp: u64) -> String { + let payload = URL_SAFE_NO_PAD.encode(format!(r#"{{"exp":{exp}}}"#)); + format!("header.{payload}.signature") + } + + #[test] + fn the_token_is_returned_under_the_provider_name() { + let token = token_expiring_at(NOW + 3600); + let response = respond(&arguments(), &request(Some(&token)), NOW).unwrap(); + + assert_eq!(response.api_version, "credentialprovider.kubelet.k8s.io/v1"); + let credential = &response.auth["registry.example.com:5000"]; + assert_eq!(credential.username, "kube"); + assert_eq!(credential.password, token); + } + + /// The kubelet may reuse the credential for the token's remaining life, so + /// a pod pulling several images runs the plugin once. + #[test] + fn the_credential_is_cached_until_the_token_nears_expiry() { + let response = respond( + &arguments(), + &request(Some(&token_expiring_at(NOW + 3600))), + NOW, + ) + .unwrap(); + + assert_eq!(response.cache_duration, "3540s"); + } + + /// Anything that leaves the expiry unknown or already reached must not be + /// held: an expired credential fails the pull it was kept for. + #[test] + fn a_token_that_is_spent_or_unreadable_is_not_cached() { + for token in [ + token_expiring_at(NOW + EXPIRY_MARGIN_SECS), + token_expiring_at(NOW - 1), + "not.a.jwt".to_string(), + "opaque".to_string(), + ] { + let response = respond(&arguments(), &request(Some(&token)), NOW).unwrap(); + + assert_eq!(response.cache_duration, "0s", "token: {token}"); + } + } + + /// A clock the plugin cannot read reports `u64::MAX`, which must shorten the + /// cache rather than overflow into a long one. + #[test] + fn an_unreadable_clock_disables_caching() { + let response = respond( + &arguments(), + &request(Some(&token_expiring_at(NOW + 3600))), + u64::MAX, + ) + .unwrap(); + + assert_eq!(response.cache_duration, "0s"); + } + + #[test] + fn a_request_without_a_token_names_the_missing_setting() { + let Err(error) = respond(&arguments(), &request(None), NOW) else { + panic!("a request carrying no token must not yield a credential"); + }; + + assert!( + error.contains("tokenAttributes.serviceAccountTokenAudience"), + "got: {error}" + ); + } + + /// A `matchImages` wider than this provider must cost a failed pull, never + /// the token: whoever receives it can replay it against the registry. + #[test] + fn a_request_for_another_registry_is_refused() { + let token = token_expiring_at(NOW + 3600); + let elsewhere = CredentialProviderRequest { + image: "evil.example.com/team/app:1.0".to_string(), + ..request(Some(&token)) + }; + + let Err(error) = respond(&arguments(), &elsewhere, NOW) else { + panic!("a registry this provider does not serve must not receive the token"); + }; + + assert!(error.contains("evil.example.com"), "got: {error}"); + assert!( + !error.contains(&token), + "the error leaked the token: {error}" + ); + } + + /// A containerd mirror pulls an image whose reference names the mirrored + /// registry, so every host angos answers for is served. + #[test] + fn each_configured_registry_is_served() { + let arguments = Arguments { + provider: "kube".to_string(), + registry: vec![ + "registry.example.com:5000".to_string(), + "docker.io".to_string(), + ], + }; + let token = token_expiring_at(NOW + 3600); + let mirrored = CredentialProviderRequest { + image: "docker.io/library/nginx:1.29".to_string(), + ..request(Some(&token)) + }; + + let response = respond(&arguments, &mirrored, NOW).unwrap(); + + assert_eq!(response.auth["docker.io"].password, token); + assert!( + respond(&arguments, &request(Some(&token)), NOW).is_ok(), + "the other configured registry must keep working" + ); + } + + /// Listing no registry defers to `matchImages`, which is what routed the + /// request here in the first place. + #[test] + fn without_a_configured_registry_every_routed_host_is_served() { + let arguments = Arguments { + provider: "kube".to_string(), + registry: Vec::new(), + }; + let token = token_expiring_at(NOW + 3600); + let elsewhere = CredentialProviderRequest { + image: "mirror.example.com/team/app:1.0".to_string(), + ..request(Some(&token)) + }; + + let response = respond(&arguments, &elsewhere, NOW).unwrap(); + + assert_eq!(response.auth["mirror.example.com"].password, token); + } + + /// Hosts are compared as DNS names, so casing in the pod spec is not a + /// mismatch. + #[test] + fn the_registry_host_matches_case_insensitively() { + let token = token_expiring_at(NOW + 3600); + let shouted = CredentialProviderRequest { + image: "Registry.Example.COM:5000/team/app:1.0".to_string(), + ..request(Some(&token)) + }; + + assert!(respond(&arguments(), &shouted, NOW).is_ok()); + } + + #[test] + fn an_image_without_a_path_is_its_own_registry() { + assert_eq!(registry_of("registry.example.com"), "registry.example.com"); + } +} diff --git a/doc/how-to/configure-generic-oidc.md b/doc/how-to/configure-generic-oidc.md index 4da0b8fb..4286d5ac 100644 --- a/doc/how-to/configure-generic-oidc.md +++ b/doc/how-to/configure-generic-oidc.md @@ -105,60 +105,9 @@ required_audience = "api://your-app-id" ### Kubernetes API Server -A pod pulls with its projected service-account token, which angos validates -against the cluster's JWKS. The cluster CA signs the issuer, so point -`server_ca_bundle` at it rather than trusting that CA for every outbound -connection. - -```toml -[auth.oidc.kube] -issuer = "https://kubernetes.default.svc" -server_ca_bundle = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" -required_audience = "angos" -``` - -Project the token with that audience so the apiserver mints it for angos: - -```yaml -volumes: - - name: angos-token - projected: - sources: - - serviceAccountToken: - audience: angos - expirationSeconds: 3600 - path: token -``` - -Most clusters answer the `.well-known/openid-configuration` and JWKS fetches -with `401`, because reading discovery takes the -`system:service-account-issuer-discovery` role and no unauthenticated user holds -it. Give angos an identity of its own and bind that role to it alone, rather -than granting it to `system:unauthenticated`, which publishes the cluster's -signing keys and issuer metadata to everyone who can reach the apiserver. - -Issue a client certificate through the `kubernetes.io/kube-apiserver-client` -signer, or from any CA in the apiserver's `--client-ca-file`. Its subject `CN` -becomes the username the apiserver authenticates, and its `O` values the groups, -so the binding names the `CN` you signed: - -```bash -kubectl create clusterrolebinding angos-issuer-discovery \ - --clusterrole=system:service-account-issuer-discovery \ - --user=angos -``` - -```toml -[auth.oidc.kube] -issuer = "https://kubernetes.default.svc" -server_ca_bundle = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" -client_certificate_bundle = "/certs/angos-client.pem" # CN=angos -client_private_key = "/certs/angos-client-key.pem" -required_audience = "angos" -``` - -Configuring one of the two without the other fails startup, so a half-configured -pair cannot degrade into an anonymous fetch. +Validating a cluster's service-account tokens takes a private CA and an +authenticated discovery fetch, and pulls take a kubelet plugin on top: +[Configure Kubernetes OIDC](configure-kubernetes-oidc.md). --- @@ -314,3 +263,4 @@ Subject: user@example.com - [Set Up Access Control](set-up-access-control.md) for comprehensive policies - [Configure GitHub Actions OIDC](configure-github-actions-oidc.md) for CI/CD +- [Configure Kubernetes OIDC](configure-kubernetes-oidc.md) for service-account tokens and image pulls diff --git a/doc/how-to/configure-kubernetes-oidc.md b/doc/how-to/configure-kubernetes-oidc.md new file mode 100644 index 00000000..6e385fb2 --- /dev/null +++ b/doc/how-to/configure-kubernetes-oidc.md @@ -0,0 +1,176 @@ +--- +displayed_sidebar: howto +sidebar_position: 6 +title: "Kubernetes OIDC" +--- + +# Configure Kubernetes OIDC + +Let workloads authenticate with the projected service-account token the cluster +already mints for them, instead of a static `imagePullSecret`. A pod reaching +angos on its own does so with the token in its filesystem; a pull, which the +kubelet performs before that filesystem exists, needs a credential provider +plugin. + +## Prerequisites + +- Angos running with network access to the apiserver +- For image pulls, nodes whose kubelet flags you can set + +## Trust the Cluster's Issuer + +A pod pulls with its projected service-account token, which angos validates +against the cluster's JWKS. The cluster CA signs the issuer, so point +`server_ca_bundle` at it rather than trusting that CA for every outbound +connection. + +```toml +[auth.oidc.kube] +issuer = "https://kubernetes.default.svc" +server_ca_bundle = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" +required_audience = "angos" +``` + +Project the token with that audience so the apiserver mints it for angos: + +```yaml +volumes: + - name: angos-token + projected: + sources: + - serviceAccountToken: + audience: angos + expirationSeconds: 3600 + path: token +``` + +Most clusters answer the `.well-known/openid-configuration` and JWKS fetches +with `401`, because reading discovery takes the +`system:service-account-issuer-discovery` role and no unauthenticated user holds +it. Give angos an identity of its own and bind that role to it alone, rather +than granting it to `system:unauthenticated`, which publishes the cluster's +signing keys and issuer metadata to everyone who can reach the apiserver. + +Issue a client certificate through the `kubernetes.io/kube-apiserver-client` +signer, or from any CA in the apiserver's `--client-ca-file`. Its subject `CN` +becomes the username the apiserver authenticates, and its `O` values the groups, +so the binding names the `CN` you signed: + +```bash +kubectl create clusterrolebinding angos-issuer-discovery \ + --clusterrole=system:service-account-issuer-discovery \ + --user=angos +``` + +```toml +[auth.oidc.kube] +issuer = "https://kubernetes.default.svc" +server_ca_bundle = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" +client_certificate_bundle = "/certs/angos-client.pem" # CN=angos +client_private_key = "/certs/angos-client-key.pem" +required_audience = "angos" +``` + +Configuring one of the two without the other fails startup, so a half-configured +pair cannot degrade into an anonymous fetch. + +--- + +## Pulling Images with the Kubelet + +:::warning The credential provider is experimental +`contrib/kubelet-credential-provider` is new: its flags and its behaviour may +change in any release, and the kubelet API it speaks is itself still beta. Keep +a static `imagePullSecret` you can fall back to. +::: + +A projected token lives in the pod's filesystem, which the image pull predates, +so the kubelet never sends one to a registry on its own. It obtains registry +credentials from a credential provider plugin instead, and +`contrib/kubelet-credential-provider` is the one that speaks angos: the kubelet +mints a token for the pod's service account and the plugin returns it as the +password under the provider name. + +Build it and install the binary on every node, under the name the kubelet will +look up. Target musl as the registry image does, so one statically linked binary +runs whatever distribution the nodes are on: + +```bash +rustup target add x86_64-unknown-linux-musl +cargo build --release --target x86_64-unknown-linux-musl -p kubelet-credential-provider +install -m 0755 \ + target/x86_64-unknown-linux-musl/release/angos-credential-provider \ + /var/lib/kubelet/credential-providers/ +``` + +Point the kubelet at it with `--image-credential-provider-config` and +`--image-credential-provider-bin-dir`, and declare the audience it should mint +tokens for: + +```yaml +apiVersion: kubelet.config.k8s.io/v1 +kind: CredentialProviderConfig +providers: + - name: angos-credential-provider # matches the binary name in the bin dir + matchImages: ["registry.example.com"] + defaultCacheDuration: "0s" + apiVersion: credentialprovider.kubelet.k8s.io/v1 + # --provider is the [auth.oidc.kube] section name; --registry lists the hosts + # the plugin will hand a token to, whatever matchImages routes here. + args: ["--provider", "kube", "--registry", "registry.example.com"] + tokenAttributes: + serviceAccountTokenAudience: angos # matches required_audience + cacheType: Token # the credential is the token + requireServiceAccount: true +``` + +`tokenAttributes` needs 1.33 or later, where it is alpha behind the +`KubeletServiceAccountTokenForCredentialProviders` gate; 1.34 promotes it to +beta and enables it by default. Without it the kubelet sends no token and the +plugin exits with a message saying so. Nodes whose kubelet flags you cannot +set, as on most managed control planes, cannot run a credential provider at +all. + +Each pull then authenticates as the workload rather than as a shared secret, so +policies name the service account: + +```toml +[repository."team".access_policy] +default = "deny" +rules = [ + '''identity.oidc != null && + identity.oidc.provider_name == "kube" && + identity.oidc.claims["sub"].startsWith("system:serviceaccount:production:")''' +] +``` + +`cacheType: Token` says the credential is the token itself rather than something +derived from the account, so the kubelet keys its cache by token instead of by +service account. The plugin reads the token's `exp` and caches for the life it +has left, one minute short of it, which spares a pod pulling several images a +plugin run per image; a token whose expiry it cannot read is never cached. A +request for any host `--registry` does not list fails the pull rather than +disclosing the token, since whoever receives one can replay it here. + +Repeat `--registry` for every host whose pulls angos answers, and widen +`matchImages` to match. A containerd mirror keeps the image's own name, so a pod +pulling `docker.io/library/nginx` from an angos mirror asks for credentials +under `docker.io`: + +```yaml + matchImages: ["registry.example.com", "docker.io"] + args: ["--provider", "kube", "--registry", "registry.example.com", "--registry", "docker.io"] +``` + +`matchImages` decides which pulls reach the plugin; `--registry` decides which +ones leave with a token. Passing none leaves the decision to `matchImages` +alone, which is enough while that list names only hosts angos serves, and stops +being enough the moment someone widens it. +`cacheType` is required from the 1.34 beta onward; drop it on a cluster still +running the 1.33 alpha. + +## Next Steps + +- [Configure OIDC](configure-generic-oidc.md) for the options every provider shares +- [Deploy on Kubernetes](deploy-kubernetes.md) +- [Set Up Access Control](set-up-access-control.md) for comprehensive policies diff --git a/src/auth/oidc/mod.rs b/src/auth/oidc/mod.rs index 124ffa25..25e30580 100644 --- a/src/auth/oidc/mod.rs +++ b/src/auth/oidc/mod.rs @@ -28,14 +28,11 @@ use crate::{ #[derive(Clone, Debug, Deserialize)] pub struct Config { pub issuer: String, - /// CA bundle trusted for the discovery and JWKS fetches, for an issuer whose - /// certificate the system roots do not cover, such as a kube-apiserver. + /// CA bundle trusted for the discovery and JWKS fetches. #[serde(default)] pub server_ca_bundle: Option, - /// Client certificate and key presented on those same fetches, for an issuer - /// that refuses an anonymous caller, such as a kube-apiserver whose - /// `system:service-account-issuer-discovery` role no unauthenticated user - /// holds. Both or neither; a lone one is refused at startup. + /// Client certificate and key presented on those same fetches. + /// Both or neither; a lone one is refused at startup. #[serde(default)] pub client_certificate_bundle: Option, #[serde(default)] diff --git a/website/sidebars.ts b/website/sidebars.ts index b54f7217..4187a140 100644 --- a/website/sidebars.ts +++ b/website/sidebars.ts @@ -24,6 +24,7 @@ const sidebars: SidebarsConfig = { 'how-to/configure-mtls', 'how-to/configure-github-actions-oidc', 'how-to/configure-generic-oidc', + 'how-to/configure-kubernetes-oidc', 'how-to/set-up-access-control', 'how-to/configure-retention-policies', 'how-to/protect-tags-immutability',