From 9a175f92aa3b797e4f8f8d1c7304fd7eb1103476 Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 14:05:17 +1100 Subject: [PATCH 1/2] fix(tls): run every listener on scalo ServerTls Each listener's tls block loads into scalo's ServerTls. This covers /ingest, webhook, Splunk HEC, remote write, OTLP HTTP and gRPC, Vector gRPC, syslog, Fluent Forward, GELF and Lumberjack. The next handshake uses the rotated certificate, key or client CA without a restart. Before, only a secret-backed certificate reloaded, and only on five listeners. - The sync and async acceptor builders, the RwLock certificate provider, the tonic identity loader and five copies of the handshake-with-timeout block are gone. - The receiver handles the handshake instead of tonic. Failed gRPC handshakes count towards receiver_tls_handshake_failures_total. The connection cap applies before the handshake. - Config loading rejects a certificate, key or CA set as both a file and a secret. It used to prefer the secret. - scalo is pinned to the commit that adds ServerTls until a release has it, so deny.toml allows that git source. Closes #270 --- Cargo.lock | 192 ++++++++- Cargo.toml | 14 +- config.example.yaml | 8 +- deny.toml | 2 + docs/config-schema.json | 14 +- docs/config-schema.yaml | 19 +- docs/hardening.md | 2 +- docs/internals.md | 2 +- docs/metrics.md | 1 + src/config/mod.rs | 83 +++- src/main.rs | 2 +- src/server/fluent/mod.rs | 42 +- src/server/gelf/mod.rs | 42 +- src/server/grpc/mod.rs | 140 ++++-- src/server/http/mod.rs | 88 +--- src/server/lumberjack/mod.rs | 43 +- src/server/otlp/mod.rs | 83 ++-- src/server/prometheus_rw/mod.rs | 34 +- src/server/splunk_hec/mod.rs | 34 +- src/server/syslog/mod.rs | 21 +- src/server/syslog/tcp.rs | 26 +- src/server/tls.rs | 739 +++++++++----------------------- src/server/webhook/mod.rs | 44 +- tests/integration.rs | 2 + tests/integration/tls_reload.rs | 552 ++++++++++++++++++++++++ tests/integration/vault_auth.rs | 51 ++- 26 files changed, 1350 insertions(+), 930 deletions(-) create mode 100644 tests/integration/tls_reload.rs diff --git a/Cargo.lock b/Cargo.lock index c68c5da..ba12aed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -200,6 +200,45 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom 7.1.3", + "num-traits", + "rusticata-macros", + "thiserror 2.0.21", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure 0.13.2", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "async-trait" version = "0.1.92" @@ -282,6 +321,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "048006790404103c488e6c5616b3e7e702a71292aadda08e8f7698a9e0d2eb53" dependencies = [ "aws-lc-sys", + "untrusted 0.7.1", "zeroize", ] @@ -749,6 +789,15 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] + [[package]] name = "bit-vec" version = "0.10.1" @@ -1312,6 +1361,26 @@ dependencies = [ "parking_lot_core 0.9.12", ] +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom 7.1.3", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "deranged" version = "0.5.9" @@ -1419,13 +1488,13 @@ dependencies = [ "prost", "prost-types", "quinn-udp 0.6.3", + "rcgen", "rdkafka", "reqwest 0.13.5", "rmp", "rmpv", "rustc-hash", "rustls", - "rustls-pki-types", "scalo", "schemars", "serde", @@ -2891,6 +2960,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -2994,6 +3073,15 @@ dependencies = [ "objc2-foundation", ] +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -3250,6 +3338,16 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "pem" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" +dependencies = [ + "base64 0.23.1", + "serde_core", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -3805,6 +3903,20 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rcgen" +version = "0.14.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8" +dependencies = [ + "aws-lc-rs", + "pem", + "rustls-pki-types", + "time", + "x509-parser", + "yasna", +] + [[package]] name = "rdkafka" version = "0.39.0" @@ -4014,7 +4126,7 @@ dependencies = [ "cfg-if", "getrandom 0.2.17", "libc", - "untrusted", + "untrusted 0.9.0", "windows-sys 0.52.0", ] @@ -4089,6 +4201,15 @@ dependencies = [ "semver", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom 7.1.3", +] + [[package]] name = "rustify" version = "0.7.0" @@ -4209,7 +4330,7 @@ dependencies = [ "aws-lc-rs", "ring", "rustls-pki-types", - "untrusted", + "untrusted 0.9.0", ] [[package]] @@ -4268,12 +4389,12 @@ dependencies = [ [[package]] name = "scalo" -version = "2.14.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aee9c374451af23a6ebf90c02ce4ae33ea37fda7f397622825a3e0618988c566" +version = "0.0.0" +source = "git+https://github.com/hyperi-io/scalo-rs?rev=77f9039972f606275d1451cd9370b719d91649ff#77f9039972f606275d1451cd9370b719d91649ff" dependencies = [ "aes-gcm", "aho-corasick", + "arc-swap", "aws-config", "aws-sdk-secretsmanager", "axum", @@ -4314,6 +4435,9 @@ dependencies = [ "regex-syntax", "reqwest 0.13.5", "rmpv", + "rustls", + "rustls-native-certs", + "rustls-pki-types", "schemars", "serde", "serde-saphyr", @@ -4326,6 +4450,7 @@ dependencies = [ "tempfile", "thiserror 2.0.21", "tokio", + "tokio-rustls", "tokio-stream", "tokio-util", "tonic", @@ -4339,6 +4464,7 @@ dependencies = [ "tracing-throttle", "uuid", "vaultrs", + "webpki-roots", "wide", "yaque", "zstd", @@ -4819,6 +4945,17 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "synstructure" version = "0.14.0" @@ -5524,6 +5661,12 @@ dependencies = [ "ctutils", ] +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "untrusted" version = "0.9.0" @@ -5743,6 +5886,15 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "webpki-roots" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7265c9301c16a8711c992c2d8da90ee77ef54b1ed51482d6dfeb00dc756b1297" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "wide" version = "1.7.1" @@ -6149,6 +6301,24 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "aws-lc-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom 7.1.3", + "oid-registry", + "rusticata-macros", + "thiserror 2.0.21", + "time", +] + [[package]] name = "xmlparser" version = "0.13.6" @@ -6176,6 +6346,16 @@ dependencies = [ "sysinfo 0.28.4", ] +[[package]] +name = "yasna" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" +dependencies = [ + "bit-vec 0.9.1", + "time", +] + [[package]] name = "yoke" version = "0.8.4" diff --git a/Cargo.toml b/Cargo.toml index 9dec868..3ed2007 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,8 @@ doctest = false [dependencies] # The floor moves with `release.helm.library` in .hyperi-ci.yaml. -scalo = { version = "2.14.30", features = [ +# Pinned to the merge that adds `ServerTls`, whose placeholder version is 0.0.0, until the release that ships it replaces this with a version. +scalo = { version = "0.0.0", git = "https://github.com/hyperi-io/scalo-rs", rev = "77f9039972f606275d1451cd9370b719d91649ff", features = [ "config", "config-reload", "config-schema", @@ -42,6 +43,8 @@ scalo = { version = "2.14.30", features = [ "secrets-vault", # `aws:` secret refs resolve through scalo's AWS provider, compiled in only with this. "secrets-aws", + # `ServerTls`: every TLS listener's certificate, its reload and the handshake. + "tls", "scaling", # The webhook CEL `filter` calls scalo::expression directly. `rules` turns it on too. "expression", @@ -87,9 +90,6 @@ tonic-prost = "0.14.9" prost = "0.14.4" prost-types = "0.14.4" -rustls = "0.23.46" -# PEM parsing, as scalo's src/tls.rs does: rustls-pemfile is unmaintained (RUSTSEC-2025-0134). -rustls-pki-types = { version = ">=1.15.1, <2", features = ["std"] } tokio-rustls = "0.26.6" tower_governor = { version = "0.8", features = ["axum"] } @@ -154,6 +154,10 @@ proptest = "1.12.0" tokio-test = "0.4.6" reqwest = { version = "0.13.5", features = ["json"] } dotenvy = "0.15.7" +# The TLS client the listener tests handshake with. +rustls = "0.23.46" +# aws-lc-rs only, like the runtime: ring beside it leaves rustls without a default provider. +rcgen = { version = ">=0.14.10, <0.15", default-features = false, features = ["pem", "aws_lc_rs"] } rdkafka = { version = "0.39", features = ["cmake-build", "ssl", "sasl"] } temp-env = { version = "0.3.6", features = ["async_closure"] } pcap-file = ">=2.0, <3" @@ -162,7 +166,7 @@ sha2 = ">=0.11.1, <0.12" allocation-counter = "0.8.1" # Test support: contract e2e helpers, the Vector-compat gRPC client and the shared Kafka broker. -scalo = { version = "2.14.30", features = [ +scalo = { version = "0.0.0", git = "https://github.com/hyperi-io/scalo-rs", rev = "77f9039972f606275d1451cd9370b719d91649ff", features = [ "deployment-test-support", "transport-grpc-vector-compat", "transport-kafka-test-support", diff --git a/config.example.yaml b/config.example.yaml index 14b672e..2aa5ed8 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -102,11 +102,11 @@ server: key_file: "/etc/ssl/receiver.key" ca_file: "/etc/ssl/ca.crt" client_auth: "none" # none, optional, required - # Secret manager sources (override file paths when set) + # Secret manager sources, in place of the file paths: set cert_secret or cert_file, never both # cert_secret: "vault:secret/tls:cert" # key_secret: "vault:secret/tls:key" # ca_secret: "vault:secret/tls:ca" - # refresh_interval_secs: 3600 # Hot-reload interval (1 hour) + # refresh_interval_secs: 3600 # How often cert, key and CA are re-read, from files and secrets alike auth: # none, header, bearer, mtls, both. Anything else refuses to start. header @@ -140,9 +140,9 @@ grpc: key_file: "/etc/ssl/grpc-receiver.key" # ca_file: "/etc/ssl/ca.crt" # For mTLS # client_auth: "none" # none, optional, required - # cert_secret: "vault:secret/tls:cert" # Load from secret manager + # cert_secret: "vault:secret/tls:cert" # Instead of cert_file # key_secret: "vault:secret/tls:key" - # refresh_interval_secs: 3600 # Hot-reload interval (1 hour) + # refresh_interval_secs: 3600 # Re-read interval for cert, key and CA (1 hour) # server.ip_filter and server.rate_limit do NOT reach this port: tonic owns # the accept loop, so there is nowhere to reject a connection before it is diff --git a/deny.toml b/deny.toml index a8a7511..6402215 100644 --- a/deny.toml +++ b/deny.toml @@ -58,3 +58,5 @@ multiple-versions = "warn" [sources] unknown-registry = "deny" unknown-git = "deny" +# scalo is pinned to a git commit until the release that ships ServerTls replaces it. +allow-git = ["https://github.com/hyperi-io/scalo-rs"] diff --git a/docs/config-schema.json b/docs/config-schema.json index f94050d..e6de597 100644 --- a/docs/config-schema.json +++ b/docs/config-schema.json @@ -760,7 +760,7 @@ "default": false }, "cert_file": { - "description": "Path to certificate file (local file path).", + "description": "Path to the PEM certificate chain, leaf first. Set this or `cert_secret`, not both.", "type": [ "string", "null" @@ -768,7 +768,7 @@ "default": null }, "key_file": { - "description": "Path to private key file (local file path).", + "description": "Path to the PEM private key. Set this or `key_secret`, not both.", "type": [ "string", "null" @@ -776,7 +776,7 @@ "default": null }, "ca_file": { - "description": "Path to CA certificate for client verification (mTLS).", + "description": "Path to the PEM CA certificate client certificates must chain to (mTLS). Set this or `ca_secret`, not both.", "type": [ "string", "null" @@ -789,7 +789,7 @@ "default": "none" }, "cert_secret": { - "description": "Secret source for certificate (overrides cert_file).\nFormat: \"provider:path:key\" (e.g., \"vault:secret/tls:cert\")", + "description": "Secret source for the certificate, instead of `cert_file`.\nFormat: \"provider:path:key\" (e.g., \"vault:secret/tls:cert\")", "type": [ "string", "null" @@ -797,7 +797,7 @@ "default": null }, "key_secret": { - "description": "Secret source for private key (overrides key_file).\nFormat: \"provider:path:key\" (e.g., \"vault:secret/tls:key\")", + "description": "Secret source for the private key, instead of `key_file`.\nFormat: \"provider:path:key\" (e.g., \"vault:secret/tls:key\")", "type": [ "string", "null" @@ -805,7 +805,7 @@ "default": null }, "ca_secret": { - "description": "Secret source for CA certificate (overrides ca_file).\nFormat: \"provider:path:key\" (e.g., \"vault:secret/tls:ca\")", + "description": "Secret source for the CA certificate, instead of `ca_file`.\nFormat: \"provider:path:key\" (e.g., \"vault:secret/tls:ca\")", "type": [ "string", "null" @@ -813,7 +813,7 @@ "default": null }, "refresh_interval_secs": { - "description": "Refresh interval for secrets in seconds.\nDefault: 3600 (1 hour)", + "description": "How often the certificate, key and CA are re-read, in seconds, from files and secrets alike.\nA rotated certificate is served from the next handshake, with no restart. 0 turns reload off.\nDefault: 3600 (1 hour)", "type": "integer", "format": "uint64", "minimum": 0, diff --git a/docs/config-schema.yaml b/docs/config-schema.yaml index d1a9213..99caff2 100644 --- a/docs/config-schema.yaml +++ b/docs/config-schema.yaml @@ -710,19 +710,23 @@ $defs: type: boolean default: false cert_file: - description: Path to certificate file (local file path). + description: >- + Path to the PEM certificate chain, leaf first. Set this or `cert_secret`, not + both. type: - string - "null" default: null key_file: - description: Path to private key file (local file path). + description: Path to the PEM private key. Set this or `key_secret`, not both. type: - string - "null" default: null ca_file: - description: Path to CA certificate for client verification (mTLS). + description: >- + Path to the PEM CA certificate client certificates must chain to (mTLS). Set + this or `ca_secret`, not both. type: - string - "null" @@ -733,7 +737,7 @@ $defs: default: none cert_secret: description: |- - Secret source for certificate (overrides cert_file). + Secret source for the certificate, instead of `cert_file`. Format: "provider:path:key" (e.g., "vault:secret/tls:cert") type: - string @@ -741,7 +745,7 @@ $defs: default: null key_secret: description: |- - Secret source for private key (overrides key_file). + Secret source for the private key, instead of `key_file`. Format: "provider:path:key" (e.g., "vault:secret/tls:key") type: - string @@ -749,7 +753,7 @@ $defs: default: null ca_secret: description: |- - Secret source for CA certificate (overrides ca_file). + Secret source for the CA certificate, instead of `ca_file`. Format: "provider:path:key" (e.g., "vault:secret/tls:ca") type: - string @@ -757,7 +761,8 @@ $defs: default: null refresh_interval_secs: description: |- - Refresh interval for secrets in seconds. + How often the certificate, key and CA are re-read, in seconds, from files and secrets alike. + A rotated certificate is served from the next handshake, with no restart. 0 turns reload off. Default: 3600 (1 hour) type: integer format: uint64 diff --git a/docs/hardening.md b/docs/hardening.md index 2f38771..a1b8ed5 100644 --- a/docs/hardening.md +++ b/docs/hardening.md @@ -25,7 +25,7 @@ Application-level hardening, and cost-conscious fronting on Kubernetes and AWS. | Flow per-source rate limiting | Token bucket per UDP source, opt-in. The bucket map never holds more than `cache_size` (default 4096) sources: a further source past it evicts an idle bucket where one is among the first 32 looked at, else the first, counted on `flow_rate_limit_evicted_total`, so spoofed sources cannot grow it | `flow.rate_limit.*` | | IP filter (allowlist/denylist) | CIDR trie, connection-level reject, in every accept loop the receiver runs. The gRPC and OTLP gRPC ports have none: tonic runs their accept loops. An unknown mode, a bad CIDR or an empty allowlist stops startup | `server.ip_filter.*` | | 503 backpressure | HTTP + gRPC ingest shed load when pipeline not ready | `Retry-After: 5` | -| TLS termination | Per-protocol. The ingest, webhook, HEC, remote write and OTLP HTTP listeners reload certificate material given as a secret reference every `refresh_interval_secs`. A certificate file, every other listener and every other `tls.*` setting load once at start, so a change to them is a restart | `*.tls.enabled`, `*.tls.cert_secret` | +| TLS termination | Per-protocol. Every TLS listener re-reads its certificate, key and client CA every `refresh_interval_secs`, from a file or a secret reference, and serves a rotated certificate from the next handshake. Every other `tls.*` setting loads once at start, so a change to it is a restart | `*.tls.enabled`, `*.tls.cert_file`, `*.tls.cert_secret` | | TLS handshake timeout | 10s, every TLS listener the receiver accepts on | `server.connections.tls_handshake_timeout_ms` | | mTLS client auth | Per-protocol | `*.tls.client_auth: required` | | Bearer token auth | Per-protocol, hot-reloadable | `*.auth.mode: bearer` | diff --git a/docs/internals.md b/docs/internals.md index a892278..66cdb91 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -63,7 +63,7 @@ Static tokens, `accepted_headers[].values` and `header_values` are `SensitiveStr Tokens come from the config (development) or a secret source, refreshed every `auth.bearer.refresh_interval_secs`, so a token rotates without a restart. A config reload replaces a listener's whole auth section at once: mode, static tokens, accepted headers and secret source, with the replaced secret refresh stopped. It reaches every listener that authenticates -- `server`, `splunk_hec`, `prometheus_rw`, `otlp` (both ports) and `grpc` -- as soon as the reload lands, so a revoked credential is refused on the next request. A reloaded section that does not build, a secret source that will not load say, keeps the current one and logs an error. A webhook caller's secret rotates on the caller's `refresh_interval_secs`. -Certificate material given as a secret reference refreshes on the ingest, webhook, HEC, remote write and OTLP HTTP listeners, every `tls.refresh_interval_secs`. Everything else under `tls.*`, the client-certificate policy included, is read once at start. A change to it needs a restart. A reload into `mtls` or `both` is refused on a listener with a running handshake that admits clients with no certificate: the current auth stays and an error is logged. +The certificate, key and client CA of every TLS listener, from a file or a secret reference, are re-read every `tls.refresh_interval_secs`, and a rotated certificate is served from the next handshake. A certificate, key or CA is set as a file or as a secret, never both: startup refuses both. Everything else under `tls.*`, the client-certificate policy included, is read once at start. A change to it needs a restart. A reload into `mtls` or `both` is refused on a listener with a running handshake that admits clients with no certificate: the current auth stays and an error is logged. The gRPC and OTLP gRPC ports authenticate in a tonic interceptor that checks bearer tokens and nothing else, so `header` and `both` are refused at startup there. Under `mtls` the interceptor checks no token, because the handshake verified the client certificate before it. Startup rejects `mtls` there without `tls.client_auth: required` and the CA. A port started with `auth.mode: none` has no interceptor to reload, so turning its auth on or off takes a restart. diff --git a/docs/metrics.md b/docs/metrics.md index ff0b568..692b33b 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -80,6 +80,7 @@ A DFE deployment scales on that score through dfe-keda-shim: dfe-infra adds a `m | `spool_*` | The disk spool, with spillover on: `spool_bytes`, `spool_messages`, `spool_disk_available` and the rest | | `worker_pool_*` | scalo's worker pool | | flow | `flow_records_emitted_total{transport,mode}`, `flow_rate_limited_total{transport}`, `flow_template_evicted_total`, `flow_kernel_drops_total`, `transport_decode_err_total{transport,reason}`, `transport_drops_total{transport,reason}`, `handler_experimental{handler}` | +| `tls_*` | Every TLS listener's handshakes and certificate reloads: `tls_handshakes_total{protocol,key_exchange}`, `tls_handshake_failures_total{reason}`, `tls_certificate_reloads_total{result}` | | `process_*`, `container_*` | Resident and virtual memory, open file descriptors, start time, and the cgroup memory and CPU limits, not in the manifest | ## Alerts diff --git a/src/config/mod.rs b/src/config/mod.rs index 7dcb8a9..fab2990 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -37,7 +37,8 @@ use serde::{Deserialize, Serialize}; use scalo::scaling::{ScalingComponent, ScalingPressure, ScalingPressureConfig}; use crate::error::{Error, Result}; -use crate::server::tls::{ClientAuth, Handshake}; +use crate::server::tls::{Handshake, parse_client_auth}; +use scalo::tls::ClientAuth; /// Environment variable prefix for configuration. pub const ENV_PREFIX: &str = "DFE_RECEIVER"; @@ -556,6 +557,7 @@ impl Config { ("webhook", &self.webhook.tls), ] { known_client_auth(scope, tls)?; + one_source_each(scope, tls)?; } // A mode the parser does not recognise is refused on every listener, @@ -749,7 +751,7 @@ fn header_credentials_configured(scope: &str, auth: &AuthConfig) -> Result<()> { /// Refuse a `tls.client_auth` the TLS builders do not recognise, an empty one included. fn known_client_auth(scope: &str, tls: &TlsConfig) -> Result<()> { - if ClientAuth::parse(&tls.client_auth).is_some() { + if parse_client_auth(&tls.client_auth).is_some() { return Ok(()); } Err(Error::Config(format!( @@ -758,6 +760,29 @@ fn known_client_auth(scope: &str, tls: &TlsConfig) -> Result<()> { ))) } +/// Refuse an enabled TLS block that sets the certificate, the key or the CA both +/// as a file and as a secret: scalo's `ServerTls` reads one source for each, and +/// which of two to prefer is not a guess a listener should make. +fn one_source_each(scope: &str, tls: &TlsConfig) -> Result<()> { + if !tls.enabled { + return Ok(()); + } + for (part, file, secret) in [ + ("cert", &tls.cert_file, &tls.cert_secret), + ("key", &tls.key_file, &tls.key_secret), + ("ca", &tls.ca_file, &tls.ca_secret), + ] { + if file.is_some() && secret.is_some() { + return Err(Error::Config(format!( + "{scope}.tls.{part}_file and {scope}.tls.{part}_secret are both set -- \ + set one, the secret to read it from a secret manager or the file to read \ + it from disk" + ))); + } + } + Ok(()) +} + /// Refuse a listener whose wire protocol carries no credential unless its /// clients are authenticated at the handshake or `accept_unauthenticated` is set. fn unauthenticated_is_accepted( @@ -817,7 +842,7 @@ fn cert_half_is_enforced(scope: &str, auth: &AuthConfig, tls: &TlsConfig) -> Res accepts every client unauthenticated" ))); } - if ClientAuth::parse(&tls.client_auth) != Some(ClientAuth::Required) { + if parse_client_auth(&tls.client_auth) != Some(ClientAuth::Required) { return Err(Error::Config(format!( "{scope}.auth.mode is '{mode}' but {scope}.tls.client_auth is '{}' -- \ it must be 'required', or clients presenting no certificate are \ @@ -1284,31 +1309,32 @@ pub struct TlsConfig { /// Enable TLS. pub enabled: bool, - /// Path to certificate file (local file path). + /// Path to the PEM certificate chain, leaf first. Set this or `cert_secret`, not both. pub cert_file: Option, - /// Path to private key file (local file path). + /// Path to the PEM private key. Set this or `key_secret`, not both. pub key_file: Option, - /// Path to CA certificate for client verification (mTLS). + /// Path to the PEM CA certificate client certificates must chain to (mTLS). Set this or `ca_secret`, not both. pub ca_file: Option, /// Client certificate requirement (none, optional, required). pub client_auth: String, - /// Secret source for certificate (overrides cert_file). + /// Secret source for the certificate, instead of `cert_file`. /// Format: "provider:path:key" (e.g., "vault:secret/tls:cert") pub cert_secret: Option, - /// Secret source for private key (overrides key_file). + /// Secret source for the private key, instead of `key_file`. /// Format: "provider:path:key" (e.g., "vault:secret/tls:key") pub key_secret: Option, - /// Secret source for CA certificate (overrides ca_file). + /// Secret source for the CA certificate, instead of `ca_file`. /// Format: "provider:path:key" (e.g., "vault:secret/tls:ca") pub ca_secret: Option, - /// Refresh interval for secrets in seconds. + /// How often the certificate, key and CA are re-read, in seconds, from files and secrets alike. + /// A rotated certificate is served from the next handshake, with no restart. 0 turns reload off. /// Default: 3600 (1 hour) pub refresh_interval_secs: u64, } @@ -3586,6 +3612,43 @@ mod tests { assert!(with_listener("lumberjack", tls, false).validate().is_ok()); } + /// scalo's `ServerTls` reads one source for the certificate, key and CA, so a + /// block that sets both is refused at config load, on every listener that has TLS. + #[test] + fn a_tls_part_set_as_file_and_secret_is_refused_on_every_listener() { + let both = |part: &str| { + let mut tls = client_certificates_required(); + let spec = Some("vault:secret/tls:value".to_string()); + match part { + "cert" => tls.cert_secret = spec, + "key" => tls.key_secret = spec, + _ => tls.ca_secret = spec, + } + tls + }; + for scope in ["lumberjack", "fluent", "gelf", "syslog"] { + for part in ["cert", "key", "ca"] { + let err = with_listener(scope, both(part), true) + .validate() + .expect_err(scope) + .to_string(); + assert!( + err.contains(&format!("{scope}.tls.{part}_file")) + && err.contains(&format!("{scope}.tls.{part}_secret")), + "{scope} {part}: {err}" + ); + } + } + + let mut config = auth_base(); + config.server.tls = both("cert"); + assert!(config.validate().is_err(), "server.tls"); + + // A disabled block is not read, so what it holds is not refused. + config.server.tls.enabled = false; + assert!(config.validate().is_ok()); + } + /// Client certificates on syslog's TLS listener leave UDP and plain TCP open. #[test] fn syslog_needs_the_opt_out_even_with_client_certificates() { diff --git a/src/main.rs b/src/main.rs index afc1d84..793b647 100644 --- a/src/main.rs +++ b/src/main.rs @@ -253,7 +253,7 @@ impl ServiceApp for App { const SHUTDOWN_GRACE: Duration = Duration::from_secs(5); fn main() -> ExitCode { - dfe_receiver::server::tls::install_crypto_provider(); + scalo::tls::install_default_provider(); let runtime = match tokio::runtime::Runtime::new() { Ok(runtime) => runtime, Err(e) => { diff --git a/src/server/fluent/mod.rs b/src/server/fluent/mod.rs index 40bebf3..f2e97e1 100644 --- a/src/server/fluent/mod.rs +++ b/src/server/fluent/mod.rs @@ -20,9 +20,9 @@ pub mod scan; use std::net::SocketAddr; use std::sync::Arc; +use scalo::tls::{ServerTls, ServerTlsOptions}; use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; use tokio::net::TcpListener; -use tokio::time::timeout; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; @@ -34,6 +34,7 @@ use crate::server::frame_clock::Clocked; use crate::server::hold::hold_until_settled; use crate::server::http::{Accept, ConnectionLimits}; use crate::server::ip_filter::IpFilter; +use crate::server::tls::handshake; use crate::server::traits::{BoundAddr, ProtocolHandler}; use crate::tenancy::{Ingest, Principal}; use convert::{Refused, extract_chunk_id, fluent_to_json, unsupported_compression}; @@ -228,7 +229,7 @@ async fn run_tcp( metrics: Arc, acks: Acks, shutdown: CancellationToken, - tls_acceptor: Option, + tls: Option, max_buffer_size: usize, ip_filter: IpFilter, bound: BoundAddr, @@ -238,7 +239,7 @@ async fn run_tcp( .map_err(|e| Error::Server(format!("failed to bind Fluent Forward listener: {e}")))?; let _serving = bound.publish(&listener.local_addr()); - let tls_enabled = tls_acceptor.is_some(); + let tls_enabled = tls.is_some(); info!(addr = %bind_addr, tls = tls_enabled, "Fluent Forward listener started"); let accept = Accept { @@ -275,26 +276,17 @@ async fn run_tcp( let conn_shutdown = shutdown.clone(); let accept = accept.clone(); - if let Some(ref acceptor) = tls_acceptor { - let acceptor = acceptor.clone(); + if let Some(ref tls) = tls { + let tls = tls.clone(); tokio::spawn(async move { let _open = open; if !accept.first_byte(&stream).await { return; } - let tls_result = timeout(accept.limits.tls_handshake, acceptor.accept(stream)).await; - let tls_stream = match tls_result { - Ok(Ok(s)) => s, - Ok(Err(e)) => { - metrics.inc_tls_handshake_failure(); - debug!(peer = %peer_addr, error = %e, "Fluent Forward TLS handshake failed"); - return; - } - Err(_) => { - metrics.inc_tls_handshake_failure(); - warn!(peer = %peer_addr, "Fluent Forward TLS handshake timeout"); - return; - } + let Some(tls_stream) = + handshake(&tls, stream, peer_addr, &metrics).await + else { + return; }; handle_tcp_connection( @@ -371,12 +363,12 @@ impl ProtocolHandler for FluentHandler { .parse() .map_err(|e| Error::Config(format!("invalid Fluent Forward bind address: {e}")))?; - // Build TLS acceptor if enabled - let tls_acceptor = if self.config.tls.enabled { - super::tls::build_tls_acceptor_async(&self.config.tls).await? - } else { - None - }; + let tls = super::tls::load_server_tls( + &self.config.tls, + &self.pipeline.config().server.connections, + ServerTlsOptions::default(), + ) + .await?; // Forward carries no credential, so the IP filter and the TLS // handshake are the whole admission surface on this port. @@ -392,7 +384,7 @@ impl ProtocolHandler for FluentHandler { self.metrics.clone(), acks, shutdown, - tls_acceptor, + tls, self.config.max_message_size, ip_filter, self.bound.clone(), diff --git a/src/server/gelf/mod.rs b/src/server/gelf/mod.rs index 8c771ab..ce6e0ee 100644 --- a/src/server/gelf/mod.rs +++ b/src/server/gelf/mod.rs @@ -18,9 +18,9 @@ use std::net::SocketAddr; use std::sync::Arc; use bytes::{Buf, Bytes, BytesMut}; +use scalo::tls::{ServerTls, ServerTlsOptions}; use tokio::io::{AsyncRead, AsyncWrite}; use tokio::net::TcpListener; -use tokio::time::timeout; use tokio_util::codec::{Decoder, FramedRead}; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; @@ -33,6 +33,7 @@ use crate::server::frame_clock::Clocked; use crate::server::hold::hold_until_settled; use crate::server::http::{Accept, ConnectionLimits}; use crate::server::ip_filter::IpFilter; +use crate::server::tls::handshake; use crate::server::traits::{BoundAddr, ProtocolHandler}; use crate::tenancy::{Ingest, Principal}; use convert::gelf_to_json; @@ -193,7 +194,7 @@ async fn run_tcp( pipeline: Arc, metrics: Arc, shutdown: CancellationToken, - tls_acceptor: Option, + tls: Option, max_message_size: usize, ip_filter: IpFilter, bound: BoundAddr, @@ -203,7 +204,7 @@ async fn run_tcp( .map_err(|e| Error::Server(format!("failed to bind GELF TCP listener: {e}")))?; let _serving = bound.publish(&listener.local_addr()); - let tls_enabled = tls_acceptor.is_some(); + let tls_enabled = tls.is_some(); info!(addr = %bind_addr, tls = tls_enabled, "GELF TCP listener started"); let accept = Accept { @@ -239,26 +240,17 @@ async fn run_tcp( let conn_shutdown = shutdown.clone(); let accept = accept.clone(); - if let Some(ref acceptor) = tls_acceptor { - let acceptor = acceptor.clone(); + if let Some(ref tls) = tls { + let tls = tls.clone(); tokio::spawn(async move { let _open = open; if !accept.first_byte(&stream).await { return; } - let tls_result = timeout(accept.limits.tls_handshake, acceptor.accept(stream)).await; - let tls_stream = match tls_result { - Ok(Ok(s)) => s, - Ok(Err(e)) => { - metrics.inc_tls_handshake_failure(); - debug!(peer = %peer_addr, error = %e, "GELF TLS handshake failed"); - return; - } - Err(_) => { - metrics.inc_tls_handshake_failure(); - warn!(peer = %peer_addr, "GELF TLS handshake timeout"); - return; - } + let Some(tls_stream) = + handshake(&tls, stream, peer_addr, &metrics).await + else { + return; }; handle_tcp_connection( @@ -337,12 +329,12 @@ impl ProtocolHandler for GelfHandler { let max_msg = self.config.max_message_size; - // Build TLS acceptor if enabled - let tls_acceptor = if self.config.tls.enabled { - super::tls::build_tls_acceptor_async(&self.config.tls).await? - } else { - None - }; + let tls = super::tls::load_server_tls( + &self.config.tls, + &self.pipeline.config().server.connections, + ServerTlsOptions::default(), + ) + .await?; // GELF carries no credential, so the IP filter and the TLS handshake // are the whole admission surface on this port. @@ -353,7 +345,7 @@ impl ProtocolHandler for GelfHandler { self.pipeline.clone(), self.metrics.clone(), shutdown, - tls_acceptor, + tls, max_msg, ip_filter, self.bound.clone(), diff --git a/src/server/grpc/mod.rs b/src/server/grpc/mod.rs index 6f2d13f..30cc2ab 100644 --- a/src/server/grpc/mod.rs +++ b/src/server/grpc/mod.rs @@ -24,9 +24,12 @@ use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; +use scalo::tls::ServerTls; use scalo::transport::grpc::sender_deadline; use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; use tokio::net::TcpStream; +use tokio::sync::mpsc; +use tokio_rustls::server::TlsStream; use tokio_stream::Stream; use tokio_util::sync::CancellationToken; use tonic::service::interceptor::InterceptedService; @@ -40,7 +43,7 @@ use crate::metrics::{ConnectionGuard, Metrics}; use crate::pipeline::{Acks, BatchOutcome, PipelineState}; use crate::server::auth::{AuthMode, AuthState, grpc_auth_interceptor, principal_or_listener}; use crate::server::http::{ConnectionLimits, Slot, create_auth_state}; -use crate::server::tls::Handshake; +use crate::server::tls::{Handshake, grpc_options, handshake, load_server_tls}; use crate::server::traits::{BoundAddr, ProtocolHandler}; use crate::tenancy::Ingest; @@ -206,21 +209,24 @@ fn push_answer( } /// An accepted gRPC connection, counted open until it drops. -pub(crate) struct CountedConnection { - stream: TcpStream, +/// +/// `S` is the TCP stream, or the TLS stream once its handshake is done. +pub(crate) struct CountedConnection { + stream: S, + info: TcpConnectInfo, _open: ConnectionGuard, _slot: Slot, } -impl Connected for CountedConnection { +impl Connected for CountedConnection { type ConnectInfo = TcpConnectInfo; fn connect_info(&self) -> Self::ConnectInfo { - self.stream.connect_info() + self.info.clone() } } -impl AsyncRead for CountedConnection { +impl AsyncRead for CountedConnection { fn poll_read( self: Pin<&mut Self>, cx: &mut Context<'_>, @@ -230,7 +236,7 @@ impl AsyncRead for CountedConnection { } } -impl AsyncWrite for CountedConnection { +impl AsyncWrite for CountedConnection { fn poll_write( self: Pin<&mut Self>, cx: &mut Context<'_>, @@ -267,7 +273,7 @@ pub(crate) fn count_connections( metrics: Arc, transport: &'static str, limits: ConnectionLimits, -) -> impl Stream> { +) -> impl Stream>> { tokio_stream::StreamExt::filter_map(incoming, move |conn| match conn { Ok(stream) => { let Some(slot) = limits.slot() else { @@ -275,6 +281,7 @@ pub(crate) fn count_connections( return None; }; Some(Ok(CountedConnection { + info: stream.connect_info(), stream, _open: metrics.open_connection(transport), _slot: slot, @@ -284,6 +291,76 @@ pub(crate) fn count_connections( }) } +/// How many handshaken connections wait for tonic to take them. +const HANDSHAKED_BACKLOG: usize = 1024; + +/// `connections` after their TLS handshake on `tls`. +/// +/// Each handshake runs in its own task, bounded by the handshake timeout, so a +/// slow client holds up nobody else. A connection is counted open and holds its +/// place under the listener's cap from before its handshake, and one whose +/// handshake fails is dropped and counted. Accepting stops once tonic drops +/// the stream. +pub(crate) fn handshake_connections( + connections: impl Stream>> + Send + 'static, + tls: ServerTls, + metrics: Arc, +) -> impl Stream>>> { + let (sender, receiver) = mpsc::channel(HANDSHAKED_BACKLOG); + tokio::spawn(async move { + let mut connections = std::pin::pin!(connections); + loop { + let next = tokio::select! { + () = sender.closed() => break, + next = tokio_stream::StreamExt::next(&mut connections) => next, + }; + match next { + None => break, + Some(Err(e)) => { + if sender.send(Err(e)).await.is_err() { + break; + } + } + Some(Ok(connection)) => { + let (tls, metrics, sender) = (tls.clone(), metrics.clone(), sender.clone()); + tokio::spawn(async move { + let secured = connection.handshake(&tls, &metrics).await; + if let Some(secured) = secured { + let _ = sender.send(Ok(secured)).await; + } + }); + } + } + } + }); + tokio_stream::wrappers::ReceiverStream::new(receiver) +} + +impl CountedConnection { + /// This connection once its TLS handshake on `tls` is done, `None` when + /// the handshake failed. + async fn handshake( + self, + tls: &ServerTls, + metrics: &Metrics, + ) -> Option>> { + let Self { + stream, + info, + _open: open, + _slot: slot, + } = self; + let peer = info.remote_addr()?; + let stream = handshake(tls, stream, peer, metrics).await?; + Some(CountedConnection { + stream, + info, + _open: open, + _slot: slot, + }) + } +} + /// The HTTP/2 keep-alive a tonic listener pings with, from `server.connections`. pub(crate) fn keep_alive( builder: tonic::transport::Server, @@ -457,24 +534,11 @@ async fn serve( let acks = pipeline.acks(TRANSPORT, config.grpc.acknowledgements, None); let service = VectorService::new(pipeline, metrics.clone(), acks); - // Build TLS config if enabled - let tls_config = if config.grpc.tls.enabled { - let identity = super::tls::build_grpc_tls_config(&config.grpc.tls).await?; - Some(identity) - } else { - None - }; + let tls = load_server_tls(&config.grpc.tls, &config.server.connections, grpc_options()).await?; let limits = ConnectionLimits::new(&config.server.connections); - let mut builder = keep_alive(tonic::transport::Server::builder(), &limits); - - // Apply TLS - if let Some(tls) = tls_config { - builder = builder - .tls_config(tls) - .map_err(|e| Error::Tls(format!("gRPC TLS config error: {e}")))?; - } - let mut builder = builder.layer(count_oversize(metrics.clone(), TRANSPORT)); + let mut builder = keep_alive(tonic::transport::Server::builder(), &limits) + .layer(count_oversize(metrics.clone(), TRANSPORT)); // Gzip on both directions: scalo's VectorCompatClient compresses // unconditionally, and a server without the encoding enabled rejects the RPC. @@ -497,15 +561,25 @@ async fn serve( .with_nodelay(Some(true)); let _serving = bound.publish(&incoming.local_addr()); - info!(addr = %addr, "gRPC server listening"); - - router - .serve_with_incoming_shutdown( - count_connections(incoming, metrics, TRANSPORT, limits), - shutdown.cancelled_owned(), - ) - .await - .map_err(|e| Error::Server(format!("gRPC server error: {e}")))?; + info!(addr = %addr, tls = tls.is_some(), "gRPC server listening"); + + let connections = count_connections(incoming, metrics.clone(), TRANSPORT, limits); + let served = match tls { + Some(tls) => { + router + .serve_with_incoming_shutdown( + handshake_connections(connections, tls, metrics), + shutdown.cancelled_owned(), + ) + .await + } + None => { + router + .serve_with_incoming_shutdown(connections, shutdown.cancelled_owned()) + .await + } + }; + served.map_err(|e| Error::Server(format!("gRPC server error: {e}")))?; info!("gRPC server stopped"); Ok(()) diff --git a/src/server/http/mod.rs b/src/server/http/mod.rs index 7825200..fc7dc22 100644 --- a/src/server/http/mod.rs +++ b/src/server/http/mod.rs @@ -33,16 +33,16 @@ use axum::routing::{get, post}; use axum::{Extension, Router}; use tokio::net::{TcpListener, TcpStream}; use tokio::time::{MissedTickBehavior, timeout}; -use tokio_rustls::TlsAcceptor; use tokio_util::sync::CancellationToken; use tower::limit::GlobalConcurrencyLimitLayer; use tower_governor::governor::GovernorConfigBuilder; use tower_governor::{GovernorError, GovernorLayer}; use tower_http::limit::RequestBodyLimitLayer; use tower_http::timeout::TimeoutLayer; -use tracing::{debug, error, info, warn}; +use tracing::{debug, error, info}; -use scalo::logger::security::{self, SecurityOutcome}; +use scalo::logger::security; +use scalo::tls::{ServerTls, ServerTlsOptions}; use crate::config::{AuthConfig, SharedConfig}; use crate::error::{Error, Result, unavailable_response}; @@ -56,7 +56,7 @@ use crate::server::client_ip::{ClientIpKey, TrustedProxies, attribute_client}; use crate::server::content_encoding; use crate::server::frame_clock::Clocked; use crate::server::ip_filter::IpFilter; -use crate::server::tls::{Handshake, TlsCertProvider, build_tls_acceptor, uses_secrets}; +use crate::server::tls::{Handshake, handshake, load_server_tls}; use crate::server::traits::{BoundAddr, ProtocolHandler}; use crate::tenancy::{Ingest, Principal}; use crate::validation::depth::{MAX_BATCH_DEPTH, MAX_PARSE_DEPTH, json_depth_within}; @@ -333,21 +333,12 @@ async fn serve( max_body_size: config.server.max_body_size, }; - // Build TLS: use TlsCertProvider with hot-reload if secrets configured, - // otherwise one-shot load - let tls_provider = if config.server.tls.enabled && uses_secrets(&config.server.tls) { - let provider = TlsCertProvider::new(config.server.tls.clone()).await?; - provider.start_refresh_task(); - Some(provider) - } else { - None - }; - - let tls_acceptor = if tls_provider.is_some() { - None // Handled by provider below - } else { - build_tls_acceptor(&config.server.tls)? - }; + let tls = load_server_tls( + &config.server.tls, + &config.server.connections, + ServerTlsOptions::default(), + ) + .await?; // Security configuration let max_body_size = config.server.max_body_size; @@ -436,14 +427,9 @@ async fn serve( transport: TRANSPORT, limits: ConnectionLimits::new(&config.server.connections), }; - if let Some(ref provider) = tls_provider { - let acceptor_handle = provider.acceptor_handle(); - info!(addr = %addr, tls = true, hot_reload = true, "HTTP server listening"); - run_tls_server(listener, app, acceptor_handle, accept, shutdown).await - } else if let Some(acceptor) = tls_acceptor { - let acceptor_handle = Arc::new(parking_lot::RwLock::new(acceptor)); - info!(addr = %addr, tls = true, hot_reload = false, "HTTP server listening"); - run_tls_server(listener, app, acceptor_handle, accept, shutdown).await + if let Some(tls) = tls { + info!(addr = %addr, tls = true, "HTTP server listening"); + run_tls_server(listener, app, tls, accept, shutdown).await } else { info!(addr = %addr, tls = false, "HTTP server listening"); run_plain_server(listener, app, accept, shutdown).await @@ -575,7 +561,6 @@ pub(crate) struct ConnectionLimits { pub(crate) first_byte: Duration, pub(crate) frame: Duration, pub(crate) idle: Duration, - pub(crate) tls_handshake: Duration, pub(crate) http2_keep_alive: Duration, } @@ -587,7 +572,6 @@ impl ConnectionLimits { first_byte: Duration::from_millis(config.first_byte_timeout_ms), frame: Duration::from_millis(config.frame_timeout_ms), idle: Duration::from_millis(config.idle_timeout_ms), - tls_handshake: Duration::from_millis(config.tls_handshake_timeout_ms), http2_keep_alive: Duration::from_millis(config.http2_keep_alive_interval_ms), } } @@ -929,12 +913,12 @@ fn connection_service( /// Run HTTP server with TLS. /// -/// Accepts an `Arc>` to support hot-reload of certificates. -/// The RwLock read is held only to clone the acceptor (cheap - wraps Arc). +/// Each connection's handshake runs on `tls`, so a certificate it has reloaded +/// is served from the next connection. pub(crate) async fn run_tls_server( listener: TcpListener, app: Router, - acceptor: Arc>, + tls: ServerTls, accept: Accept, shutdown: CancellationToken, ) -> Result<()> { @@ -959,10 +943,9 @@ pub(crate) async fn run_tls_server( continue; }; - let acceptor = acceptor.read().clone(); + let tls = tls.clone(); let app = app.clone(); let shutdown = shutdown.clone(); - let metrics = accept.metrics.clone(); let accept = accept.clone(); tokio::spawn(async move { @@ -970,40 +953,11 @@ pub(crate) async fn run_tls_server( if !accept.first_byte(&stream).await { return; } - // TLS handshake with timeout to prevent slow TLS attacks - let tls_result = - timeout(accept.limits.tls_handshake, acceptor.accept(stream)).await; - - let tls_stream = match tls_result { - Ok(Ok(stream)) => stream, - Ok(Err(e)) => { - // Track TLS failure in metrics - metrics.inc_tls_handshake_failure(); - debug!(peer = %peer_addr, error = %e, "tls_handshake_failed"); - security::tls_event( - "handshake", - SecurityOutcome::Failure, - Some(&e.to_string()), - Some(peer_addr.ip()), - ); - return; - } - Err(_) => { - // Track TLS timeout in metrics - metrics.inc_tls_handshake_failure(); - warn!(peer = %peer_addr, "tls_handshake_timeout"); - security::tls_event( - "handshake", - SecurityOutcome::Failure, - Some("handshake_timeout"), - Some(peer_addr.ip()), - ); - return; - } + let Some(tls_stream) = + handshake(&tls, stream, peer_addr, &accept.metrics).await + else { + return; }; - - debug!(peer = %peer_addr, "TLS connection established"); - let io = hyper_util::rt::TokioIo::new(tls_stream); serve_connection(io, app, peer_addr, &accept, &shutdown).await; }); diff --git a/src/server/lumberjack/mod.rs b/src/server/lumberjack/mod.rs index 85341f6..7214566 100644 --- a/src/server/lumberjack/mod.rs +++ b/src/server/lumberjack/mod.rs @@ -19,9 +19,9 @@ use std::net::SocketAddr; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; +use scalo::tls::ServerTlsOptions; use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, BufReader}; use tokio::net::TcpListener; -use tokio::time::timeout; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; @@ -34,6 +34,7 @@ use crate::metrics::{DropReason, Metrics}; use crate::pipeline::{Acks, PipelineState}; use crate::server::frame_clock::Clocked; use crate::server::http::Accept; +use crate::server::tls::handshake; use crate::server::traits::{BoundAddr, ProtocolHandler}; use crate::tenancy::{Ingest, Principal}; use codec::{CompressedFrames, Frame, encode_ack, read_frame}; @@ -362,15 +363,13 @@ impl ProtocolHandler for LumberjackHandler { .pipeline .acks("lumberjack", self.config.acknowledgements, None); - // Build TLS acceptor if enabled - let tls_acceptor = if self.config.tls.enabled { - let acceptor = super::tls::build_tls_acceptor_async(&self.config.tls).await?; - info!(addr = %addr, tls = true, "Lumberjack server listening"); - acceptor - } else { - info!(addr = %addr, tls = false, "Lumberjack server listening"); - None - }; + let tls = super::tls::load_server_tls( + &self.config.tls, + &self.pipeline.config().server.connections, + ServerTlsOptions::default(), + ) + .await?; + info!(addr = %addr, tls = tls.is_some(), "Lumberjack server listening"); // Published once TLS is ready, so a failed TLS setup never reads as serving. let _serving = self.bound.publish(&listener.local_addr()); @@ -402,30 +401,18 @@ impl ProtocolHandler for LumberjackHandler { let conn_shutdown = shutdown.clone(); let accept = accept.clone(); - if let Some(ref acceptor) = tls_acceptor { - let acceptor = acceptor.clone(); + if let Some(ref tls) = tls { + let tls = tls.clone(); tokio::spawn(async move { let _open = open; if !accept.first_byte(&stream).await { return; } - // TLS handshake with timeout - let tls_result = timeout(accept.limits.tls_handshake, acceptor.accept(stream)).await; - let tls_stream = match tls_result { - Ok(Ok(s)) => s, - Ok(Err(e)) => { - metrics.inc_tls_handshake_failure(); - debug!(peer = %peer_addr, error = %e, "Lumberjack TLS handshake failed"); - return; - } - Err(_) => { - metrics.inc_tls_handshake_failure(); - warn!(peer = %peer_addr, "Lumberjack TLS handshake timeout"); - return; - } + let Some(tls_stream) = + handshake(&tls, stream, peer_addr, &metrics).await + else { + return; }; - - debug!(peer = %peer_addr, "Lumberjack TLS connection established"); let intake = Intake { pipeline: &pipeline, metrics: &metrics, acks: &acks, peer_addr }; handle_connection(accept.clock(BufReader::new(tls_stream)), intake, conn_shutdown).await; }); diff --git a/src/server/otlp/mod.rs b/src/server/otlp/mod.rs index c7733fa..25fa0f7 100644 --- a/src/server/otlp/mod.rs +++ b/src/server/otlp/mod.rs @@ -29,6 +29,7 @@ use std::net::SocketAddr; use std::sync::Arc; use std::time::Duration; +use scalo::tls::ServerTlsOptions; use scalo::transport::grpc::sender_deadline; use tokio_util::sync::CancellationToken; use tonic::service::interceptor::InterceptedService; @@ -42,10 +43,12 @@ use crate::metrics::Metrics; use crate::pipeline::{Acks, BatchOutcome, PipelineState}; use crate::server::auth::{AuthMode, TokenAuth, grpc_auth_interceptor, principal_or_listener}; use crate::server::content_encoding; -use crate::server::grpc::{count_connections, count_oversize, count_unsupported_encoding}; +use crate::server::grpc::{ + count_connections, count_oversize, count_unsupported_encoding, handshake_connections, +}; use crate::server::http::ConnectionLimits; use crate::server::http::{Accept, create_auth_state}; -use crate::server::tls::{Handshake, TlsCertProvider, build_tls_acceptor, uses_secrets}; +use crate::server::tls::{Handshake, grpc_options, load_server_tls}; use crate::server::traits::{BoundAddr, Listeners, ProtocolHandler}; use crate::tenancy::{Ingest, Principal}; use convert::OtlpMode; @@ -352,13 +355,7 @@ async fn run_grpc_server( let shared_config = pipeline.shared_config(); let service = OtlpService::new(pipeline, metrics.clone(), mode, acks); - // Build TLS config if enabled - let tls_config = if config.tls.enabled { - let identity = super::tls::build_grpc_tls_config(&config.tls).await?; - Some(identity) - } else { - None - }; + let tls = load_server_tls(&config.tls, &pipeline_server.connections, grpc_options()).await?; // Build auth interceptor if configured. // @@ -385,14 +382,7 @@ async fn run_grpc_server( } let limits = ConnectionLimits::new(&pipeline_server.connections); - let mut builder = crate::server::grpc::keep_alive(tonic::transport::Server::builder(), &limits); - - if let Some(tls) = tls_config { - builder = builder - .tls_config(tls) - .map_err(|e| Error::Tls(format!("OTLP gRPC TLS error: {e}")))?; - } - let mut builder = builder + let mut builder = crate::server::grpc::keep_alive(tonic::transport::Server::builder(), &limits) .layer(count_oversize(metrics.clone(), TRANSPORT)) .layer(count_unsupported_encoding(metrics.clone(), TRANSPORT)); @@ -436,15 +426,25 @@ async fn run_grpc_server( .with_nodelay(Some(true)); let _serving = bound.publish(&incoming.local_addr()); - info!(addr = %addr, mode = ?mode, "OTLP gRPC server listening"); - - router - .serve_with_incoming_shutdown( - count_connections(incoming, metrics, TRANSPORT, limits), - shutdown.cancelled_owned(), - ) - .await - .map_err(|e| Error::Server(format!("OTLP gRPC server error: {e}")))?; + info!(addr = %addr, mode = ?mode, tls = tls.is_some(), "OTLP gRPC server listening"); + + let connections = count_connections(incoming, metrics.clone(), TRANSPORT, limits); + let served = match tls { + Some(tls) => { + router + .serve_with_incoming_shutdown( + handshake_connections(connections, tls, metrics), + shutdown.cancelled_owned(), + ) + .await + } + None => { + router + .serve_with_incoming_shutdown(connections, shutdown.cancelled_owned()) + .await + } + }; + served.map_err(|e| Error::Server(format!("OTLP gRPC server error: {e}")))?; info!("OTLP gRPC server stopped"); Ok(()) @@ -619,35 +619,22 @@ async fn run_http_server( .await .map_err(|e| Error::Server(format!("failed to bind OTLP HTTP: {e}")))?; - // Same TLS wiring as the Splunk HEC listener, over the same `tls:` block - // the gRPC endpoint uses. - let tls_provider = if config.tls.enabled && uses_secrets(&config.tls) { - let provider = TlsCertProvider::new(config.tls.clone()).await?; - provider.start_refresh_task(); - Some(provider) - } else { - None - }; - let tls_acceptor = if tls_provider.is_some() { - None - } else { - build_tls_acceptor(&config.tls)? - }; + // The same `tls:` block the gRPC endpoint uses. + let tls = load_server_tls( + &config.tls, + &server.connections, + ServerTlsOptions::default(), + ) + .await?; let accept = Accept::new(&server, metrics, TRANSPORT); - let acceptor_handle = if let Some(ref provider) = tls_provider { - Some(provider.acceptor_handle()) - } else { - tls_acceptor.map(|a| Arc::new(parking_lot::RwLock::new(a))) - }; - // Published once TLS is ready, so a failed TLS setup never reads as serving. let _serving = bound.publish(&listener.local_addr()); - if let Some(handle) = acceptor_handle { + if let Some(tls) = tls { info!(addr = %addr, mode = ?mode, tls = true, "OTLP HTTP server listening"); - crate::server::http::run_tls_server(listener, app, handle, accept, shutdown).await?; + crate::server::http::run_tls_server(listener, app, tls, accept, shutdown).await?; } else { info!(addr = %addr, mode = ?mode, tls = false, "OTLP HTTP server listening"); // The shared accept loop, not `axum::serve`: it runs the IP filter and diff --git a/src/server/prometheus_rw/mod.rs b/src/server/prometheus_rw/mod.rs index 13acf4c..05e7523 100644 --- a/src/server/prometheus_rw/mod.rs +++ b/src/server/prometheus_rw/mod.rs @@ -34,6 +34,7 @@ use axum::response::{IntoResponse, Response}; use axum::routing::post; use axum::{Extension, Router}; use prost::Message; +use scalo::tls::ServerTlsOptions; use tokio::net::TcpListener; use tokio_util::sync::CancellationToken; use tracing::{debug, info, warn}; @@ -45,7 +46,7 @@ use crate::pipeline::{Acks, PipelineState}; use crate::server::auth::{TokenAuth, principal_or_listener}; use crate::server::budget::Shape; use crate::server::http::{Accept, create_auth_state}; -use crate::server::tls::{Handshake, TlsCertProvider, build_tls_acceptor, uses_secrets}; +use crate::server::tls::{Handshake, load_server_tls}; use crate::server::traits::{BoundAddr, ProtocolHandler}; use crate::tenancy::{Ingest, Principal}; @@ -210,34 +211,21 @@ async fn run_prometheus_rw_server( .await .map_err(|e| Error::Server(format!("Prometheus RW failed to bind: {e}")))?; - // TLS setup (same dual-path pattern as Splunk HEC) - let tls_provider = if config.tls.enabled && uses_secrets(&config.tls) { - let provider = TlsCertProvider::new(config.tls.clone()).await?; - provider.start_refresh_task(); - Some(provider) - } else { - None - }; - - let tls_acceptor = if tls_provider.is_some() { - None - } else { - build_tls_acceptor(&config.tls)? - }; + let tls = load_server_tls( + &config.tls, + &server.connections, + ServerTlsOptions::default(), + ) + .await?; let accept = Accept::new(&server, metrics, TRANSPORT); // Published once TLS is ready, so a failed TLS setup never reads as serving. let _serving = bound.publish(&listener.local_addr()); - if let Some(ref provider) = tls_provider { - let acceptor_handle = provider.acceptor_handle(); - info!(addr = %addr, tls = true, hot_reload = true, "Prometheus Remote Write server listening"); - crate::server::http::run_tls_server(listener, app, acceptor_handle, accept, shutdown).await - } else if let Some(acceptor) = tls_acceptor { - let acceptor_handle = Arc::new(parking_lot::RwLock::new(acceptor)); - info!(addr = %addr, tls = true, hot_reload = false, "Prometheus Remote Write server listening"); - crate::server::http::run_tls_server(listener, app, acceptor_handle, accept, shutdown).await + if let Some(tls) = tls { + info!(addr = %addr, tls = true, "Prometheus Remote Write server listening"); + crate::server::http::run_tls_server(listener, app, tls, accept, shutdown).await } else { info!(addr = %addr, tls = false, "Prometheus Remote Write server listening"); // The shared accept loop, not `axum::serve`: it runs the IP filter and diff --git a/src/server/splunk_hec/mod.rs b/src/server/splunk_hec/mod.rs index bd4442a..ea11c2f 100644 --- a/src/server/splunk_hec/mod.rs +++ b/src/server/splunk_hec/mod.rs @@ -30,6 +30,7 @@ use axum::http::{HeaderMap, StatusCode}; use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use axum::{Extension, Json, Router}; +use scalo::tls::ServerTlsOptions; use serde::Serialize; use tokio::net::TcpListener; use tokio_util::sync::CancellationToken; @@ -42,7 +43,7 @@ use crate::pipeline::{Acks, BatchOutcome, PipelineState}; use crate::server::auth::{TokenAuth, principal_or_listener}; use crate::server::content_encoding::{self, EncodingError}; use crate::server::http::{Accept, create_auth_state}; -use crate::server::tls::{Handshake, TlsCertProvider, build_tls_acceptor, uses_secrets}; +use crate::server::tls::{Handshake, load_server_tls}; use crate::server::traits::{BoundAddr, ProtocolHandler}; use crate::tenancy::{Ingest, Principal}; @@ -356,34 +357,21 @@ async fn run_hec_server( .await .map_err(|e| Error::Server(format!("HEC failed to bind: {e}")))?; - // TLS setup (same pattern as HTTP handler) - let tls_provider = if config.tls.enabled && uses_secrets(&config.tls) { - let provider = TlsCertProvider::new(config.tls.clone()).await?; - provider.start_refresh_task(); - Some(provider) - } else { - None - }; - - let tls_acceptor = if tls_provider.is_some() { - None - } else { - build_tls_acceptor(&config.tls)? - }; + let tls = load_server_tls( + &config.tls, + &server.connections, + ServerTlsOptions::default(), + ) + .await?; let accept = Accept::new(&server, metrics, TRANSPORT); // Published once TLS is ready, so a failed TLS setup never reads as serving. let _serving = bound.publish(&listener.local_addr()); - if let Some(ref provider) = tls_provider { - let acceptor_handle = provider.acceptor_handle(); - info!(addr = %addr, tls = true, hot_reload = true, "Splunk HEC server listening"); - crate::server::http::run_tls_server(listener, app, acceptor_handle, accept, shutdown).await - } else if let Some(acceptor) = tls_acceptor { - let acceptor_handle = Arc::new(parking_lot::RwLock::new(acceptor)); - info!(addr = %addr, tls = true, hot_reload = false, "Splunk HEC server listening"); - crate::server::http::run_tls_server(listener, app, acceptor_handle, accept, shutdown).await + if let Some(tls) = tls { + info!(addr = %addr, tls = true, "Splunk HEC server listening"); + crate::server::http::run_tls_server(listener, app, tls, accept, shutdown).await } else { info!(addr = %addr, tls = false, "Splunk HEC server listening"); // The shared accept loop, not `axum::serve`: it runs the IP filter and diff --git a/src/server/syslog/mod.rs b/src/server/syslog/mod.rs index 2f85329..52c7f20 100644 --- a/src/server/syslog/mod.rs +++ b/src/server/syslog/mod.rs @@ -24,6 +24,7 @@ use std::net::SocketAddr; use std::sync::Arc; use std::time::Duration; +use scalo::tls::ServerTlsOptions; use tokio_util::sync::CancellationToken; use tracing::info; @@ -149,13 +150,15 @@ impl ProtocolHandler for SyslogHandler { .tls_bind_address .parse() .map_err(|e| Error::Config(format!("invalid syslog TLS bind address: {e}")))?; - // The config asks for a TLS listener, so no acceptor fails the start. - let acceptor = super::tls::build_tls_acceptor_async(&self.config.tls) - .await? - .ok_or_else(|| { - Error::Tls("syslog TLS is enabled but no TLS acceptor was built".into()) - })?; - Some((tls_addr, acceptor)) + // The config asks for a TLS listener, so no TLS fails the start. + let tls = super::tls::load_server_tls( + &self.config.tls, + &self.pipeline.config().server.connections, + ServerTlsOptions::default(), + ) + .await? + .ok_or_else(|| Error::Tls("syslog TLS is enabled but no TLS was loaded".into()))?; + Some((tls_addr, tls)) } else { None }; @@ -201,8 +204,8 @@ impl ProtocolHandler for SyslogHandler { listeners.spawn("syslog UDP", udp.run(shutdown.clone())); let plain = tcp_listener(tcp_addr, None, "TCP", self.tcp_bound.clone()); listeners.spawn("syslog TCP", plain.run(shutdown.clone())); - if let Some((tls_addr, acceptor)) = tls { - let secure = tcp_listener(tls_addr, Some(acceptor), "TLS", self.tls_bound.clone()); + if let Some((tls_addr, tls)) = tls { + let secure = tcp_listener(tls_addr, Some(tls), "TLS", self.tls_bound.clone()); listeners.spawn("syslog TLS", secure.run(shutdown.clone())); } diff --git a/src/server/syslog/tcp.rs b/src/server/syslog/tcp.rs index 9999215..a6b462e 100644 --- a/src/server/syslog/tcp.rs +++ b/src/server/syslog/tcp.rs @@ -23,12 +23,14 @@ use bytes::Bytes; use socket2::{SockRef, TcpKeepalive}; use tokio::io::{AsyncRead, AsyncWrite}; use tokio::net::{TcpListener, TcpStream}; -use tokio::time::{Instant, timeout}; +use tokio::time::Instant; use tokio_stream::StreamExt; use tokio_util::codec::{Decoder, FramedRead}; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; +use scalo::tls::ServerTls; + use super::convert::Converter; use super::framing::SyslogFrameDecoder; use super::{TRANSPORT, decode_lossy, listener_ingest}; @@ -39,6 +41,7 @@ use crate::server::frame_clock::{Clocked, Framing}; use crate::server::hold::{Held, hold}; use crate::server::http::{Accept, ConnectionLimits}; use crate::server::ip_filter::IpFilter; +use crate::server::tls::handshake; use crate::server::traits::BoundAddr; /// The idle timer's period with idle closing off, when it is never polled. @@ -59,7 +62,7 @@ pub(super) struct TcpListenerTask { pub(super) bind_addr: SocketAddr, pub(super) pipeline: Arc, pub(super) metrics: Arc, - pub(super) tls: Option, + pub(super) tls: Option, pub(super) connection: Connection, pub(super) label: &'static str, pub(super) ip_filter: IpFilter, @@ -129,25 +132,14 @@ impl TcpListenerTask { if !accept.first_byte(&stream).await { return; } - let Some(acceptor) = tls else { + let Some(tls) = tls else { reader.read(accept.clock(stream)).await; return; }; - let handshake = timeout(accept.limits.tls_handshake, acceptor.accept(stream)).await; - let tls_stream = match handshake { - Ok(Ok(s)) => s, - Ok(Err(e)) => { - reader.metrics.inc_tls_handshake_failure(); - debug!(peer = %peer_addr, error = %e, "Syslog TLS handshake failed"); - return; - } - Err(_) => { - reader.metrics.inc_tls_handshake_failure(); - warn!(peer = %peer_addr, "Syslog TLS handshake timeout"); - return; - } + let Some(tls_stream) = handshake(&tls, stream, peer_addr, &reader.metrics).await + else { + return; }; - debug!(peer = %peer_addr, "Syslog TLS connection established"); reader.read(accept.clock(tls_stream)).await; }); } diff --git a/src/server/tls.rs b/src/server/tls.rs index cf9303b..4a636a7 100644 --- a/src/server/tls.rs +++ b/src/server/tls.rs @@ -1,155 +1,43 @@ // Project: dfe-receiver // File: src/server/tls.rs -// Purpose: TLS configuration and utilities +// Purpose: Every listener's TLS: scalo ServerTls load and handshake // Language: Rust // // License: BUSL-1.1 // Copyright: (c) 2026 HYPERI PTY LIMITED -//! TLS configuration for HTTP and gRPC servers. +//! TLS for every receiver listener. //! -//! Supports TLS termination and mTLS client certificate validation. -//! Certificates come from local files, or from any `provider:path[:key]` spec -//! [`crate::secrets`] reads -- an OpenBao field (`vault:`) or a mounted -//! Kubernetes Secret (`file:`). +//! A listener's `tls:` block loads into a scalo [`ServerTls`]. It reads the +//! certificate, key and client CA from files or from [`crate::secrets`] +//! specs (an OpenBao field with `vault:`, a mounted Kubernetes Secret with +//! `file:`), re-reads them every `tls.refresh_interval_secs`, and runs the +//! handshake. A rotated certificate is served from the next handshake on every +//! listener built from it, with no restart. -use std::fs::File; -use std::io::{BufReader, Cursor}; -use std::path::Path; -use std::sync::Arc; +use std::path::PathBuf; -use rustls::pki_types::pem::PemObject; -use rustls::pki_types::{CertificateDer, PrivateKeyDer}; -use rustls::server::WebPkiClientVerifier; -use rustls::{RootCertStore, ServerConfig}; -use tokio_rustls::TlsAcceptor; -use tracing::{debug, info}; +use scalo::tls::{ClientAuth, ServerTls, ServerTlsConfig, ServerTlsOptions}; +use tokio::io::{AsyncRead, AsyncWrite}; +use tokio_rustls::server::TlsStream; +use tracing::debug; -use scalo::logger::security::{self, SecurityOutcome}; - -use crate::config::TlsConfig; +use crate::config::{ConnectionsConfig, TlsConfig}; use crate::error::{Error, Result}; +use crate::metrics::Metrics; use crate::secrets; -/// Load certificates from a PEM file. -fn load_certs_from_file(path: &Path) -> Result>> { - let file = File::open(path) - .map_err(|e| Error::Tls(format!("failed to open cert file {}: {e}", path.display())))?; - let mut reader = BufReader::new(file); - - CertificateDer::pem_reader_iter(&mut reader) - .collect::, _>>() - .map_err(|e| { - Error::Tls(format!( - "failed to parse certs from {}: {e}", - path.display() - )) - }) -} - -/// Load certificates from PEM bytes. -fn load_certs_from_bytes(pem_data: &[u8], source: &str) -> Result>> { - let mut reader = Cursor::new(pem_data); - - CertificateDer::pem_reader_iter(&mut reader) - .collect::, _>>() - .map_err(|e| Error::Tls(format!("failed to parse certs from {source}: {e}"))) -} - -/// Load a private key from a PEM file. -fn load_private_key_from_file(path: &Path) -> Result> { - let file = File::open(path) - .map_err(|e| Error::Tls(format!("failed to open key file {}: {e}", path.display())))?; - let mut reader = BufReader::new(file); - - PrivateKeyDer::from_pem_reader(&mut reader).map_err(|e| { - Error::Tls(format!( - "failed to load private key from {}: {e}", - path.display() - )) - }) -} - -/// Load a private key from PEM bytes. -fn load_private_key_from_bytes(pem_data: &[u8], source: &str) -> Result> { - let mut reader = Cursor::new(pem_data); - - PrivateKeyDer::from_pem_reader(&mut reader) - .map_err(|e| Error::Tls(format!("failed to load private key from {source}: {e}"))) -} - -/// Load CA certificates into a root store from a file. -fn load_ca_certs_from_file(path: &Path) -> Result { - let ca_certs = load_certs_from_file(path)?; - build_root_store(ca_certs, &path.display().to_string()) -} - -/// Load CA certificates into a root store from bytes. -fn load_ca_certs_from_bytes(pem_data: &[u8], source: &str) -> Result { - let ca_certs = load_certs_from_bytes(pem_data, source)?; - build_root_store(ca_certs, source) -} - -/// Build a root certificate store from certificates. -fn build_root_store(ca_certs: Vec>, source: &str) -> Result { - let mut root_store = RootCertStore::empty(); - - for cert in ca_certs { - root_store - .add(cert) - .map_err(|e| Error::Tls(format!("failed to add CA cert: {e}")))?; - } - - if root_store.is_empty() { - return Err(Error::Tls(format!( - "no valid CA certificates found in {source}" - ))); - } - - Ok(root_store) -} - -/// Make aws-lc-rs the process's rustls provider, unless one is installed already. +/// The mode `s` names, `None` for a value that names none. /// -/// rustls picks a provider for itself only when exactly one is compiled in. A -/// dependency that adds ring alongside aws-lc-rs would otherwise panic the first -/// `ServerConfig::builder()`, which is the one tonic calls too. `main` calls this -/// at start, and every builder here calls it again for callers that skip `main`. -pub fn install_crypto_provider() { - static INSTALL: std::sync::Once = std::sync::Once::new(); - INSTALL.call_once(|| { - // Err means a provider is installed already, and rustls uses that one. - let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); - }); -} - -/// Client authentication mode. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ClientAuth { - /// No client certificate required. - None, - /// Client certificate optional (validated if provided). - Optional, - /// Client certificate required. - Required, -} - -impl ClientAuth { - /// The mode `s` names, `None` for a value that names none. - pub fn parse(s: &str) -> Option { - match s.to_ascii_lowercase().as_str() { - "none" => Some(Self::None), - "required" | "require" => Some(Self::Required), - "optional" | "request" => Some(Self::Optional), - _ => None, - } - } - - /// Parse from string. Startup refuses a value [`parse`](Self::parse) does - /// not know, so the fallback to no client certificate is never reached - /// from a validated config. - pub fn from_str(s: &str) -> Self { - Self::parse(s).unwrap_or(Self::None) +/// Case-insensitive, and `require` and `request` read as `required` and +/// `optional`. +#[must_use] +pub fn parse_client_auth(s: &str) -> Option { + match s.to_ascii_lowercase().as_str() { + "none" => Some(ClientAuth::None), + "required" | "require" => Some(ClientAuth::Required), + "optional" | "request" => Some(ClientAuth::Optional), + _ => None, } } @@ -170,7 +58,7 @@ impl Handshake { #[must_use] pub fn of(tls: &TlsConfig) -> Self { if tls.enabled - && ClientAuth::parse(&tls.client_auth) == Some(ClientAuth::Required) + && parse_client_auth(&tls.client_auth) == Some(ClientAuth::Required) && (tls.ca_file.is_some() || tls.ca_secret.is_some()) { Self::RequiresClientCertificate @@ -180,334 +68,91 @@ impl Handshake { } } -/// Build a TLS acceptor from configuration (sync version for backwards compatibility). -/// -/// Use `build_tls_acceptor_async` if loading from secrets. -pub fn build_tls_acceptor(config: &TlsConfig) -> Result> { - if !config.enabled { - return Ok(None); - } - install_crypto_provider(); - - // If secrets are configured, fail - caller should use async version - if config.cert_secret.is_some() || config.key_secret.is_some() || config.ca_secret.is_some() { - return Err(Error::Tls( - "TLS secrets configured - use build_tls_acceptor_async instead".into(), - )); +/// What a gRPC listener's handshake offers: ALPN `h2`, which gRPC clients +/// require and tonic offered itself when it ran the handshake. +#[must_use] +pub fn grpc_options() -> ServerTlsOptions { + ServerTlsOptions { + alpn_protocols: vec![b"h2".to_vec()], + ..ServerTlsOptions::default() } - - let cert_path = config - .cert_file - .as_ref() - .ok_or_else(|| Error::Tls("TLS enabled but cert_file not specified".into()))?; - - let key_path = config - .key_file - .as_ref() - .ok_or_else(|| Error::Tls("TLS enabled but key_file not specified".into()))?; - - let client_auth = ClientAuth::from_str(&config.client_auth); - - // Check for ca_file early if client auth is required - if client_auth != ClientAuth::None && config.ca_file.is_none() { - return Err(Error::Tls( - "client_auth requires ca_file or ca_secret".into(), - )); - } - - let certs = load_certs_from_file(Path::new(cert_path))?; - let key = load_private_key_from_file(Path::new(key_path))?; - - let tls_config = match client_auth { - ClientAuth::None => ServerConfig::builder() - .with_no_client_auth() - .with_single_cert(certs, key) - .map_err(|e| Error::Tls(format!("failed to build TLS config: {e}")))?, - - ClientAuth::Optional | ClientAuth::Required => { - let ca_path = config - .ca_file - .as_ref() - .ok_or_else(|| Error::Tls("client_auth requires ca_file".into()))?; - - let root_store = load_ca_certs_from_file(Path::new(ca_path))?; - - let verifier = if client_auth == ClientAuth::Required { - WebPkiClientVerifier::builder(Arc::new(root_store)) - .build() - .map_err(|e| Error::Tls(format!("failed to build client verifier: {e}")))? - } else { - WebPkiClientVerifier::builder(Arc::new(root_store)) - .allow_unauthenticated() - .build() - .map_err(|e| Error::Tls(format!("failed to build client verifier: {e}")))? - }; - - ServerConfig::builder() - .with_client_cert_verifier(verifier) - .with_single_cert(certs, key) - .map_err(|e| Error::Tls(format!("failed to build TLS config: {e}")))? - } - }; - - Ok(Some(TlsAcceptor::from(Arc::new(tls_config)))) } -/// Build a TLS acceptor from configuration with secret manager support. +/// Load a listener's `tls:` block, `None` when `tls.enabled` is off. /// -/// Supports loading certificates from: -/// - Local files (cert_file, key_file, ca_file) -/// - Secret managers (cert_secret, key_secret, ca_secret) +/// The handshake timeout is `connections.tls_handshake_timeout_ms`. Call inside +/// the tokio runtime: the reload task starts here and stops once the last clone +/// of the returned `ServerTls` drops. /// -/// Secret format: "provider:path:key" (e.g., "vault:secret/tls:cert") -pub async fn build_tls_acceptor_async(config: &TlsConfig) -> Result> { - if !config.enabled { +/// # Errors +/// +/// A secret spec naming no provider, a certificate, key or CA set both as a +/// file and as a secret or set neither, a client CA missing for `optional` or +/// `required`, an unreadable or unparsable source, and a key that does not +/// match its certificate. +pub async fn load_server_tls( + tls: &TlsConfig, + connections: &ConnectionsConfig, + options: ServerTlsOptions, +) -> Result> { + if !tls.enabled { return Ok(None); } - install_crypto_provider(); - - let client_auth = ClientAuth::from_str(&config.client_auth); - - // Load certificate (secret takes precedence over file) - let certs = if let Some(ref secret) = config.cert_secret { - info!(source = %secret, "Loading TLS certificate from secret"); - let pem_data = secrets::read_bytes(secret).await?; - load_certs_from_bytes(&pem_data, secret)? - } else if let Some(ref path) = config.cert_file { - debug!(path = %path, "Loading TLS certificate from file"); - load_certs_from_file(Path::new(path))? - } else { - return Err(Error::Tls( - "TLS enabled but neither cert_file nor cert_secret specified".into(), - )); - }; - - // Load private key (secret takes precedence over file) - let key = if let Some(ref secret) = config.key_secret { - info!(source = %secret, "Loading TLS private key from secret"); - let pem_data = secrets::read_bytes(secret).await?; - load_private_key_from_bytes(&pem_data, secret)? - } else if let Some(ref path) = config.key_file { - debug!(path = %path, "Loading TLS private key from file"); - load_private_key_from_file(Path::new(path))? - } else { - return Err(Error::Tls( - "TLS enabled but neither key_file nor key_secret specified".into(), - )); - }; - - // Build TLS config based on client auth mode - let tls_config = match client_auth { - ClientAuth::None => ServerConfig::builder() - .with_no_client_auth() - .with_single_cert(certs, key) - .map_err(|e| Error::Tls(format!("failed to build TLS config: {e}")))?, - - ClientAuth::Optional | ClientAuth::Required => { - // Load CA certificate (secret takes precedence over file) - let root_store = if let Some(ref secret) = config.ca_secret { - info!(source = %secret, "Loading CA certificate from secret"); - let pem_data = secrets::read_bytes(secret).await?; - load_ca_certs_from_bytes(&pem_data, secret)? - } else if let Some(ref path) = config.ca_file { - debug!(path = %path, "Loading CA certificate from file"); - load_ca_certs_from_file(Path::new(path))? - } else { - return Err(Error::Tls( - "client_auth requires ca_file or ca_secret".into(), - )); - }; - - let verifier = if client_auth == ClientAuth::Required { - WebPkiClientVerifier::builder(Arc::new(root_store)) - .build() - .map_err(|e| Error::Tls(format!("failed to build client verifier: {e}")))? - } else { - WebPkiClientVerifier::builder(Arc::new(root_store)) - .allow_unauthenticated() - .build() - .map_err(|e| Error::Tls(format!("failed to build client verifier: {e}")))? - }; - - ServerConfig::builder() - .with_client_cert_verifier(verifier) - .with_single_cert(certs, key) - .map_err(|e| Error::Tls(format!("failed to build TLS config: {e}")))? - } - }; - - info!("TLS acceptor built successfully"); - Ok(Some(TlsAcceptor::from(Arc::new(tls_config)))) -} - -/// Check if TLS config uses secrets (requires async loading). -pub fn uses_secrets(config: &TlsConfig) -> bool { - config.cert_secret.is_some() || config.key_secret.is_some() || config.ca_secret.is_some() -} - -/// Build a tonic `ServerTlsConfig` for the gRPC server. -/// -/// Loads certificates from files or secret managers and returns -/// a `ServerTlsConfig` suitable for `tonic::transport::Server::builder().tls_config()`. -pub async fn build_grpc_tls_config( - config: &TlsConfig, -) -> Result { - install_crypto_provider(); - // Load certificate (secret takes precedence over file) - let cert_pem = if let Some(ref secret) = config.cert_secret { - info!(source = %secret, "Loading gRPC TLS certificate from secret"); - secrets::read_bytes(secret).await? - } else if let Some(ref path) = config.cert_file { - debug!(path = %path, "Loading gRPC TLS certificate from file"); - std::fs::read(path) - .map_err(|e| Error::Tls(format!("failed to read cert file {path}: {e}")))? - } else { - return Err(Error::Tls( - "gRPC TLS enabled but neither cert_file nor cert_secret specified".into(), - )); - }; - - // Load private key (secret takes precedence over file) - let key_pem = if let Some(ref secret) = config.key_secret { - info!(source = %secret, "Loading gRPC TLS private key from secret"); - secrets::read_bytes(secret).await? - } else if let Some(ref path) = config.key_file { - debug!(path = %path, "Loading gRPC TLS private key from file"); - std::fs::read(path) - .map_err(|e| Error::Tls(format!("failed to read key file {path}: {e}")))? - } else { - return Err(Error::Tls( - "gRPC TLS enabled but neither key_file nor key_secret specified".into(), - )); - }; - - let identity = tonic::transport::Identity::from_pem(cert_pem, key_pem); - let mut tls_config = tonic::transport::ServerTlsConfig::new().identity(identity); - - // Load CA for client certificate verification (mTLS) - let client_auth = ClientAuth::from_str(&config.client_auth); - if client_auth != ClientAuth::None { - let ca_pem = if let Some(ref secret) = config.ca_secret { - info!(source = %secret, "Loading gRPC CA certificate from secret"); - secrets::read_bytes(secret).await? - } else if let Some(ref path) = config.ca_file { - debug!(path = %path, "Loading gRPC CA certificate from file"); - std::fs::read(path) - .map_err(|e| Error::Tls(format!("failed to read CA file {path}: {e}")))? - } else { - return Err(Error::Tls( - "gRPC client_auth requires ca_file or ca_secret".into(), - )); - }; - - let ca_cert = tonic::transport::Certificate::from_pem(ca_pem); - tls_config = tls_config.client_ca_root(ca_cert); + for spec in [&tls.cert_secret, &tls.key_secret, &tls.ca_secret] + .into_iter() + .flatten() + { + secrets::check_spec(spec) + .map_err(|reason| Error::Config(format!("secret source: {reason}")))?; } - - info!("gRPC TLS config built successfully"); - Ok(tls_config) + let client_auth = parse_client_auth(&tls.client_auth).ok_or_else(|| { + Error::Config(format!( + "tls.client_auth is '{}', which is not one of none, optional, required", + tls.client_auth + )) + })?; + let config = ServerTlsConfig { + cert_file: tls.cert_file.as_ref().map(PathBuf::from), + cert_secret: tls.cert_secret.clone(), + key_file: tls.key_file.as_ref().map(PathBuf::from), + key_secret: tls.key_secret.clone(), + ca_file: tls.ca_file.as_ref().map(PathBuf::from), + ca_secret: tls.ca_secret.clone(), + client_auth, + reload_interval_secs: tls.refresh_interval_secs, + handshake_timeout_ms: connections.tls_handshake_timeout_ms, + }; + // Boxed for the reason `secrets::read` boxes: the AWS provider's future, awaited + // inline, overflows rustc's layout query depth in every listener's async block. + Box::pin(ServerTls::load_with(&config, options)) + .await + .map(Some) + .map_err(|e| Error::Tls(e.to_string())) } -/// TLS certificate provider with background hot-reload. +/// Run the TLS handshake on `io` from `peer`, counting a failure on `metrics`. /// -/// Follows the same pattern as `BearerTokenProvider` in `auth.rs`. -/// Wraps a `TlsAcceptor` behind a `RwLock` and periodically reloads -/// certificates from secret managers. -pub struct TlsCertProvider { - acceptor: Arc>, - config: TlsConfig, - shutdown: tokio_util::sync::CancellationToken, -} - -impl TlsCertProvider { - /// Create a new TLS cert provider and perform initial certificate load. - pub async fn new(config: TlsConfig) -> Result { - let acceptor = build_tls_acceptor_async(&config) - .await? - .ok_or_else(|| Error::Tls("TLS provider created but TLS is disabled".into()))?; - - Ok(Self { - acceptor: Arc::new(parking_lot::RwLock::new(acceptor)), - config, - shutdown: tokio_util::sync::CancellationToken::new(), - }) - } - - /// Get the shared acceptor handle for the HTTP server accept loop. - /// - /// The server clones the `TlsAcceptor` from behind the `RwLock` per connection. - /// This is cheap because `TlsAcceptor` wraps `Arc`. - pub fn acceptor_handle(&self) -> Arc> { - self.acceptor.clone() - } - - /// Start background certificate refresh task. - /// - /// Periodically reloads certificates from secret managers and swaps - /// the TLS acceptor behind the `RwLock`. Existing connections are unaffected; - /// new connections use the updated certificates. A zero - /// `refresh_interval_secs` refreshes never. - pub fn start_refresh_task(&self) { - let acceptor = self.acceptor.clone(); - let config = self.config.clone(); - let shutdown = self.shutdown.clone(); - let interval = std::time::Duration::from_secs(config.refresh_interval_secs); - // A zero-period timer panics the task it runs in. - if interval.is_zero() { - return; +/// `None` for a handshake that failed or timed out. [`ServerTls::accept`] has +/// logged it and counted it on scalo's own metrics, and the connection is +/// dropped. +pub(crate) async fn handshake( + tls: &ServerTls, + io: IO, + peer: std::net::SocketAddr, + metrics: &Metrics, +) -> Option> +where + IO: AsyncRead + AsyncWrite + Unpin, +{ + match tls.accept(io, Some(peer)).await { + Ok(stream) => { + debug!(peer = %peer, "TLS connection established"); + Some(stream) + } + Err(_) => { + metrics.inc_tls_handshake_failure(); + None } - - tokio::spawn(async move { - let mut ticker = tokio::time::interval(interval); - ticker.tick().await; // Skip immediate tick - - loop { - tokio::select! { - _ = shutdown.cancelled() => { - info!("TLS cert refresh task stopping"); - break; - } - _ = ticker.tick() => { - match build_tls_acceptor_async(&config).await { - Ok(Some(new_acceptor)) => { - *acceptor.write() = new_acceptor; - info!("TLS certificates refreshed successfully"); - security::tls_event("cert_refresh", SecurityOutcome::Success, None, None); - } - Ok(None) => { - tracing::error!("TLS refresh returned None (TLS disabled?)"); - security::tls_event( - "cert_refresh", - SecurityOutcome::Error, - Some("tls_disabled_after_refresh"), - None, - ); - } - Err(e) => { - tracing::error!(error = %e, "Failed to refresh TLS certificates"); - security::tls_event( - "cert_refresh", - SecurityOutcome::Failure, - Some(&e.to_string()), - None, - ); - } - } - } - } - } - }); - } - - /// Shutdown the refresh task. - pub fn shutdown(&self) { - self.shutdown.cancel(); - } -} - -impl Drop for TlsCertProvider { - fn drop(&mut self) { - self.shutdown(); } } @@ -527,97 +172,117 @@ pub struct ClientCertInfo { mod tests { use super::*; - fn base_config() -> TlsConfig { + fn enabled() -> TlsConfig { TlsConfig { - enabled: false, - cert_file: None, - key_file: None, - ca_file: None, - client_auth: "none".to_string(), - cert_secret: None, - key_secret: None, - ca_secret: None, - refresh_interval_secs: 3600, + enabled: true, + cert_file: Some("/etc/ssl/receiver.crt".into()), + key_file: Some("/etc/ssl/receiver.key".into()), + ..TlsConfig::default() } } #[test] - fn test_client_auth_parsing() { - assert_eq!(ClientAuth::from_str("none"), ClientAuth::None); - assert_eq!(ClientAuth::from_str("optional"), ClientAuth::Optional); - assert_eq!(ClientAuth::from_str("required"), ClientAuth::Required); - assert_eq!(ClientAuth::from_str("REQUIRED"), ClientAuth::Required); - assert_eq!(ClientAuth::from_str("unknown"), ClientAuth::None); - assert_eq!(ClientAuth::parse("Require"), Some(ClientAuth::Required)); - assert_eq!(ClientAuth::parse("requried"), None); - assert_eq!(ClientAuth::parse(""), None); - } - - #[test] - fn test_tls_disabled() { - let config = base_config(); - - let result = build_tls_acceptor(&config); - assert!(result.is_ok()); - assert!(result.unwrap().is_none()); - } - - #[test] - fn test_tls_enabled_missing_cert() { - let mut config = base_config(); - config.enabled = true; - config.key_file = Some("/path/to/key".into()); - - let result = build_tls_acceptor(&config); - assert!(result.is_err()); - let err = result.err().unwrap(); - assert!(err.to_string().contains("cert_file")); - } - - #[test] - fn test_tls_enabled_missing_key() { - let mut config = base_config(); - config.enabled = true; - config.cert_file = Some("/path/to/cert".into()); - - let result = build_tls_acceptor(&config); - assert!(result.is_err()); - let err = result.err().unwrap(); - assert!(err.to_string().contains("key_file")); - } - - #[test] - fn test_mtls_missing_ca() { - let mut config = base_config(); - config.enabled = true; - config.cert_file = Some("/path/to/cert".into()); - config.key_file = Some("/path/to/key".into()); - config.client_auth = "required".to_string(); - - let result = build_tls_acceptor(&config); - assert!(result.is_err()); - let err = result.err().unwrap(); - assert!(err.to_string().contains("ca_file")); + fn client_auth_names_parse_case_insensitively() { + for (name, mode) in [ + ("none", Some(ClientAuth::None)), + ("optional", Some(ClientAuth::Optional)), + ("request", Some(ClientAuth::Optional)), + ("required", Some(ClientAuth::Required)), + ("REQUIRED", Some(ClientAuth::Required)), + ("Require", Some(ClientAuth::Required)), + ("requried", None), + ("unknown", None), + ("", None), + ] { + assert_eq!(parse_client_auth(name), mode, "{name:?}"); + } } - #[test] - fn test_uses_secrets() { - let mut config = base_config(); - assert!(!uses_secrets(&config)); - - config.cert_secret = Some("vault:secret/tls:cert".into()); - assert!(uses_secrets(&config)); + #[tokio::test] + async fn a_disabled_block_loads_nothing() { + let loaded = load_server_tls( + &TlsConfig::default(), + &ConnectionsConfig::default(), + ServerTlsOptions::default(), + ) + .await + .unwrap(); + assert!(loaded.is_none()); } - #[test] - fn test_sync_rejects_secrets() { - let mut config = base_config(); - config.enabled = true; - config.cert_secret = Some("vault:secret/tls:cert".into()); - config.key_file = Some("/path/to/key".into()); - - let result = build_tls_acceptor(&config); - assert!(result.is_err()); - assert!(result.err().unwrap().to_string().contains("async")); + /// Each refusal names the key the operator has to change. + #[tokio::test] + async fn a_block_that_cannot_be_served_is_refused_naming_the_key() { + let cases = [ + ( + "cert as file and secret", + TlsConfig { + cert_secret: Some("vault:secret/tls:cert".into()), + ..enabled() + }, + "cert_secret", + ), + ( + "key as file and secret", + TlsConfig { + key_secret: Some("file:/etc/ssl/receiver.key".into()), + ..enabled() + }, + "key_secret", + ), + ( + "no certificate", + TlsConfig { + cert_file: None, + ..enabled() + }, + "cert_file", + ), + ( + "no key", + TlsConfig { + key_file: None, + ..enabled() + }, + "key_file", + ), + ( + "client certificates with no CA", + TlsConfig { + client_auth: "required".into(), + ..enabled() + }, + "ca_file", + ), + ( + "a secret naming no provider", + TlsConfig { + cert_file: None, + cert_secret: Some("eyJhbGciOiJIUzI1NiJ9.pasted".into()), + ..enabled() + }, + "provider", + ), + ( + "an unknown client_auth", + TlsConfig { + client_auth: "requried".into(), + ..enabled() + }, + "client_auth", + ), + ]; + for (case, tls, key) in cases { + let err = load_server_tls( + &tls, + &ConnectionsConfig::default(), + ServerTlsOptions::default(), + ) + .await + .err() + .unwrap_or_else(|| panic!("{case} must be refused")) + .to_string(); + assert!(err.contains(key), "{case}: {err}"); + } } } diff --git a/src/server/webhook/mod.rs b/src/server/webhook/mod.rs index 76255ba..e7f884d 100644 --- a/src/server/webhook/mod.rs +++ b/src/server/webhook/mod.rs @@ -39,6 +39,7 @@ use axum::response::{IntoResponse, Response}; use axum::routing::post; use axum::{Json, Router}; use rustc_hash::FxHashMap; +use scalo::tls::ServerTlsOptions; use tokio::net::TcpListener; use tokio_util::sync::CancellationToken; use tracing::{debug, info, warn}; @@ -49,7 +50,7 @@ use crate::metrics::Metrics; use crate::pipeline::{Acks, PipelineState}; use crate::server::auth::record_auth_failure; use crate::server::http::{Accept, apply_request_limits, split_json_array}; -use crate::server::tls::{TlsCertProvider, build_tls_acceptor, uses_secrets}; +use crate::server::tls::load_server_tls; use crate::server::traits::{BoundAddr, ProtocolHandler}; use crate::tenancy::{Ingest, Principal}; use crate::validation::depth::{self, MAX_BATCH_DEPTH, MAX_PARSE_DEPTH}; @@ -334,42 +335,19 @@ async fn run_webhook_server( let accept = Accept::new(&config.server, metrics, TRANSPORT); - let tls_provider = if webhook.tls.enabled && uses_secrets(&webhook.tls) { - let provider = TlsCertProvider::new(webhook.tls.clone()).await?; - provider.start_refresh_task(); - Some(provider) - } else { - None - }; - let tls_acceptor = if tls_provider.is_some() { - None - } else { - build_tls_acceptor(&webhook.tls)? - }; + let tls = load_server_tls( + &webhook.tls, + &config.server.connections, + ServerTlsOptions::default(), + ) + .await?; // Published once TLS is ready, so a failed TLS setup never reads as serving. let _serving = bound.publish(&listener.local_addr()); - if let Some(ref provider) = tls_provider { - info!(addr = %addr, tls = true, hot_reload = true, "webhook server listening"); - crate::server::http::run_tls_server( - listener, - app, - provider.acceptor_handle(), - accept, - shutdown, - ) - .await - } else if let Some(acceptor) = tls_acceptor { - info!(addr = %addr, tls = true, hot_reload = false, "webhook server listening"); - crate::server::http::run_tls_server( - listener, - app, - Arc::new(parking_lot::RwLock::new(acceptor)), - accept, - shutdown, - ) - .await + if let Some(tls) = tls { + info!(addr = %addr, tls = true, "webhook server listening"); + crate::server::http::run_tls_server(listener, app, tls, accept, shutdown).await } else { info!(addr = %addr, tls = false, "webhook server listening"); crate::server::http::run_plain_server(listener, app, accept, shutdown).await diff --git a/tests/integration.rs b/tests/integration.rs index b8ff967..d53afe7 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -97,6 +97,8 @@ mod splunk_hec; mod syslog; #[path = "integration/tls_openssl.rs"] mod tls_openssl; +#[path = "integration/tls_reload.rs"] +mod tls_reload; #[path = "integration/untrusted_input.rs"] mod untrusted_input; #[path = "integration/vault_auth.rs"] diff --git a/tests/integration/tls_reload.rs b/tests/integration/tls_reload.rs new file mode 100644 index 0000000..e28b918 --- /dev/null +++ b/tests/integration/tls_reload.rs @@ -0,0 +1,552 @@ +// Project: dfe-receiver +// File: tests/integration/tls_reload.rs +// Purpose: A rotated certificate reaches every TLS listener family with no restart +// Language: Rust +// +// License: BUSL-1.1 +// Copyright: (c) 2026 HYPERI PTY LIMITED + +//! A certificate rotated on disk, or behind a `file:` secret, is served from the +//! next handshake on every TLS listener, with no restart. +//! +//! Each case starts one handler with `tls.refresh_interval_secs: 1`, reads the +//! certificate a real TLS client is served, replaces the certificate and key, +//! and waits for the client to be served the new one. The three families run +//! their own accept path: the hyper listeners (`/ingest`, webhook, Splunk HEC, +//! remote write), the raw TCP listeners (syslog, Fluent Forward, GELF, +//! Lumberjack) and the tonic listeners (Vector gRPC, OTLP). The tonic listeners +//! also answer a request over the handshake the receiver runs for them. + +#![allow(clippy::unwrap_used)] +#![allow(clippy::expect_used)] +#![allow(clippy::large_futures)] + +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; + +use dfe_receiver::config::{ + Config, SharedConfig, TlsConfig, WebhookAuthConfig, WebhookAuthMode, WebhookCallerConfig, +}; +use dfe_receiver::metrics::Metrics; +use dfe_receiver::pipeline::PipelineState; +use dfe_receiver::server::fluent::FluentHandler; +use dfe_receiver::server::gelf::GelfHandler; +use dfe_receiver::server::grpc::{GrpcVectorHandler, pb as grpc_pb}; +use dfe_receiver::server::http::HttpHandler; +use dfe_receiver::server::lumberjack::LumberjackHandler; +use dfe_receiver::server::prometheus_rw::PrometheusRwHandler; +use dfe_receiver::server::splunk_hec::SplunkHecHandler; +use dfe_receiver::server::syslog::SyslogHandler; +use dfe_receiver::server::traits::ProtocolHandler; +use dfe_receiver::server::webhook::WebhookHandler; +use rcgen::{ + BasicConstraints, CertificateParams, CertifiedIssuer, DnType, ExtendedKeyUsagePurpose, IsCa, + KeyPair, KeyUsagePurpose, +}; +use rustls::pki_types::pem::PemObject; +use rustls::pki_types::{CertificateDer, ServerName}; +use rustls::{ClientConfig, RootCertStore}; +use tempfile::TempDir; +use tokio::net::TcpStream; +use tokio_rustls::TlsConnector; +use tokio_util::sync::CancellationToken; + +use crate::common::bound_addr; + +/// How long a rotated certificate gets to reach a listener, which re-reads +/// every second. +const ROTATION_DEADLINE: Duration = Duration::from_secs(20); + +/// A throwaway CA that signs the certificates a case rotates between. +struct Ca { + issuer: CertifiedIssuer<'static, KeyPair>, +} + +/// A certificate and its key, signed by a [`Ca`]. +struct Leaf { + cert_pem: String, + key_pem: String, + der: CertificateDer<'static>, +} + +impl Ca { + fn new() -> Self { + let mut params = CertificateParams::new(Vec::::new()).unwrap(); + params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + params.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::CrlSign]; + params + .distinguished_name + .push(DnType::CommonName, "tls reload test ca"); + let issuer = CertifiedIssuer::self_signed(params, KeyPair::generate().unwrap()).unwrap(); + Self { issuer } + } + + /// A `localhost` server certificate named `name`. + fn server(&self, name: &str) -> Leaf { + let mut params = CertificateParams::new(vec!["localhost".to_string()]).unwrap(); + params.distinguished_name.push(DnType::CommonName, name); + params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth]; + let key = KeyPair::generate().unwrap(); + let cert = params.signed_by(&key, &self.issuer).unwrap(); + Leaf { + cert_pem: cert.pem(), + key_pem: key.serialize_pem(), + der: cert.der().clone(), + } + } + + /// A TLS client that trusts only this CA. + fn client(&self) -> TlsConnector { + let mut roots = RootCertStore::empty(); + for cert in CertificateDer::pem_slice_iter(self.issuer.pem().as_bytes()) { + roots.add(cert.unwrap()).unwrap(); + } + let provider = Arc::new(rustls::crypto::aws_lc_rs::default_provider()); + let config = ClientConfig::builder_with_provider(provider) + .with_safe_default_protocol_versions() + .unwrap() + .with_root_certificates(roots) + .with_no_client_auth(); + TlsConnector::from(Arc::new(config)) + } +} + +/// Replace `path` in one rename, so a reader never sees half a file. +fn write(path: &Path, contents: &str) { + let staged = path.with_extension("staged"); + std::fs::write(&staged, contents).unwrap(); + std::fs::rename(&staged, path).unwrap(); +} + +/// Where a listener's certificate and key come from. +#[derive(Debug, Clone, Copy)] +enum Source { + File, + /// A `file:` secret spec, read through the same resolver as `vault:`. + Secret, +} + +/// The certificate and key a case rotates, in a directory. +struct Files { + dir: TempDir, +} + +impl Files { + fn new(leaf: &Leaf) -> Self { + let files = Self { + dir: TempDir::new().unwrap(), + }; + files.serve(leaf); + files + } + + fn cert(&self) -> PathBuf { + self.dir.path().join("cert.pem") + } + + fn key(&self) -> PathBuf { + self.dir.path().join("key.pem") + } + + fn serve(&self, leaf: &Leaf) { + write(&self.cert(), &leaf.cert_pem); + write(&self.key(), &leaf.key_pem); + } + + /// A `tls:` block reading these files from `source`, re-read every second. + fn tls(&self, source: Source) -> TlsConfig { + let tls = TlsConfig { + enabled: true, + refresh_interval_secs: 1, + ..TlsConfig::default() + }; + match source { + Source::File => TlsConfig { + cert_file: Some(self.cert().display().to_string()), + key_file: Some(self.key().display().to_string()), + ..tls + }, + Source::Secret => TlsConfig { + cert_secret: Some(format!("file:{}", self.cert().display())), + key_secret: Some(format!("file:{}", self.key().display())), + ..tls + }, + } + } +} + +/// One TLS listener under test. +#[derive(Debug, Clone, Copy)] +enum Listener { + Http, + Webhook, + SplunkHec, + PrometheusRw, + Syslog, + Fluent, + Gelf, + Lumberjack, + Grpc, + #[cfg(feature = "otlp")] + Otlp, +} + +/// A handler started on free ports, and what stops it. +struct Running { + /// Each TLS listener it bound, by name. + tls: Vec<(&'static str, SocketAddr)>, + shutdown: CancellationToken, + /// Holds the webhook caller's secret for as long as it runs. + _scratch: TempDir, +} + +fn base_config() -> Config { + let mut config = Config::default(); + // The loader on its memory transport: accepted, sent nowhere, no broker. + config.destinations.default = "loader".into(); + config.loader.transport = "memory".to_string(); + config.server.bind_address = "127.0.0.1:0".to_string(); + config.server.auth.mode = "none".to_string(); + config +} + +impl Listener { + /// Start the listener with `tls` and wait for every listener it binds. + async fn start(self, tls: TlsConfig) -> Running { + let any = || "127.0.0.1:0".to_string(); + let scratch = TempDir::new().unwrap(); + let mut config = base_config(); + match self { + Self::Http => config.server.tls = tls, + Self::Webhook => { + let secret = scratch.path().join("webhook-secret"); + std::fs::write(&secret, "probe-secret").unwrap(); + // A caller delivers to a Kafka topic, so the config names a broker. + config.kafka.brokers = vec!["127.0.0.1:1".to_string()]; + config.webhook.enabled = true; + config.webhook.bind_address = Some(any()); + config.webhook.callers = vec![WebhookCallerConfig { + name: "probe".to_string(), + topic: "probe".to_string(), + auth: WebhookAuthConfig { + mode: WebhookAuthMode::Header, + secret_source: format!("file:{}", secret.display()), + header: "x-probe-secret".to_string(), + ..WebhookAuthConfig::default() + }, + ..WebhookCallerConfig::default() + }]; + config.webhook.tls = tls; + } + Self::SplunkHec => { + config.splunk_hec.enabled = true; + config.splunk_hec.bind_address = any(); + config.splunk_hec.auth.mode = "none".to_string(); + config.splunk_hec.tls = tls; + } + Self::PrometheusRw => { + config.prometheus_rw.enabled = true; + config.prometheus_rw.bind_address = any(); + config.prometheus_rw.auth.mode = "none".to_string(); + config.prometheus_rw.tls = tls; + } + Self::Syslog => { + config.syslog.enabled = true; + config.syslog.udp_bind_address = any(); + config.syslog.tcp_bind_address = any(); + config.syslog.tls_bind_address = any(); + config.syslog.accept_unauthenticated = true; + config.syslog.tls = tls; + } + Self::Fluent => { + config.fluent.enabled = true; + config.fluent.bind_address = any(); + config.fluent.accept_unauthenticated = true; + config.fluent.tls = tls; + } + Self::Gelf => { + config.gelf.enabled = true; + config.gelf.bind_address = any(); + config.gelf.accept_unauthenticated = true; + config.gelf.tls = tls; + } + Self::Lumberjack => { + config.lumberjack.enabled = true; + config.lumberjack.bind_address = any(); + config.lumberjack.accept_unauthenticated = true; + config.lumberjack.tls = tls; + } + Self::Grpc => { + config.grpc.enabled = true; + config.grpc.bind_address = any(); + config.grpc.tls = tls; + } + #[cfg(feature = "otlp")] + Self::Otlp => { + config.otlp.enabled = true; + config.otlp.grpc_bind_address = any(); + config.otlp.http_bind_address = any(); + config.otlp.auth.mode = "none".to_string(); + config.otlp.tls = tls; + } + } + let pipeline = Arc::new( + PipelineState::new(SharedConfig::new(config.clone()), CancellationToken::new()) + .await + .expect("build pipeline"), + ); + let metrics = Arc::new(Metrics::default()); + + // The handler, and the name and index of each of its TLS listeners. + let (handler, tls_listeners): (Box, Vec<(&'static str, usize)>) = + match self { + Self::Http => ( + Box::new(HttpHandler::new( + config.server.bind_address.clone(), + pipeline, + metrics, + )), + vec![("http", 0)], + ), + Self::Webhook => ( + Box::new(WebhookHandler::new(config.clone(), pipeline, metrics)), + vec![("webhook", 0)], + ), + Self::SplunkHec => ( + Box::new(SplunkHecHandler::new( + config.splunk_hec.clone(), + config.raw_capture_for(&config.splunk_hec.raw_capture), + pipeline, + metrics, + )), + vec![("splunk_hec", 0)], + ), + Self::PrometheusRw => ( + Box::new(PrometheusRwHandler::new( + config.prometheus_rw.clone(), + pipeline, + metrics, + )), + vec![("prometheus_rw", 0)], + ), + // UDP and plain TCP bind first, the TLS listener last. + Self::Syslog => ( + Box::new(SyslogHandler::new( + config.syslog.clone(), + config.raw_capture_for(&config.syslog.raw_capture), + pipeline, + metrics, + )), + vec![("syslog tls", 2)], + ), + Self::Fluent => ( + Box::new(FluentHandler::new(config.fluent.clone(), pipeline, metrics)), + vec![("fluent", 0)], + ), + Self::Gelf => ( + Box::new(GelfHandler::new(config.gelf.clone(), pipeline, metrics)), + vec![("gelf", 0)], + ), + Self::Lumberjack => ( + Box::new(LumberjackHandler::new( + config.lumberjack.clone(), + pipeline, + metrics, + )), + vec![("lumberjack", 0)], + ), + Self::Grpc => ( + Box::new(GrpcVectorHandler::new(config.clone(), pipeline, metrics)), + vec![("grpc", 0)], + ), + #[cfg(feature = "otlp")] + Self::Otlp => ( + Box::new(dfe_receiver::server::otlp::OtlpHandler::new( + config.otlp.clone(), + pipeline, + metrics, + )), + vec![("otlp grpc", 0), ("otlp http", 1)], + ), + }; + + let name = handler.name(); + let bounds = handler.listeners(); + let shutdown = CancellationToken::new(); + let token = shutdown.clone(); + let mut task = tokio::spawn(async move { handler.start(token).await }); + let mut addrs = Vec::with_capacity(bounds.len()); + for bound in &bounds { + addrs.push(bound_addr(name, bound, &mut task).await); + } + Running { + tls: tls_listeners + .into_iter() + .map(|(label, index)| (label, addrs[index])) + .collect(), + shutdown, + _scratch: scratch, + } + } +} + +/// The certificate `addr` serves a client that trusts `ca`, `None` when the +/// handshake does not complete. +async fn served(client: &TlsConnector, addr: SocketAddr) -> Option> { + let stream = TcpStream::connect(addr).await.ok()?; + let secured = client + .connect(ServerName::try_from("localhost").unwrap(), stream) + .await + .ok()?; + let (_, connection) = secured.get_ref(); + connection.peer_certificates()?.first().cloned() +} + +/// Whether `addr` serves `expected` within [`ROTATION_DEADLINE`]. +async fn serves_within_deadline( + client: &TlsConnector, + addr: SocketAddr, + expected: &CertificateDer<'static>, +) -> bool { + let deadline = tokio::time::Instant::now() + ROTATION_DEADLINE; + loop { + if served(client, addr).await.as_ref() == Some(expected) { + return true; + } + if tokio::time::Instant::now() >= deadline { + return false; + } + tokio::time::sleep(Duration::from_millis(250)).await; + } +} + +/// Start `listener` on `source`, check it serves the first certificate, rotate, +/// and check it serves the second without a restart. +async fn rotation_reaches(listener: Listener, source: Source) { + let ca = Ca::new(); + let (first, second) = (ca.server("first"), ca.server("second")); + let files = Files::new(&first); + let client = ca.client(); + + let running = listener.start(files.tls(source)).await; + for (name, addr) in &running.tls { + assert!( + serves_within_deadline(&client, *addr, &first.der).await, + "{listener:?} {source:?}: {name} does not serve the first certificate" + ); + } + + files.serve(&second); + for (name, addr) in &running.tls { + assert!( + serves_within_deadline(&client, *addr, &second.der).await, + "{listener:?} {source:?}: {name} still serves the retired certificate" + ); + } + running.shutdown.cancel(); +} + +/// Every listener of `family`, on a file and on a secret, at once. +async fn family_rotates(family: &[Listener]) { + let mut cases = tokio::task::JoinSet::new(); + for &listener in family { + for source in [Source::File, Source::Secret] { + cases.spawn(rotation_reaches(listener, source)); + } + } + while let Some(outcome) = cases.join_next().await { + outcome.expect("a rotation case panicked"); + } +} + +#[tokio::test] +async fn the_hyper_listeners_serve_a_rotated_certificate() { + family_rotates(&[ + Listener::Http, + Listener::Webhook, + Listener::SplunkHec, + Listener::PrometheusRw, + ]) + .await; +} + +#[tokio::test] +async fn the_raw_tcp_listeners_serve_a_rotated_certificate() { + family_rotates(&[ + Listener::Syslog, + Listener::Fluent, + Listener::Gelf, + Listener::Lumberjack, + ]) + .await; +} + +#[tokio::test] +async fn the_tonic_listeners_serve_a_rotated_certificate() { + #[cfg(feature = "otlp")] + family_rotates(&[Listener::Grpc, Listener::Otlp]).await; + #[cfg(not(feature = "otlp"))] + family_rotates(&[Listener::Grpc]).await; +} + +/// A tonic channel to `addr` that trusts `ca_pem`, speaking TLS to `localhost`. +async fn grpc_channel(ca_pem: &str, addr: SocketAddr) -> tonic::transport::Channel { + let tls = tonic::transport::ClientTlsConfig::new() + .ca_certificate(tonic::transport::Certificate::from_pem(ca_pem)) + .domain_name("localhost"); + tonic::transport::Channel::from_shared(format!("https://{addr}")) + .unwrap() + .tls_config(tls) + .unwrap() + .connect() + .await + .expect("a gRPC client completes the TLS handshake") +} + +/// The handshake is run by the receiver rather than tonic, so a gRPC client +/// has to negotiate `h2` and get an answer back over the stream it hands tonic. +#[tokio::test] +async fn grpc_answers_requests_over_the_receivers_handshake() { + use grpc_pb::vector::{HealthCheckRequest, HealthCheckResponse}; + + let ca = Ca::new(); + let files = Files::new(&ca.server("grpc")); + let running = Listener::Grpc.start(files.tls(Source::File)).await; + let (_, addr) = running.tls[0]; + + let channel = grpc_channel(&ca.issuer.pem(), addr).await; + let mut client = tonic::client::Grpc::new(channel); + client.ready().await.unwrap(); + let codec = tonic_prost::ProstCodec::::default(); + let path = http::uri::PathAndQuery::from_static("/vector.Vector/HealthCheck"); + client + .unary( + tonic::Request::new(HealthCheckRequest::default()), + path, + codec, + ) + .await + .expect("the health check is answered over TLS"); + running.shutdown.cancel(); +} + +#[cfg(feature = "otlp")] +#[tokio::test] +async fn otlp_grpc_answers_requests_over_the_receivers_handshake() { + use dfe_receiver::server::otlp::pb::collector::logs::v1::ExportLogsServiceRequest; + use dfe_receiver::server::otlp::pb::collector::logs::v1::logs_service_client::LogsServiceClient; + + let ca = Ca::new(); + let files = Files::new(&ca.server("otlp")); + let running = Listener::Otlp.start(files.tls(Source::File)).await; + let (_, addr) = running.tls[0]; + + let mut client = LogsServiceClient::new(grpc_channel(&ca.issuer.pem(), addr).await); + client + .export(ExportLogsServiceRequest::default()) + .await + .expect("an empty export is accepted over TLS"); + running.shutdown.cancel(); +} diff --git a/tests/integration/vault_auth.rs b/tests/integration/vault_auth.rs index 2ecca8f..860e577 100644 --- a/tests/integration/vault_auth.rs +++ b/tests/integration/vault_auth.rs @@ -29,11 +29,14 @@ use axum::extract::State; use axum::http::{HeaderMap, StatusCode, header}; use axum::response::{IntoResponse, Response}; use axum::routing::post; -use dfe_receiver::config::{BearerConfig, TlsConfig, WebhookAuthConfig, WebhookAuthMode}; +use dfe_receiver::config::{ + BearerConfig, ConnectionsConfig, TlsConfig, WebhookAuthConfig, WebhookAuthMode, +}; use dfe_receiver::server::auth::BearerTokenProvider; -use dfe_receiver::server::tls::build_tls_acceptor_async; +use dfe_receiver::server::tls::load_server_tls; use dfe_receiver::server::webhook::auth::{AuthFailure, CallerAuth}; use dfe_receiver::tenancy::Principal; +use scalo::tls::ServerTlsOptions; use tempfile::NamedTempFile; use crate::common::{OPENBAO_ROOT_TOKEN, docker_available, shared_openbao}; @@ -482,13 +485,14 @@ async fn test_tls_material_loads_from_file_specs() { ..TlsConfig::default() }; - let acceptor = build_tls_acceptor_async(&config) - .await - .expect("cert and key resolve through the file provider"); - assert!( - acceptor.is_some(), - "an enabled TLS config must produce an acceptor" - ); + let tls = load_server_tls( + &config, + &ConnectionsConfig::default(), + ServerTlsOptions::default(), + ) + .await + .expect("cert and key resolve through the file provider"); + assert!(tls.is_some(), "an enabled TLS config must load a ServerTls"); } /// A `file:` spec naming a path that is not there must name the path, not fail @@ -503,11 +507,14 @@ async fn test_tls_material_from_a_missing_file_names_the_path() { ..TlsConfig::default() }; - let err = build_tls_acceptor_async(&config) - .await - .err() - .expect("a missing cert file cannot load") - .to_string(); + let err = load_server_tls( + &config, + &ConnectionsConfig::default(), + ServerTlsOptions::default(), + ) + .await + .expect_err("a missing cert file cannot load") + .to_string(); assert!( err.contains("/nonexistent/dfe-receiver-test/server.crt"), "the error must name the path: {err}" @@ -660,14 +667,18 @@ async fn test_tls_material_loads_from_vault_container() { ..TlsConfig::default() }; - let acceptor = bao - .with_env(async { build_tls_acceptor_async(&config).await }) + let tls = bao + .with_env(async { + load_server_tls( + &config, + &ConnectionsConfig::default(), + ServerTlsOptions::default(), + ) + .await + }) .await .expect("the certificate and the key both resolve from OpenBao"); - assert!( - acceptor.is_some(), - "an enabled TLS config must produce an acceptor" - ); + assert!(tls.is_some(), "an enabled TLS config must load a ServerTls"); } /// A `header`-mode webhook caller's secret loads from a `vault:` reference and From 0c68d1c91b8f69aacd28e4e1070ec37e819febf1 Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 14:21:23 +1100 Subject: [PATCH 2/2] fix(tls): take the reload interval from scalo by default tls.refresh_interval_secs is unset by default, so ServerTls reloads on scalo's interval instead of the receiver's 3600. A value set in the config still wins. The example config, config schema and tests match this default. --- config.example.yaml | 16 +++++------ docs/config-schema.json | 49 +++++++++++++++++---------------- docs/config-schema.yaml | 48 ++++++++++++++++---------------- src/config/mod.rs | 6 ++-- src/server/tls.rs | 21 +++++++++++++- tests/integration/tls_reload.rs | 2 +- 6 files changed, 83 insertions(+), 59 deletions(-) diff --git a/config.example.yaml b/config.example.yaml index 2aa5ed8..58ed972 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -106,7 +106,7 @@ server: # cert_secret: "vault:secret/tls:cert" # key_secret: "vault:secret/tls:key" # ca_secret: "vault:secret/tls:ca" - # refresh_interval_secs: 3600 # How often cert, key and CA are re-read, from files and secrets alike + # refresh_interval_secs: 60 # How often cert, key and CA are re-read, from files and secrets alike. Unset: scalo's default auth: # none, header, bearer, mtls, both. Anything else refuses to start. header @@ -142,7 +142,7 @@ grpc: # client_auth: "none" # none, optional, required # cert_secret: "vault:secret/tls:cert" # Instead of cert_file # key_secret: "vault:secret/tls:key" - # refresh_interval_secs: 3600 # Re-read interval for cert, key and CA (1 hour) + # refresh_interval_secs: 60 # Re-read interval for cert, key and CA. Unset: scalo's default # server.ip_filter and server.rate_limit do NOT reach this port: tonic owns # the accept loop, so there is nowhere to reject a connection before it is @@ -190,7 +190,7 @@ lumberjack: # client_auth: "none" # none, optional, required # cert_secret: "vault:secret/tls:cert" # key_secret: "vault:secret/tls:key" - # refresh_interval_secs: 3600 + # refresh_interval_secs: 60 # Unset: scalo's default # NO application auth on this listener. A Lumberjack frame carries no # credential, so nothing in the handler reads `lumberjack.auth` -- setting a @@ -221,7 +221,7 @@ splunk_hec: # client_auth: "none" # cert_secret: "vault:secret/tls:cert" # key_secret: "vault:secret/tls:key" - # refresh_interval_secs: 3600 + # refresh_interval_secs: 60 # Unset: scalo's default # Auth -- supports both "Authorization: Splunk " and "Authorization: Bearer " auth: @@ -254,7 +254,7 @@ syslog: # client_auth: "none" # cert_secret: "vault:secret/tls:cert" # key_secret: "vault:secret/tls:key" - # refresh_interval_secs: 3600 + # refresh_interval_secs: 60 # Unset: scalo's default # NO application auth on this listener. A syslog line carries no credential, # so nothing in the handler reads `syslog.auth` -- setting a mode there is @@ -292,7 +292,7 @@ prometheus_rw: # client_auth: "none" # cert_secret: "vault:secret/tls:cert" # key_secret: "vault:secret/tls:key" - # refresh_interval_secs: 3600 + # refresh_interval_secs: 60 # Unset: scalo's default # Auth (same options as HTTP auth) auth: @@ -378,7 +378,7 @@ fluent: # client_auth: "none" # cert_secret: "vault:secret/tls:cert" # key_secret: "vault:secret/tls:key" - # refresh_interval_secs: 3600 + # refresh_interval_secs: 60 # Unset: scalo's default # GELF (Graylog Extended Log Format, null-delimited JSON over TCP) # @@ -401,7 +401,7 @@ gelf: # client_auth: "none" # cert_secret: "vault:secret/tls:cert" # key_secret: "vault:secret/tls:key" - # refresh_interval_secs: 3600 + # refresh_interval_secs: 60 # Unset: scalo's default # Raw payload retention -- the common-header `_raw` field # diff --git a/docs/config-schema.json b/docs/config-schema.json index e6de597..1b91b36 100644 --- a/docs/config-schema.json +++ b/docs/config-schema.json @@ -44,7 +44,7 @@ "cert_secret": null, "key_secret": null, "ca_secret": null, - "refresh_interval_secs": 3600 + "refresh_interval_secs": null }, "auth": { "mode": "none", @@ -77,7 +77,7 @@ "cert_secret": null, "key_secret": null, "ca_secret": null, - "refresh_interval_secs": 3600 + "refresh_interval_secs": null }, "auth": { "mode": "none", @@ -113,7 +113,7 @@ "cert_secret": null, "key_secret": null, "ca_secret": null, - "refresh_interval_secs": 3600 + "refresh_interval_secs": null }, "auth": { "mode": "none", @@ -148,7 +148,7 @@ "cert_secret": null, "key_secret": null, "ca_secret": null, - "refresh_interval_secs": 3600 + "refresh_interval_secs": null }, "auth": { "mode": "none", @@ -184,7 +184,7 @@ "cert_secret": null, "key_secret": null, "ca_secret": null, - "refresh_interval_secs": 3600 + "refresh_interval_secs": null }, "auth": { "mode": "none", @@ -231,7 +231,7 @@ "cert_secret": null, "key_secret": null, "ca_secret": null, - "refresh_interval_secs": 3600 + "refresh_interval_secs": null }, "auth": { "mode": "none", @@ -270,7 +270,7 @@ "cert_secret": null, "key_secret": null, "ca_secret": null, - "refresh_interval_secs": 3600 + "refresh_interval_secs": null }, "auth": { "mode": "none", @@ -304,7 +304,7 @@ "cert_secret": null, "key_secret": null, "ca_secret": null, - "refresh_interval_secs": 3600 + "refresh_interval_secs": null }, "accept_unauthenticated": false, "acknowledgements": { @@ -328,7 +328,7 @@ "cert_secret": null, "key_secret": null, "ca_secret": null, - "refresh_interval_secs": 3600 + "refresh_interval_secs": null }, "accept_unauthenticated": false } @@ -395,7 +395,7 @@ "cert_secret": null, "key_secret": null, "ca_secret": null, - "refresh_interval_secs": 3600 + "refresh_interval_secs": null }, "callers": [], "acknowledgements": { @@ -631,7 +631,7 @@ "cert_secret": null, "key_secret": null, "ca_secret": null, - "refresh_interval_secs": 3600 + "refresh_interval_secs": null } }, "auth": { @@ -813,11 +813,14 @@ "default": null }, "refresh_interval_secs": { - "description": "How often the certificate, key and CA are re-read, in seconds, from files and secrets alike.\nA rotated certificate is served from the next handshake, with no restart. 0 turns reload off.\nDefault: 3600 (1 hour)", - "type": "integer", + "description": "How often the certificate, key and CA are re-read, in seconds, from files and secrets alike.\nA rotated certificate is served from the next handshake, with no restart. 0 turns reload off.\nUnset, scalo's default applies.", + "type": [ + "integer", + "null" + ], "format": "uint64", "minimum": 0, - "default": 3600 + "default": null } } }, @@ -955,7 +958,7 @@ "cert_secret": null, "key_secret": null, "ca_secret": null, - "refresh_interval_secs": 3600 + "refresh_interval_secs": null } }, "auth": { @@ -1025,7 +1028,7 @@ "cert_secret": null, "key_secret": null, "ca_secret": null, - "refresh_interval_secs": 3600 + "refresh_interval_secs": null } }, "auth": { @@ -1092,7 +1095,7 @@ "cert_secret": null, "key_secret": null, "ca_secret": null, - "refresh_interval_secs": 3600 + "refresh_interval_secs": null } }, "auth": { @@ -1164,7 +1167,7 @@ "cert_secret": null, "key_secret": null, "ca_secret": null, - "refresh_interval_secs": 3600 + "refresh_interval_secs": null } }, "auth": { @@ -1334,7 +1337,7 @@ "cert_secret": null, "key_secret": null, "ca_secret": null, - "refresh_interval_secs": 3600 + "refresh_interval_secs": null } }, "auth": { @@ -1417,7 +1420,7 @@ "cert_secret": null, "key_secret": null, "ca_secret": null, - "refresh_interval_secs": 3600 + "refresh_interval_secs": null } }, "auth": { @@ -1477,7 +1480,7 @@ "cert_secret": null, "key_secret": null, "ca_secret": null, - "refresh_interval_secs": 3600 + "refresh_interval_secs": null } }, "accept_unauthenticated": { @@ -1527,7 +1530,7 @@ "cert_secret": null, "key_secret": null, "ca_secret": null, - "refresh_interval_secs": 3600 + "refresh_interval_secs": null } }, "accept_unauthenticated": { @@ -1882,7 +1885,7 @@ "cert_secret": null, "key_secret": null, "ca_secret": null, - "refresh_interval_secs": 3600 + "refresh_interval_secs": null } }, "callers": { diff --git a/docs/config-schema.yaml b/docs/config-schema.yaml index 99caff2..f64ff69 100644 --- a/docs/config-schema.yaml +++ b/docs/config-schema.yaml @@ -42,7 +42,7 @@ properties: cert_secret: null key_secret: null ca_secret: null - refresh_interval_secs: 3600 + refresh_interval_secs: null auth: mode: none accepted_headers: [] @@ -69,7 +69,7 @@ properties: cert_secret: null key_secret: null ca_secret: null - refresh_interval_secs: 3600 + refresh_interval_secs: null auth: mode: none accepted_headers: [] @@ -99,7 +99,7 @@ properties: cert_secret: null key_secret: null ca_secret: null - refresh_interval_secs: 3600 + refresh_interval_secs: null auth: mode: none accepted_headers: [] @@ -128,7 +128,7 @@ properties: cert_secret: null key_secret: null ca_secret: null - refresh_interval_secs: 3600 + refresh_interval_secs: null auth: mode: none accepted_headers: [] @@ -158,7 +158,7 @@ properties: cert_secret: null key_secret: null ca_secret: null - refresh_interval_secs: 3600 + refresh_interval_secs: null auth: mode: none accepted_headers: [] @@ -198,7 +198,7 @@ properties: cert_secret: null key_secret: null ca_secret: null - refresh_interval_secs: 3600 + refresh_interval_secs: null auth: mode: none accepted_headers: [] @@ -231,7 +231,7 @@ properties: cert_secret: null key_secret: null ca_secret: null - refresh_interval_secs: 3600 + refresh_interval_secs: null auth: mode: none accepted_headers: [] @@ -259,7 +259,7 @@ properties: cert_secret: null key_secret: null ca_secret: null - refresh_interval_secs: 3600 + refresh_interval_secs: null accept_unauthenticated: false acknowledgements: enabled: true @@ -279,7 +279,7 @@ properties: cert_secret: null key_secret: null ca_secret: null - refresh_interval_secs: 3600 + refresh_interval_secs: null accept_unauthenticated: false flow: description: Flow (NetFlow v5/v9 + IPFIX + sFlow v5) receiver configuration. @@ -334,7 +334,7 @@ properties: cert_secret: null key_secret: null ca_secret: null - refresh_interval_secs: 3600 + refresh_interval_secs: null callers: [] acknowledgements: enabled: true @@ -547,7 +547,7 @@ $defs: cert_secret: null key_secret: null ca_secret: null - refresh_interval_secs: 3600 + refresh_interval_secs: null auth: description: Authentication configuration. $ref: "#/$defs/AuthConfig" @@ -763,11 +763,13 @@ $defs: description: |- How often the certificate, key and CA are re-read, in seconds, from files and secrets alike. A rotated certificate is served from the next handshake, with no restart. 0 turns reload off. - Default: 3600 (1 hour) - type: integer + Unset, scalo's default applies. + type: + - integer + - "null" format: uint64 minimum: 0 - default: 3600 + default: null AuthConfig: description: Authentication configuration. type: object @@ -910,7 +912,7 @@ $defs: cert_secret: null key_secret: null ca_secret: null - refresh_interval_secs: 3600 + refresh_interval_secs: null auth: description: Authentication configuration for gRPC server. $ref: "#/$defs/AuthConfig" @@ -973,7 +975,7 @@ $defs: cert_secret: null key_secret: null ca_secret: null - refresh_interval_secs: 3600 + refresh_interval_secs: null auth: description: Authentication configuration for OTLP endpoints. $ref: "#/$defs/AuthConfig" @@ -1038,7 +1040,7 @@ $defs: cert_secret: null key_secret: null ca_secret: null - refresh_interval_secs: 3600 + refresh_interval_secs: null auth: description: Authentication configuration. $ref: "#/$defs/AuthConfig" @@ -1105,7 +1107,7 @@ $defs: cert_secret: null key_secret: null ca_secret: null - refresh_interval_secs: 3600 + refresh_interval_secs: null auth: description: Authentication configuration. $ref: "#/$defs/AuthConfig" @@ -1265,7 +1267,7 @@ $defs: cert_secret: null key_secret: null ca_secret: null - refresh_interval_secs: 3600 + refresh_interval_secs: null auth: description: Authentication configuration. $ref: "#/$defs/AuthConfig" @@ -1343,7 +1345,7 @@ $defs: cert_secret: null key_secret: null ca_secret: null - refresh_interval_secs: 3600 + refresh_interval_secs: null auth: description: Authentication configuration. $ref: "#/$defs/AuthConfig" @@ -1402,7 +1404,7 @@ $defs: cert_secret: null key_secret: null ca_secret: null - refresh_interval_secs: 3600 + refresh_interval_secs: null accept_unauthenticated: description: |- Start without client certificates, accepting every client that can @@ -1457,7 +1459,7 @@ $defs: cert_secret: null key_secret: null ca_secret: null - refresh_interval_secs: 3600 + refresh_interval_secs: null accept_unauthenticated: description: |- Start without client certificates, accepting every client that can @@ -1759,7 +1761,7 @@ $defs: cert_secret: null key_secret: null ca_secret: null - refresh_interval_secs: 3600 + refresh_interval_secs: null callers: description: The callers, one entry per product. type: array diff --git a/src/config/mod.rs b/src/config/mod.rs index fab2990..852e083 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -1335,8 +1335,8 @@ pub struct TlsConfig { /// How often the certificate, key and CA are re-read, in seconds, from files and secrets alike. /// A rotated certificate is served from the next handshake, with no restart. 0 turns reload off. - /// Default: 3600 (1 hour) - pub refresh_interval_secs: u64, + /// Unset, scalo's default applies. + pub refresh_interval_secs: Option, } impl Default for TlsConfig { @@ -1350,7 +1350,7 @@ impl Default for TlsConfig { cert_secret: None, key_secret: None, ca_secret: None, - refresh_interval_secs: 3600, + refresh_interval_secs: None, } } } diff --git a/src/server/tls.rs b/src/server/tls.rs index 4a636a7..43da59f 100644 --- a/src/server/tls.rs +++ b/src/server/tls.rs @@ -68,6 +68,12 @@ impl Handshake { } } +/// `tls.refresh_interval_secs`, else the interval scalo's `ServerTls` defaults to. +fn reload_interval_secs(tls: &TlsConfig) -> u64 { + tls.refresh_interval_secs + .unwrap_or_else(|| ServerTlsConfig::default().reload_interval_secs) +} + /// What a gRPC listener's handshake offers: ALPN `h2`, which gRPC clients /// require and tonic offered itself when it ran the handshake. #[must_use] @@ -119,7 +125,7 @@ pub async fn load_server_tls( ca_file: tls.ca_file.as_ref().map(PathBuf::from), ca_secret: tls.ca_secret.clone(), client_auth, - reload_interval_secs: tls.refresh_interval_secs, + reload_interval_secs: reload_interval_secs(tls), handshake_timeout_ms: connections.tls_handshake_timeout_ms, }; // Boxed for the reason `secrets::read` boxes: the AWS provider's future, awaited @@ -198,6 +204,19 @@ mod tests { } } + #[test] + fn an_unset_refresh_interval_takes_scalos_default() { + let set = TlsConfig { + refresh_interval_secs: Some(7), + ..TlsConfig::default() + }; + assert_eq!(reload_interval_secs(&set), 7); + assert_eq!( + reload_interval_secs(&TlsConfig::default()), + ServerTlsConfig::default().reload_interval_secs + ); + } + #[tokio::test] async fn a_disabled_block_loads_nothing() { let loaded = load_server_tls( diff --git a/tests/integration/tls_reload.rs b/tests/integration/tls_reload.rs index e28b918..4e2cdd6 100644 --- a/tests/integration/tls_reload.rs +++ b/tests/integration/tls_reload.rs @@ -159,7 +159,7 @@ impl Files { fn tls(&self, source: Source) -> TlsConfig { let tls = TlsConfig { enabled: true, - refresh_interval_secs: 1, + refresh_interval_secs: Some(1), ..TlsConfig::default() }; match source {