From 3c8700f4513866c1d237fd45e906ec0f350f707b Mon Sep 17 00:00:00 2001 From: Kacy Fortner Date: Sat, 14 Nov 2026 13:35:05 -0500 Subject: [PATCH 1/4] fix: serve gRPC over TLS when tls-port is set --- crates/ember-server/src/grpc/mod.rs | 1 + crates/ember-server/src/grpc/tls.rs | 90 +++++++++++++++++++++++++++++ crates/ember-server/src/main.rs | 8 +++ crates/ember-server/src/server.rs | 32 ++++++++-- crates/ember-server/src/tls.rs | 20 ++++++- 5 files changed, 142 insertions(+), 9 deletions(-) create mode 100644 crates/ember-server/src/grpc/tls.rs diff --git a/crates/ember-server/src/grpc/mod.rs b/crates/ember-server/src/grpc/mod.rs index f7133b12..8d917f9e 100644 --- a/crates/ember-server/src/grpc/mod.rs +++ b/crates/ember-server/src/grpc/mod.rs @@ -36,6 +36,7 @@ mod server; mod sets; mod sorted_sets; mod strings; +pub(crate) mod tls; mod vector; use proto::ember_cache_server::EmberCache; diff --git a/crates/ember-server/src/grpc/tls.rs b/crates/ember-server/src/grpc/tls.rs new file mode 100644 index 00000000..8b8751e6 --- /dev/null +++ b/crates/ember-server/src/grpc/tls.rs @@ -0,0 +1,90 @@ +//! Serving gRPC over TLS with the server's own certificate. + +use std::io; +use std::net::SocketAddr; +use std::pin::Pin; +use std::task::{Context, Poll}; +use std::time::Duration; + +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::sync::mpsc; +use tokio_rustls::server::TlsStream; +use tokio_rustls::TlsAcceptor; +use tokio_stream::wrappers::ReceiverStream; +use tonic::transport::server::{Connected, TcpConnectInfo}; +use tracing::warn; + +/// A gRPC connection after its TLS handshake. It reports the TCP +/// addresses, so protected mode can check the peer as it does without TLS. +pub(crate) struct TlsConn { + stream: TlsStream, + info: TcpConnectInfo, +} + +impl Connected for TlsConn { + type ConnectInfo = TcpConnectInfo; + + fn connect_info(&self) -> TcpConnectInfo { + self.info.clone() + } +} + +impl AsyncRead for TlsConn { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + Pin::new(&mut self.get_mut().stream).poll_read(cx, buf) + } +} + +impl AsyncWrite for TlsConn { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + Pin::new(&mut self.get_mut().stream).poll_write(cx, buf) + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.get_mut().stream).poll_flush(cx) + } + + fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.get_mut().stream).poll_shutdown(cx) + } +} + +/// Accepts connections on `addr` and completes their TLS handshakes, for +/// `Server::serve_with_incoming`. Each handshake runs in its own task, so +/// a slow client doesn't hold up the others. +pub(crate) async fn incoming( + addr: SocketAddr, + acceptor: TlsAcceptor, +) -> io::Result>> { + let listener = TcpListener::bind(addr).await?; + let (tx, rx) = mpsc::channel(64); + tokio::spawn(async move { + while !tx.is_closed() { + let (stream, peer) = match listener.accept().await { + Ok(conn) => conn, + Err(e) => { + warn!("gRPC accept error: {e}"); + tokio::time::sleep(Duration::from_millis(100)).await; + continue; + } + }; + let info = stream.connect_info(); + let (acceptor, tx) = (acceptor.clone(), tx.clone()); + tokio::spawn(async move { + if let Some(stream) = crate::server::tls_handshake(acceptor, stream, peer).await { + let _ = tx.send(Ok(TlsConn { stream, info })).await; + } + }); + } + }); + Ok(ReceiverStream::new(rx)) +} diff --git a/crates/ember-server/src/main.rs b/crates/ember-server/src/main.rs index 56f7bf5b..8aca1c86 100644 --- a/crates/ember-server/src/main.rs +++ b/crates/ember-server/src/main.rs @@ -154,6 +154,7 @@ struct Args { // -- gRPC options -- /// port for gRPC connections (default 6380). set to 0 to disable it. + /// when tls-port is set, gRPC uses the same TLS settings. #[cfg(feature = "grpc")] #[arg(long, env = "EMBER_GRPC_PORT")] grpc_port: Option, @@ -707,6 +708,13 @@ async fn main() { exit_err("tls-auth-clients requires tls-ca-cert-file"); } + #[cfg(feature = "grpc")] + if cfg.grpc_port == tls_port { + exit_err(format!( + "tls-port and the gRPC port are both {tls_port}; move one, or disable gRPC with --no-grpc" + )); + } + let tls_addr = parse_bind_addr(&cfg.bind, tls_port, "TLS"); info!( diff --git a/crates/ember-server/src/server.rs b/crates/ember-server/src/server.rs index ecac452b..513bb3a8 100644 --- a/crates/ember-server/src/server.rs +++ b/crates/ember-server/src/server.rs @@ -149,7 +149,7 @@ fn accept_connection( /// Performs TLS handshake with a 10-second timeout to prevent slowloris attacks. /// Returns `None` on handshake failure or timeout. -async fn tls_handshake( +pub(crate) async fn tls_handshake( acceptor: TlsAcceptor, stream: TcpStream, peer: SocketAddr, @@ -465,16 +465,36 @@ pub async fn run_threaded( Arc::clone(&slow_log), Arc::clone(&pubsub), ); - info!("gRPC listening on {grpc_addr}"); - let server = tonic::transport::Server::builder() + // gRPC takes the TLS settings of the RESP TLS port, so a server + // that encrypts RESP doesn't serve gRPC in plaintext + let tls = match tls.as_ref() { + Some((_, tls_config)) => Some(crate::tls::load_grpc_acceptor(tls_config)?), + None => None, + }; + let router = tonic::transport::Server::builder() .concurrency_limit_per_connection(256) .http2_keepalive_interval(Some(Duration::from_secs(30))) .http2_keepalive_timeout(Some(Duration::from_secs(10))) .tcp_keepalive(Some(Duration::from_secs(60))) - .add_service(svc.into_service()) - .serve(grpc_addr); + .add_service(svc.into_service()); Some(tokio::spawn(async move { - if let Err(e) = server.await { + let result = match tls { + Some(acceptor) => match crate::grpc::tls::incoming(grpc_addr, acceptor).await { + Ok(incoming) => { + info!("gRPC listening on {grpc_addr} with TLS"); + router.serve_with_incoming(incoming).await + } + Err(e) => { + error!("failed to bind gRPC on {grpc_addr}: {e}"); + return; + } + }, + None => { + info!("gRPC listening on {grpc_addr}"); + router.serve(grpc_addr).await + } + }; + if let Err(e) = result { error!("gRPC server error: {e}"); } })) diff --git a/crates/ember-server/src/tls.rs b/crates/ember-server/src/tls.rs index 33d7776f..dcc8ffc0 100644 --- a/crates/ember-server/src/tls.rs +++ b/crates/ember-server/src/tls.rs @@ -75,12 +75,26 @@ fn require_file(path: &Path, make_err: impl FnOnce(String) -> TlsError) -> Resul } } -/// Loads TLS configuration and creates a `TlsAcceptor`. +/// Loads TLS configuration and creates a `TlsAcceptor` for RESP clients. +pub fn load_tls_acceptor(config: &TlsConfig) -> Result { + Ok(TlsAcceptor::from(Arc::new(load_server_config(config)?))) +} + +/// Creates a `TlsAcceptor` for gRPC clients from the same configuration. +/// gRPC runs over HTTP/2, which clients ask for during the handshake. +#[cfg(feature = "grpc")] +pub fn load_grpc_acceptor(config: &TlsConfig) -> Result { + let mut server_config = load_server_config(config)?; + server_config.alpn_protocols = vec![b"h2".to_vec()]; + Ok(TlsAcceptor::from(Arc::new(server_config))) +} + +/// Builds the rustls server configuration. /// /// Reads certificates and private key from PEM files. If `ca_cert_file` is /// specified, sets up client certificate verification. When `auth_clients` /// is true, clients must present valid certificates signed by the CA. -pub fn load_tls_acceptor(config: &TlsConfig) -> Result { +fn load_server_config(config: &TlsConfig) -> Result { // load server certificates let cert_path = Path::new(&config.cert_file); require_file(cert_path, TlsError::CertFileNotFound)?; @@ -146,7 +160,7 @@ pub fn load_tls_acceptor(config: &TlsConfig) -> Result { .with_single_cert(certs, key)? }; - Ok(TlsAcceptor::from(Arc::new(server_config))) + Ok(server_config) } #[cfg(test)] From f0f7d653628e094ee72070966b2bd8b90e2fd1a0 Mon Sep 17 00:00:00 2001 From: Kacy Fortner Date: Sat, 14 Nov 2026 13:35:05 -0500 Subject: [PATCH 2/4] test: gRPC over TLS and the port clash check --- Cargo.lock | 2 ++ tests/integration/Cargo.toml | 3 ++ tests/integration/src/grpc.rs | 58 +++++++++++++++++++++++++++++++++++ tests/integration/src/tls.rs | 6 ++-- 4 files changed, 67 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 50c87853..1186f2bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -711,6 +711,7 @@ dependencies = [ "ember-client", "ember-protocol", "ember-server", + "hyper-util", "prost 0.13.5", "prost-reflect", "rcgen", @@ -722,6 +723,7 @@ dependencies = [ "tokio-stream", "tonic", "tonic-build", + "tower", ] [[package]] diff --git a/tests/integration/Cargo.toml b/tests/integration/Cargo.toml index 3efcfd85..d45bada0 100644 --- a/tests/integration/Cargo.toml +++ b/tests/integration/Cargo.toml @@ -25,6 +25,9 @@ tokio-rustls = { workspace = true } tokio-stream = { workspace = true } tonic = { workspace = true } prost = { workspace = true } +# both come with tonic; the gRPC TLS test connects through them +hyper-util = { version = "0.1", features = ["tokio"] } +tower = "0.5" [build-dependencies] tonic-build = { workspace = true } diff --git a/tests/integration/src/grpc.rs b/tests/integration/src/grpc.rs index 165338f1..e93fed60 100644 --- a/tests/integration/src/grpc.rs +++ b/tests/integration/src/grpc.rs @@ -2655,3 +2655,61 @@ async fn grpc_pipeline_errors_do_not_abort() { assert_eq!(resp.id, 7); assert!(matches!(resp.result, Some(PipeResult::Error(_)))); } + +#[tokio::test] +async fn grpc_uses_the_tls_settings_of_the_tls_port() { + use std::sync::Arc; + + use hyper_util::rt::TokioIo; + use rustls::pki_types::ServerName; + use tonic::transport::Endpoint; + + let tmp = tempfile::tempdir().unwrap(); + let (cert_path, key_path, cert_der) = crate::tls::generate_test_cert(tmp.path()); + let server = TestServer::start_with(ServerOptions { + grpc: true, + tls_cert_file: Some(cert_path), + tls_key_file: Some(key_path), + ..Default::default() + }); + let port = server.grpc_port.unwrap(); + + // a plaintext client gets nowhere + let plain = match EmberCacheClient::connect(format!("http://127.0.0.1:{port}")).await { + Ok(mut c) => c.ping(PingRequest { message: None }).await.is_ok(), + Err(_) => false, + }; + assert!(!plain, "gRPC answered in plaintext"); + + // a client that trusts the server's certificate and asks for HTTP/2 + let mut config = crate::tls::client_config(cert_der); + config.alpn_protocols = vec![b"h2".to_vec()]; + let connector = tokio_rustls::TlsConnector::from(Arc::new(config)); + let channel = Endpoint::from_static("http://localhost") + .connect_with_connector(tower::service_fn(move |_| { + let connector = connector.clone(); + async move { + let tcp = tokio::net::TcpStream::connect(("127.0.0.1", port)).await?; + let name = ServerName::try_from("localhost").unwrap(); + let tls = connector.connect(name, tcp).await?; + Ok::<_, std::io::Error>(TokioIo::new(tls)) + } + })) + .await + .expect("TLS connection failed"); + let mut client = EmberCacheClient::new(channel); + let reply = client.ping(PingRequest { message: None }).await.unwrap(); + assert_eq!(reply.into_inner().message, "PONG"); +} + +#[test] +fn the_tls_port_and_the_grpc_port_must_differ() { + let output = std::process::Command::new(crate::helpers::server_binary()) + .args(["--port", "0", "--tls-port", "16998", "--grpc-port", "16998"]) + .args(["--tls-cert-file", "cert.pem", "--tls-key-file", "key.pem"]) + .output() + .unwrap(); + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("tls-port and the gRPC port"), "{stderr}"); +} diff --git a/tests/integration/src/tls.rs b/tests/integration/src/tls.rs index cd484c21..2895f248 100644 --- a/tests/integration/src/tls.rs +++ b/tests/integration/src/tls.rs @@ -17,7 +17,9 @@ use crate::helpers::{ServerOptions, TestServer}; /// Generates a self-signed cert/key pair for `localhost` and writes PEM files /// into the given directory. Returns the cert path, key path, and the raw DER /// bytes needed to build a client-side trust store. -fn generate_test_cert(dir: &std::path::Path) -> (std::path::PathBuf, std::path::PathBuf, Vec) { +pub(crate) fn generate_test_cert( + dir: &std::path::Path, +) -> (std::path::PathBuf, std::path::PathBuf, Vec) { let rcgen::CertifiedKey { cert, key_pair } = rcgen::generate_simple_self_signed(vec!["localhost".to_string()]) .expect("rcgen cert generation failed"); @@ -38,7 +40,7 @@ fn generate_test_cert(dir: &std::path::Path) -> (std::path::PathBuf, std::path:: /// /// This simulates a client that has been explicitly configured to trust the /// server's self-signed cert — the same trust model as pinned certificates. -fn client_config(cert_der: Vec) -> rustls::ClientConfig { +pub(crate) fn client_config(cert_der: Vec) -> rustls::ClientConfig { let mut root_store = rustls::RootCertStore::empty(); root_store .add(CertificateDer::from(cert_der)) From 5584b8df94a438c2c50d6f49a059d81be4d65d3e Mon Sep 17 00:00:00 2001 From: Kacy Fortner Date: Sat, 14 Nov 2026 13:35:05 -0500 Subject: [PATCH 3/4] docs: describe gRPC TLS --- CHANGELOG.md | 1 + README.md | 2 +- docs/production-checklist.md | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6811a76..f762ad51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ - cluster mode is about as fast as standalone without pipelining: about 860k requests a second against 320k on a 32-core machine. with a pipeline of 16 it serves about 2.3M against 320k, 15% below standalone. every command locked the migration state, an async mutex shared by every connection, and read the cluster state up to three times. it now reads each once, and the migration state takes a shared lock (#453) ### fixes +- gRPC uses TLS when `tls-port` is set, with the same certificate, key and client certificate settings. it served plaintext on its own port even then, so a server that encrypted RESP still sent the same data in the clear over gRPC. the server refuses to start when the TLS port and the gRPC port are the same - replication hardening (#456): - the replica checks that the primary holds the cluster secret before taking its data. only the primary checked the replica, so anything that could pose as the primary's address could replace a replica's data. each side signs the other's nonce after a label for its role, so one side's answer can't be relayed as the other's. the replication protocol version is now 3 - the primary drops a connection that hasn't finished the handshake within 10 seconds, and runs at most 16 handshakes at once. an idle connection held a task forever before authenticating diff --git a/README.md b/README.md index 5c531ec7..f9fda944 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ see [docs/compatibility.md](docs/compatibility.md) for the full command support - **RENAME** requires the source and destination keys to hash to the same shard. cross-shard renames return an error. - **transactions** (MULTI/EXEC) are fully atomic within a single shard. cross-shard transactions execute in order but are not globally atomic — the same limitation as Redis Cluster. - **geo commands** are coming in a future release. -- `redis-cli` connects on port 6379 (RESP3). the typed gRPC API is on port 6380 (when `--features grpc` is compiled in). +- `redis-cli` connects on port 6379 (RESP3). the typed gRPC API is on port 6380 (when `--features grpc` is compiled in), over TLS when `tls-port` is set. ## install diff --git a/docs/production-checklist.md b/docs/production-checklist.md index c9bb45da..0607eb43 100644 --- a/docs/production-checklist.md +++ b/docs/production-checklist.md @@ -203,13 +203,15 @@ run `INFO all` to get everything at once. ember supports TLS for encrypting client connections. configure it in your config file: ```toml -tls-port = 6380 +tls-port = 6381 tls-cert-file = /etc/ember/server.crt tls-key-file = /etc/ember/server.key ``` the plain RESP port and the TLS port operate independently. you can run both simultaneously during a migration, or disable the plain port entirely once clients are updated. +when TLS is configured, the gRPC port (6380 by default) uses the same certificate, key and client certificate settings, and no longer accepts plaintext. gRPC clients must connect with TLS and negotiate HTTP/2 (ALPN `h2`), which standard gRPC libraries do. the TLS port and the gRPC port must differ. + **client certificate verification (mtls)** to require clients to present a certificate: From 40541f169e97af78ecb0d86c9b62a49652520d8e Mon Sep 17 00:00:00 2001 From: Kacy Fortner Date: Sat, 14 Nov 2026 13:35:12 -0500 Subject: [PATCH 4/4] docs: add the PR number to the changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f762ad51..b913e587 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ - cluster mode is about as fast as standalone without pipelining: about 860k requests a second against 320k on a 32-core machine. with a pipeline of 16 it serves about 2.3M against 320k, 15% below standalone. every command locked the migration state, an async mutex shared by every connection, and read the cluster state up to three times. it now reads each once, and the migration state takes a shared lock (#453) ### fixes -- gRPC uses TLS when `tls-port` is set, with the same certificate, key and client certificate settings. it served plaintext on its own port even then, so a server that encrypted RESP still sent the same data in the clear over gRPC. the server refuses to start when the TLS port and the gRPC port are the same +- gRPC uses TLS when `tls-port` is set, with the same certificate, key and client certificate settings. it served plaintext on its own port even then, so a server that encrypted RESP still sent the same data in the clear over gRPC. the server refuses to start when the TLS port and the gRPC port are the same (#457) - replication hardening (#456): - the replica checks that the primary holds the cluster secret before taking its data. only the primary checked the replica, so anything that could pose as the primary's address could replace a replica's data. each side signs the other's nonce after a label for its role, so one side's answer can't be relayed as the other's. the replication protocol version is now 3 - the primary drops a connection that hasn't finished the handshake within 10 seconds, and runs at most 16 handshakes at once. an idle connection held a task forever before authenticating