diff --git a/CHANGELOG.md b/CHANGELOG.md index d6811a7..b913e58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ - cluster mode is about as fast as standalone without pipelining: about 860k requests a second against 320k on a 32-core machine. with a pipeline of 16 it serves about 2.3M against 320k, 15% below standalone. every command locked the migration state, an async mutex shared by every connection, and read the cluster state up to three times. it now reads each once, and the migration state takes a shared lock (#453) ### fixes +- gRPC uses TLS when `tls-port` is set, with the same certificate, key and client certificate settings. it served plaintext on its own port even then, so a server that encrypted RESP still sent the same data in the clear over gRPC. the server refuses to start when the TLS port and the gRPC port are the same (#457) - replication hardening (#456): - the replica checks that the primary holds the cluster secret before taking its data. only the primary checked the replica, so anything that could pose as the primary's address could replace a replica's data. each side signs the other's nonce after a label for its role, so one side's answer can't be relayed as the other's. the replication protocol version is now 3 - the primary drops a connection that hasn't finished the handshake within 10 seconds, and runs at most 16 handshakes at once. an idle connection held a task forever before authenticating diff --git a/Cargo.lock b/Cargo.lock index 50c8785..1186f2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -711,6 +711,7 @@ dependencies = [ "ember-client", "ember-protocol", "ember-server", + "hyper-util", "prost 0.13.5", "prost-reflect", "rcgen", @@ -722,6 +723,7 @@ dependencies = [ "tokio-stream", "tonic", "tonic-build", + "tower", ] [[package]] diff --git a/README.md b/README.md index 5c531ec..f9fda94 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ see [docs/compatibility.md](docs/compatibility.md) for the full command support - **RENAME** requires the source and destination keys to hash to the same shard. cross-shard renames return an error. - **transactions** (MULTI/EXEC) are fully atomic within a single shard. cross-shard transactions execute in order but are not globally atomic — the same limitation as Redis Cluster. - **geo commands** are coming in a future release. -- `redis-cli` connects on port 6379 (RESP3). the typed gRPC API is on port 6380 (when `--features grpc` is compiled in). +- `redis-cli` connects on port 6379 (RESP3). the typed gRPC API is on port 6380 (when `--features grpc` is compiled in), over TLS when `tls-port` is set. ## install diff --git a/crates/ember-server/src/grpc/mod.rs b/crates/ember-server/src/grpc/mod.rs index f7133b1..8d917f9 100644 --- a/crates/ember-server/src/grpc/mod.rs +++ b/crates/ember-server/src/grpc/mod.rs @@ -36,6 +36,7 @@ mod server; mod sets; mod sorted_sets; mod strings; +pub(crate) mod tls; mod vector; use proto::ember_cache_server::EmberCache; diff --git a/crates/ember-server/src/grpc/tls.rs b/crates/ember-server/src/grpc/tls.rs new file mode 100644 index 0000000..8b8751e --- /dev/null +++ b/crates/ember-server/src/grpc/tls.rs @@ -0,0 +1,90 @@ +//! Serving gRPC over TLS with the server's own certificate. + +use std::io; +use std::net::SocketAddr; +use std::pin::Pin; +use std::task::{Context, Poll}; +use std::time::Duration; + +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::sync::mpsc; +use tokio_rustls::server::TlsStream; +use tokio_rustls::TlsAcceptor; +use tokio_stream::wrappers::ReceiverStream; +use tonic::transport::server::{Connected, TcpConnectInfo}; +use tracing::warn; + +/// A gRPC connection after its TLS handshake. It reports the TCP +/// addresses, so protected mode can check the peer as it does without TLS. +pub(crate) struct TlsConn { + stream: TlsStream, + info: TcpConnectInfo, +} + +impl Connected for TlsConn { + type ConnectInfo = TcpConnectInfo; + + fn connect_info(&self) -> TcpConnectInfo { + self.info.clone() + } +} + +impl AsyncRead for TlsConn { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + Pin::new(&mut self.get_mut().stream).poll_read(cx, buf) + } +} + +impl AsyncWrite for TlsConn { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + Pin::new(&mut self.get_mut().stream).poll_write(cx, buf) + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.get_mut().stream).poll_flush(cx) + } + + fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.get_mut().stream).poll_shutdown(cx) + } +} + +/// Accepts connections on `addr` and completes their TLS handshakes, for +/// `Server::serve_with_incoming`. Each handshake runs in its own task, so +/// a slow client doesn't hold up the others. +pub(crate) async fn incoming( + addr: SocketAddr, + acceptor: TlsAcceptor, +) -> io::Result>> { + let listener = TcpListener::bind(addr).await?; + let (tx, rx) = mpsc::channel(64); + tokio::spawn(async move { + while !tx.is_closed() { + let (stream, peer) = match listener.accept().await { + Ok(conn) => conn, + Err(e) => { + warn!("gRPC accept error: {e}"); + tokio::time::sleep(Duration::from_millis(100)).await; + continue; + } + }; + let info = stream.connect_info(); + let (acceptor, tx) = (acceptor.clone(), tx.clone()); + tokio::spawn(async move { + if let Some(stream) = crate::server::tls_handshake(acceptor, stream, peer).await { + let _ = tx.send(Ok(TlsConn { stream, info })).await; + } + }); + } + }); + Ok(ReceiverStream::new(rx)) +} diff --git a/crates/ember-server/src/main.rs b/crates/ember-server/src/main.rs index 56f7bf5..8aca1c8 100644 --- a/crates/ember-server/src/main.rs +++ b/crates/ember-server/src/main.rs @@ -154,6 +154,7 @@ struct Args { // -- gRPC options -- /// port for gRPC connections (default 6380). set to 0 to disable it. + /// when tls-port is set, gRPC uses the same TLS settings. #[cfg(feature = "grpc")] #[arg(long, env = "EMBER_GRPC_PORT")] grpc_port: Option, @@ -707,6 +708,13 @@ async fn main() { exit_err("tls-auth-clients requires tls-ca-cert-file"); } + #[cfg(feature = "grpc")] + if cfg.grpc_port == tls_port { + exit_err(format!( + "tls-port and the gRPC port are both {tls_port}; move one, or disable gRPC with --no-grpc" + )); + } + let tls_addr = parse_bind_addr(&cfg.bind, tls_port, "TLS"); info!( diff --git a/crates/ember-server/src/server.rs b/crates/ember-server/src/server.rs index ecac452..513bb3a 100644 --- a/crates/ember-server/src/server.rs +++ b/crates/ember-server/src/server.rs @@ -149,7 +149,7 @@ fn accept_connection( /// Performs TLS handshake with a 10-second timeout to prevent slowloris attacks. /// Returns `None` on handshake failure or timeout. -async fn tls_handshake( +pub(crate) async fn tls_handshake( acceptor: TlsAcceptor, stream: TcpStream, peer: SocketAddr, @@ -465,16 +465,36 @@ pub async fn run_threaded( Arc::clone(&slow_log), Arc::clone(&pubsub), ); - info!("gRPC listening on {grpc_addr}"); - let server = tonic::transport::Server::builder() + // gRPC takes the TLS settings of the RESP TLS port, so a server + // that encrypts RESP doesn't serve gRPC in plaintext + let tls = match tls.as_ref() { + Some((_, tls_config)) => Some(crate::tls::load_grpc_acceptor(tls_config)?), + None => None, + }; + let router = tonic::transport::Server::builder() .concurrency_limit_per_connection(256) .http2_keepalive_interval(Some(Duration::from_secs(30))) .http2_keepalive_timeout(Some(Duration::from_secs(10))) .tcp_keepalive(Some(Duration::from_secs(60))) - .add_service(svc.into_service()) - .serve(grpc_addr); + .add_service(svc.into_service()); Some(tokio::spawn(async move { - if let Err(e) = server.await { + let result = match tls { + Some(acceptor) => match crate::grpc::tls::incoming(grpc_addr, acceptor).await { + Ok(incoming) => { + info!("gRPC listening on {grpc_addr} with TLS"); + router.serve_with_incoming(incoming).await + } + Err(e) => { + error!("failed to bind gRPC on {grpc_addr}: {e}"); + return; + } + }, + None => { + info!("gRPC listening on {grpc_addr}"); + router.serve(grpc_addr).await + } + }; + if let Err(e) = result { error!("gRPC server error: {e}"); } })) diff --git a/crates/ember-server/src/tls.rs b/crates/ember-server/src/tls.rs index 33d7776..dcc8ffc 100644 --- a/crates/ember-server/src/tls.rs +++ b/crates/ember-server/src/tls.rs @@ -75,12 +75,26 @@ fn require_file(path: &Path, make_err: impl FnOnce(String) -> TlsError) -> Resul } } -/// Loads TLS configuration and creates a `TlsAcceptor`. +/// Loads TLS configuration and creates a `TlsAcceptor` for RESP clients. +pub fn load_tls_acceptor(config: &TlsConfig) -> Result { + Ok(TlsAcceptor::from(Arc::new(load_server_config(config)?))) +} + +/// Creates a `TlsAcceptor` for gRPC clients from the same configuration. +/// gRPC runs over HTTP/2, which clients ask for during the handshake. +#[cfg(feature = "grpc")] +pub fn load_grpc_acceptor(config: &TlsConfig) -> Result { + let mut server_config = load_server_config(config)?; + server_config.alpn_protocols = vec![b"h2".to_vec()]; + Ok(TlsAcceptor::from(Arc::new(server_config))) +} + +/// Builds the rustls server configuration. /// /// Reads certificates and private key from PEM files. If `ca_cert_file` is /// specified, sets up client certificate verification. When `auth_clients` /// is true, clients must present valid certificates signed by the CA. -pub fn load_tls_acceptor(config: &TlsConfig) -> Result { +fn load_server_config(config: &TlsConfig) -> Result { // load server certificates let cert_path = Path::new(&config.cert_file); require_file(cert_path, TlsError::CertFileNotFound)?; @@ -146,7 +160,7 @@ pub fn load_tls_acceptor(config: &TlsConfig) -> Result { .with_single_cert(certs, key)? }; - Ok(TlsAcceptor::from(Arc::new(server_config))) + Ok(server_config) } #[cfg(test)] diff --git a/docs/production-checklist.md b/docs/production-checklist.md index c9bb45d..0607eb4 100644 --- a/docs/production-checklist.md +++ b/docs/production-checklist.md @@ -203,13 +203,15 @@ run `INFO all` to get everything at once. ember supports TLS for encrypting client connections. configure it in your config file: ```toml -tls-port = 6380 +tls-port = 6381 tls-cert-file = /etc/ember/server.crt tls-key-file = /etc/ember/server.key ``` the plain RESP port and the TLS port operate independently. you can run both simultaneously during a migration, or disable the plain port entirely once clients are updated. +when TLS is configured, the gRPC port (6380 by default) uses the same certificate, key and client certificate settings, and no longer accepts plaintext. gRPC clients must connect with TLS and negotiate HTTP/2 (ALPN `h2`), which standard gRPC libraries do. the TLS port and the gRPC port must differ. + **client certificate verification (mtls)** to require clients to present a certificate: diff --git a/tests/integration/Cargo.toml b/tests/integration/Cargo.toml index 3efcfd8..d45bada 100644 --- a/tests/integration/Cargo.toml +++ b/tests/integration/Cargo.toml @@ -25,6 +25,9 @@ tokio-rustls = { workspace = true } tokio-stream = { workspace = true } tonic = { workspace = true } prost = { workspace = true } +# both come with tonic; the gRPC TLS test connects through them +hyper-util = { version = "0.1", features = ["tokio"] } +tower = "0.5" [build-dependencies] tonic-build = { workspace = true } diff --git a/tests/integration/src/grpc.rs b/tests/integration/src/grpc.rs index 165338f..e93fed6 100644 --- a/tests/integration/src/grpc.rs +++ b/tests/integration/src/grpc.rs @@ -2655,3 +2655,61 @@ async fn grpc_pipeline_errors_do_not_abort() { assert_eq!(resp.id, 7); assert!(matches!(resp.result, Some(PipeResult::Error(_)))); } + +#[tokio::test] +async fn grpc_uses_the_tls_settings_of_the_tls_port() { + use std::sync::Arc; + + use hyper_util::rt::TokioIo; + use rustls::pki_types::ServerName; + use tonic::transport::Endpoint; + + let tmp = tempfile::tempdir().unwrap(); + let (cert_path, key_path, cert_der) = crate::tls::generate_test_cert(tmp.path()); + let server = TestServer::start_with(ServerOptions { + grpc: true, + tls_cert_file: Some(cert_path), + tls_key_file: Some(key_path), + ..Default::default() + }); + let port = server.grpc_port.unwrap(); + + // a plaintext client gets nowhere + let plain = match EmberCacheClient::connect(format!("http://127.0.0.1:{port}")).await { + Ok(mut c) => c.ping(PingRequest { message: None }).await.is_ok(), + Err(_) => false, + }; + assert!(!plain, "gRPC answered in plaintext"); + + // a client that trusts the server's certificate and asks for HTTP/2 + let mut config = crate::tls::client_config(cert_der); + config.alpn_protocols = vec![b"h2".to_vec()]; + let connector = tokio_rustls::TlsConnector::from(Arc::new(config)); + let channel = Endpoint::from_static("http://localhost") + .connect_with_connector(tower::service_fn(move |_| { + let connector = connector.clone(); + async move { + let tcp = tokio::net::TcpStream::connect(("127.0.0.1", port)).await?; + let name = ServerName::try_from("localhost").unwrap(); + let tls = connector.connect(name, tcp).await?; + Ok::<_, std::io::Error>(TokioIo::new(tls)) + } + })) + .await + .expect("TLS connection failed"); + let mut client = EmberCacheClient::new(channel); + let reply = client.ping(PingRequest { message: None }).await.unwrap(); + assert_eq!(reply.into_inner().message, "PONG"); +} + +#[test] +fn the_tls_port_and_the_grpc_port_must_differ() { + let output = std::process::Command::new(crate::helpers::server_binary()) + .args(["--port", "0", "--tls-port", "16998", "--grpc-port", "16998"]) + .args(["--tls-cert-file", "cert.pem", "--tls-key-file", "key.pem"]) + .output() + .unwrap(); + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("tls-port and the gRPC port"), "{stderr}"); +} diff --git a/tests/integration/src/tls.rs b/tests/integration/src/tls.rs index cd484c2..2895f24 100644 --- a/tests/integration/src/tls.rs +++ b/tests/integration/src/tls.rs @@ -17,7 +17,9 @@ use crate::helpers::{ServerOptions, TestServer}; /// Generates a self-signed cert/key pair for `localhost` and writes PEM files /// into the given directory. Returns the cert path, key path, and the raw DER /// bytes needed to build a client-side trust store. -fn generate_test_cert(dir: &std::path::Path) -> (std::path::PathBuf, std::path::PathBuf, Vec) { +pub(crate) fn generate_test_cert( + dir: &std::path::Path, +) -> (std::path::PathBuf, std::path::PathBuf, Vec) { let rcgen::CertifiedKey { cert, key_pair } = rcgen::generate_simple_self_signed(vec!["localhost".to_string()]) .expect("rcgen cert generation failed"); @@ -38,7 +40,7 @@ fn generate_test_cert(dir: &std::path::Path) -> (std::path::PathBuf, std::path:: /// /// This simulates a client that has been explicitly configured to trust the /// server's self-signed cert — the same trust model as pinned certificates. -fn client_config(cert_der: Vec) -> rustls::ClientConfig { +pub(crate) fn client_config(cert_der: Vec) -> rustls::ClientConfig { let mut root_store = rustls::RootCertStore::empty(); root_store .add(CertificateDer::from(cert_der))