//! TLS configuration helpers for the gRPC transport. //! //! Two halves: //! //! * **Client** ([`client_tls_config`]): builds tonic's [`ClientTlsConfig`] for //! outbound peer channels — pins the cluster CA and attaches this node's client //! identity for mutual TLS. Unchanged since the optional-mTLS era. //! * **Server** (m11p7): the inbound gRPC server is NO LONGER served through //! tonic's `ServerTlsConfig`. tonic 0.12 caches a FIXED `Arc` //! built once from a static identity and exposes no resolver hook, so a //! running server's certificate can never be hot-swapped through its public //! API — "rotation without restart" would force a connection-dropping rebuild. //! Instead [`build_server_config`] builds a rustls [`ServerConfig`] whose //! identity is a [`DynamicCertResolver`] (an [`ArcSwap`] over the current //! [`CertifiedKey`]). [`server`](crate::server) serves over a `tokio-rustls` //! acceptor fed by that config, and [`ServerCertReloader`] swaps the resolver's //! cert on rotation. In-flight TLS sessions keep their already-negotiated keys, //! so a rotation drops zero connections; only NEW handshakes pick up the new //! cert. mTLS is preserved exactly as tonic enforced it: a //! [`WebPkiClientVerifier`] over the cluster CA roots (NOT //! `allow_unauthenticated`), so a peer with no/foreign client cert is rejected //! at the handshake before any RPC is reached. use std::collections::hash_map::DefaultHasher; use std::fs; use std::hash::Hasher; use std::path::Path; use std::sync::{Arc, Mutex, PoisonError}; use arc_swap::ArcSwap; use rustls::pki_types::{CertificateDer, PrivateKeyDer}; use rustls::server::{ClientHello, ResolvesServerCert, WebPkiClientVerifier}; use rustls::sign::CertifiedKey; use rustls::{RootCertStore, ServerConfig}; use tonic::transport::{Certificate, ClientTlsConfig, Identity}; use crate::{config::TlsConfig, error::GrpcTransportError}; /// ALPN protocol identifier for HTTP/2. tonic's own TLS acceptor pushes this /// onto the rustls config's `alpn_protocols`; the custom acceptor MUST do the /// same or h2 negotiation fails and every gRPC handshake is rejected. const ALPN_H2: &[u8] = b"h2"; /// ALPN protocol identifier for HTTP/1.1 (the HTTP plane also serves external /// bearer clients that may speak h1). const ALPN_HTTP11: &[u8] = b"http/1.1"; /// Build a tonic [`ClientTlsConfig`] for outbound peer channels. /// /// Always pins the cluster CA. Attaches this node's client identity (for mutual /// TLS) only when BOTH `client_cert` and `client_key` are configured — a peer /// with no client identity builds a server-authenticated-only channel that any /// mTLS-enforcing peer rejects at the handshake. /// /// # Errors /// /// Returns [`GrpcTransportError`] if the CA cert (or, when mutual TLS is /// configured, the client cert/key) file cannot be read. pub fn client_tls_config(tls: &TlsConfig) -> Result { let ca_cert = fs::read(&tls.ca_cert) .map_err(|e| GrpcTransportError::TlsConfig(format!("read CA cert: {e}")))?; let ca = Certificate::from_pem(ca_cert); let mut config = ClientTlsConfig::new().ca_certificate(ca); if let (Some(cert_path), Some(key_path)) = (&tls.client_cert, &tls.client_key) { let cert = fs::read(cert_path) .map_err(|e| GrpcTransportError::TlsConfig(format!("read client cert: {e}")))?; let key = fs::read(key_path) .map_err(|e| GrpcTransportError::TlsConfig(format!("read client key: {e}")))?; config = config.identity(Identity::from_pem(cert, key)); } Ok(config) } /// Read a PEM certificate chain from `path` into rustls DER certificates. fn read_cert_chain(path: &Path) -> Result>, GrpcTransportError> { let pem = fs::read(path) .map_err(|e| GrpcTransportError::TlsConfig(format!("read cert {}: {e}", path.display())))?; rustls_pemfile::certs(&mut pem.as_slice()) .collect::, _>>() .map_err(|e| GrpcTransportError::TlsConfig(format!("parse cert {}: {e}", path.display()))) } /// Read a single PEM private key from `path` into a rustls DER key. fn read_private_key(path: &Path) -> Result, GrpcTransportError> { let pem = fs::read(path) .map_err(|e| GrpcTransportError::TlsConfig(format!("read key {}: {e}", path.display())))?; rustls_pemfile::private_key(&mut pem.as_slice()) .map_err(|e| GrpcTransportError::TlsConfig(format!("parse key {}: {e}", path.display())))? .ok_or_else(|| { GrpcTransportError::TlsConfig(format!("no private key found in {}", path.display())) }) } /// Load the server's certificate chain + private key from PEM files into a /// rustls [`CertifiedKey`] ready to hand to the [`DynamicCertResolver`]. /// /// Re-read on every rotation: the paths stay fixed (a k8s secret mount swaps the /// file *content* behind a stable path), so reloading is exactly this call again. /// /// # Errors /// /// Returns [`GrpcTransportError::TlsConfig`] if a file cannot be read, the PEM is /// malformed, or the key does not match the cert chain. pub fn load_certified_key( cert_path: &Path, key_path: &Path, ) -> Result, GrpcTransportError> { crate::transport::ensure_crypto_provider(); let chain = read_cert_chain(cert_path)?; if chain.is_empty() { return Err(GrpcTransportError::TlsConfig(format!( "no certificates found in {}", cert_path.display() ))); } let key = read_private_key(key_path)?; let provider = rustls::crypto::aws_lc_rs::default_provider(); let certified = CertifiedKey::from_der(chain, key, &provider).map_err(|e| { GrpcTransportError::TlsConfig(format!( "server cert {} / key {} do not form a valid identity: {e}", cert_path.display(), key_path.display() )) })?; Ok(Arc::new(certified)) } /// A [`ResolvesServerCert`] whose certificate can be hot-swapped (m11p7). /// /// The current [`CertifiedKey`] lives behind an [`ArcSwap`]: the TLS handshake /// path reads it lock-free on every `resolve`, and [`ServerCertReloader`] swaps /// in a freshly-loaded cert with a single atomic store. Already-negotiated TLS /// sessions are unaffected (rustls captured their keys at handshake time), so a /// rotation drops zero live connections — only new handshakes see the new cert. #[derive(Debug)] pub struct DynamicCertResolver { current: ArcSwap, } impl DynamicCertResolver { /// Build a resolver serving `initial` until the first rotation. #[must_use] pub fn new(initial: Arc) -> Self { Self { current: ArcSwap::from(initial), } } /// Atomically replace the served certificate. The next handshake uses it; /// in-flight sessions are untouched. pub fn store(&self, key: Arc) { self.current.store(key); } } impl ResolvesServerCert for DynamicCertResolver { fn resolve(&self, _client_hello: ClientHello<'_>) -> Option> { // The cluster presents one identity to every peer (SNI-independent): the // resolver ignores the ClientHello and always serves the current cert. Some(self.current.load_full()) } } /// Build the rustls [`ServerConfig`] for the inbound gRPC server (m11p7). /// /// Enforces mutual TLS exactly as tonic's `ServerTlsConfig` did — a /// [`WebPkiClientVerifier`] built over the cluster CA roots with NO /// `allow_unauthenticated`, so the handshake REQUIRES and verifies a client cert /// chained to the cluster CA. The server identity is the supplied /// [`DynamicCertResolver`] (hot-swappable), and `h2` is advertised over ALPN so /// HTTP/2 negotiates. /// /// # Errors /// /// Returns [`GrpcTransportError::TlsConfig`] if the CA cert cannot be read/parsed, /// a CA certificate is not a valid trust anchor, or the client verifier cannot be /// built. pub fn build_server_config( tls: &TlsConfig, resolver: Arc, ) -> Result, GrpcTransportError> { crate::transport::ensure_crypto_provider(); let ca_chain = read_cert_chain(&tls.ca_cert)?; let mut roots = RootCertStore::empty(); for ca in ca_chain { roots.add(ca).map_err(|e| { GrpcTransportError::TlsConfig(format!( "CA cert {} is not a valid trust anchor: {e}", tls.ca_cert.display() )) })?; } let verifier = WebPkiClientVerifier::builder(Arc::new(roots)) .build() .map_err(|e| GrpcTransportError::TlsConfig(format!("build mTLS client verifier: {e}")))?; let mut config = ServerConfig::builder() .with_client_cert_verifier(verifier) .with_cert_resolver(resolver); config.alpn_protocols = vec![ALPN_H2.to_vec()]; Ok(Arc::new(config)) } /// Build a rustls [`ServerConfig`] for the inter-node HTTP listener (m11p7). /// /// Server-authenticated TLS only — NO client-cert verifier — because the HTTP /// listener also serves EXTERNAL clients (apps with a bearer token, not cluster /// certs); per-node identity on this plane is the signed internal token, not a /// client cert. The server identity is the same hot-swappable /// [`DynamicCertResolver`] the gRPC server uses, so a cert rotation covers both /// planes. ALPN advertises `h2` then `http/1.1` so HTTP/2 and HTTP/1.1 clients /// both negotiate (reqwest forwards use h2; curl/probes may use 1.1). #[must_use] pub fn build_http_server_config(resolver: Arc) -> Arc { crate::transport::ensure_crypto_provider(); let mut config = ServerConfig::builder() .with_no_client_auth() .with_cert_resolver(resolver); config.alpn_protocols = vec![ALPN_H2.to_vec(), ALPN_HTTP11.to_vec()]; Arc::new(config) } /// A content fingerprint over the configured TLS material (m11p7 rotation). /// /// Hashes the bytes of every present cert/key file. A change in ANY of them /// (CA, server cert/key, client cert/key) yields a new fingerprint and triggers /// a reload + peer-channel rebuild. Content hashing (not mtime/inotify) is the /// robust signal for Kubernetes secret rotation, which swaps a `..data` symlink /// atomically — a mode inotify watchers routinely miss. /// /// # Errors /// /// Returns the underlying [`std::io::Error`] if a configured file cannot be read. fn cert_fingerprint(tls: &TlsConfig) -> std::io::Result { let mut hasher = DefaultHasher::new(); let paths = [ Some(tls.ca_cert.as_path()), Some(tls.server_cert.as_path()), Some(tls.server_key.as_path()), tls.client_cert.as_deref(), tls.client_key.as_deref(), ]; for path in paths.into_iter().flatten() { // Hash the path too so swapping which file is empty is still detected. hasher.write(path.to_string_lossy().as_bytes()); hasher.write(&fs::read(path)?); } Ok(hasher.finish()) } /// Watches the TLS files and hot-swaps the [`DynamicCertResolver`] when they /// change (m11p7 cert rotation without restart). /// /// One per transport, polled on a timer by [`crate::transport`]. The poll is /// content-hash based (see [`cert_fingerprint`]); the reloader holds the last /// fingerprint so an unchanged poll is a single set of file reads and no work. pub struct ServerCertReloader { resolver: Arc, tls: TlsConfig, last_fingerprint: Mutex, } impl ServerCertReloader { /// Build a reloader seeded with the fingerprint of the material already /// loaded into `resolver`, so the first changed poll — not the first poll — /// is what triggers a reload. #[must_use] pub const fn new( resolver: Arc, tls: TlsConfig, initial_fingerprint: u64, ) -> Self { Self { resolver, tls, last_fingerprint: Mutex::new(initial_fingerprint), } } /// The current fingerprint of the configured TLS files, for seeding /// [`new`](Self::new). Returns 0 if any file is unreadable at construction /// (the first poll then reloads, surfacing the real error there). #[must_use] pub fn fingerprint(tls: &TlsConfig) -> u64 { cert_fingerprint(tls).unwrap_or(0) } /// Poll the TLS files once. If their content changed since the last poll, /// reload the server identity into the resolver and return `Ok(true)`. /// /// A reload failure (a half-written cert file mid-rotation, a transient read /// error) leaves the current cert in place and is surfaced as `Err` — the /// caller logs it and keeps serving the old identity rather than going dark. /// /// # Errors /// /// Returns [`GrpcTransportError::TlsConfig`] if the files cannot be read or /// the new material is not a valid identity. pub fn poll_once(&self) -> Result { let fingerprint = cert_fingerprint(&self.tls).map_err(|e| { GrpcTransportError::TlsConfig(format!("read TLS files for rotation: {e}")) })?; let mut last = self .last_fingerprint .lock() .unwrap_or_else(PoisonError::into_inner); if fingerprint == *last { return Ok(false); } // Build the new identity BEFORE committing the fingerprint: if the files // are mid-write (cert updated, key not yet) the load fails and we retry // next poll against the still-changed fingerprint. let key = load_certified_key(&self.tls.server_cert, &self.tls.server_key)?; self.resolver.store(key); *last = fingerprint; drop(last); Ok(true) } } #[cfg(test)] #[allow(clippy::unwrap_used)] // test assertions on known-good fixtures mod tests { use std::io::Write; use super::*; /// Generate a self-signed CA + a server leaf signed by it, written as PEM /// files into `dir`. Returns a [`TlsConfig`] pointing at them (server cert/ /// key + CA; client identity reuses the server leaf for the round-trip). fn write_test_certs(dir: &std::path::Path) -> TlsConfig { let mut ca_params = rcgen::CertificateParams::new(vec!["tidal-ca".to_string()]).unwrap(); ca_params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); let ca_key = rcgen::KeyPair::generate().unwrap(); let ca = ca_params.self_signed(&ca_key).unwrap(); let leaf_params = rcgen::CertificateParams::new(vec!["tidal-node".to_string()]).unwrap(); let leaf_key = rcgen::KeyPair::generate().unwrap(); let leaf = leaf_params.signed_by(&leaf_key, &ca, &ca_key).unwrap(); let ca_path = dir.join("ca.pem"); let cert_path = dir.join("server.pem"); let key_path = dir.join("server-key.pem"); std::fs::File::create(&ca_path) .unwrap() .write_all(ca.pem().as_bytes()) .unwrap(); std::fs::File::create(&cert_path) .unwrap() .write_all(leaf.pem().as_bytes()) .unwrap(); std::fs::File::create(&key_path) .unwrap() .write_all(leaf_key.serialize_pem().as_bytes()) .unwrap(); TlsConfig { ca_cert: ca_path, server_cert: cert_path, server_key: key_path, client_cert: None, client_key: None, } } #[test] fn server_config_builds_and_enforces_mtls() { let dir = tempfile::tempdir().unwrap(); let tls = write_test_certs(dir.path()); let key = load_certified_key(&tls.server_cert, &tls.server_key).unwrap(); let resolver = Arc::new(DynamicCertResolver::new(key)); let config = build_server_config(&tls, resolver).unwrap(); // ALPN must advertise h2 or gRPC handshakes are rejected. assert_eq!(config.alpn_protocols, vec![ALPN_H2.to_vec()]); } #[test] fn reloader_detects_content_change_and_swaps() { let dir = tempfile::tempdir().unwrap(); let tls = write_test_certs(dir.path()); let key = load_certified_key(&tls.server_cert, &tls.server_key).unwrap(); let resolver = Arc::new(DynamicCertResolver::new(key)); let initial = ServerCertReloader::fingerprint(&tls); let reloader = ServerCertReloader::new(Arc::clone(&resolver), tls.clone(), initial); // No change yet. assert!( !reloader.poll_once().unwrap(), "unchanged files must not reload" ); // Rotate the server identity to a brand-new leaf, written into a SEPARATE // subdir so its filenames do not collide with the watched paths, then // overwrite the watched cert/key with the rotated content (the k8s // secret-swap shape: same path, new bytes). let rotated_dir = dir.path().join("rotated"); std::fs::create_dir_all(&rotated_dir).unwrap(); let rotated = write_test_certs(&rotated_dir); std::fs::copy(&rotated.server_cert, &tls.server_cert).unwrap(); std::fs::copy(&rotated.server_key, &tls.server_key).unwrap(); assert!( reloader.poll_once().unwrap(), "a content change must trigger a reload" ); assert!( !reloader.poll_once().unwrap(), "a second poll with no further change must not reload again" ); } #[test] fn missing_cert_file_is_a_typed_error_not_a_panic() { let dir = tempfile::tempdir().unwrap(); let err = load_certified_key( &dir.path().join("absent.pem"), &dir.path().join("absent-key.pem"), ) .expect_err("absent cert must be a typed error"); assert!(matches!(err, GrpcTransportError::TlsConfig(_))); } }