//! m11p7 security hardening — exit-gate verification (in-process, real TLS). //! //! These run in the DEFAULT test build (no OS processes, like `cluster_region.rs`) //! and exercise the m11p7 inter-node HTTP TLS primitives end to end over REAL //! rustls: the [`TlsListener`] acceptor, the hot-swappable [`DynamicCertResolver`], //! and `reqwest`'s CA-pinned client. Together with `tidal-net/tests/mtls.rs` (the //! gRPC mTLS half: a foreign/absent client cert is rejected at the handshake, so a //! foreign pod cannot ship segments) and the `cluster::security` unit tests (token //! mint/verify, foreign-key + tamper rejection, marker-pinning, key rotation), //! they cover the exit gate: //! //! * **zero plaintext inter-node links** — the HTTP listener serves TLS; a //! CA-trusting client connects, a plaintext/foreign-CA client cannot. //! * **rotation under load drops zero requests** — a cert hot-swap mid-load is //! served with no dropped request. //! * **a foreign pod cannot call internal routes** — a client that does not trust //! the cluster CA cannot establish the TLS connection at all. #![allow(clippy::unwrap_used, clippy::missing_panics_doc)] use std::net::SocketAddr; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::Duration; use axum::{Router, routing::get}; use tidal_net::TlsConfig; use tidal_server::cluster::http_tls::{HttpTls, TlsListener}; /// A self-signed CA + the leaf-signing material, kept so we can mint fresh leaves /// under the SAME CA for the rotation test. struct TestCa { ca_pem: String, ca: rcgen::Certificate, ca_key: rcgen::KeyPair, } fn generate_ca(common_name: &str) -> TestCa { let mut params = rcgen::CertificateParams::new(vec![common_name.to_string()]).unwrap(); params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); let ca_key = rcgen::KeyPair::generate().unwrap(); let ca = params.self_signed(&ca_key).unwrap(); TestCa { ca_pem: ca.pem(), ca, ca_key, } } /// Mint a leaf signed by `ca` with loopback SANs (127.0.0.1 + localhost), so a /// peer dialing `https://127.0.0.1:PORT` verifies the name. fn generate_leaf(ca: &TestCa) -> (String, String) { let params = rcgen::CertificateParams::new(vec!["127.0.0.1".to_string(), "localhost".to_string()]) .unwrap(); let key = rcgen::KeyPair::generate().unwrap(); let leaf = params.signed_by(&key, &ca.ca, &ca.ca_key).unwrap(); (leaf.pem(), key.serialize_pem()) } /// Write a CA + a leaf into `dir` and return the [`TlsConfig`] pointing at them /// (the leaf doubles as server + client identity, mirroring the cluster cert). fn write_tls(dir: &std::path::Path, ca: &TestCa) -> TlsConfig { let (cert_pem, key_pem) = generate_leaf(ca); let ca_path = dir.join("ca.pem"); let cert_path = dir.join("node.pem"); let key_path = dir.join("node-key.pem"); std::fs::write(&ca_path, ca.ca_pem.as_bytes()).unwrap(); std::fs::write(&cert_path, cert_pem.as_bytes()).unwrap(); std::fs::write(&key_path, key_pem.as_bytes()).unwrap(); TlsConfig { ca_cert: ca_path, server_cert: cert_path.clone(), server_key: key_path.clone(), client_cert: Some(cert_path), client_key: Some(key_path), } } /// Build a small async runtime + serve `GET /ping -> "pong"` over the m11p7 /// `TlsListener` fed `tls`. Returns the runtime (keep it alive) and the bound addr. fn serve_tls(tls: &TlsConfig) -> (tokio::runtime::Runtime, SocketAddr, Arc) { let rt = tokio::runtime::Builder::new_multi_thread() .worker_threads(2) .enable_all() .build() .unwrap(); let http_tls = Arc::new(HttpTls::from_tls_config(tls.clone()).expect("build HttpTls")); let addr: SocketAddr = "127.0.0.1:0".parse().unwrap(); let cfg = Arc::clone(&http_tls.server_config); let (addr, listener) = rt.block_on(async move { let listener = TlsListener::bind(addr, cfg) .await .expect("bind TLS listener"); (listener.local_addr(), listener) }); let router = Router::new().route("/ping", get(|| async { "pong" })); rt.spawn(async move { let _ = axum::serve(listener, router).await; }); // Give the listener a moment to be ready. std::thread::sleep(Duration::from_millis(150)); (rt, addr, http_tls) } /// A blocking reqwest client that trusts ONLY `ca_pem` (the cluster CA). fn client_trusting(ca_pem: &str) -> reqwest::blocking::Client { reqwest::blocking::Client::builder() .add_root_certificate(reqwest::Certificate::from_pem(ca_pem.as_bytes()).unwrap()) .timeout(Duration::from_secs(5)) .build() .unwrap() } /// EXIT GATE (encryption + foreign-pod): the HTTP listener serves TLS; a client /// trusting the cluster CA succeeds, while a client trusting a DIFFERENT CA (a /// foreign pod) cannot establish the connection at all — it never reaches a route. #[test] fn http_tls_serves_ca_trusting_client_and_rejects_foreign() { let dir = tempfile::tempdir().unwrap(); let ca = generate_ca("tidal-test-ca"); let tls = write_tls(dir.path(), &ca); let (_rt, addr, _http_tls) = serve_tls(&tls); // (1) A client trusting the cluster CA reaches the route over TLS. let ok_client = client_trusting(&ca.ca_pem); let resp = ok_client .get(format!("https://127.0.0.1:{}/ping", addr.port())) .send() .expect("CA-trusting client connects over TLS"); assert!(resp.status().is_success()); assert_eq!(resp.text().unwrap(), "pong"); // (2) A foreign client trusting a DIFFERENT CA cannot complete the TLS // handshake — the request errors before any route is reached. let foreign_ca = generate_ca("rogue-ca"); let foreign_client = client_trusting(&foreign_ca.ca_pem); let err = foreign_client .get(format!("https://127.0.0.1:{}/ping", addr.port())) .send(); assert!( err.is_err(), "a client not trusting the cluster CA must fail the TLS handshake, got {err:?}" ); // (3) A plaintext HTTP probe against the TLS port is also rejected (the // handshake never produces an HTTP/1 response). let plain = reqwest::blocking::Client::builder() .timeout(Duration::from_secs(3)) .build() .unwrap(); let plain_err = plain .get(format!("http://127.0.0.1:{}/ping", addr.port())) .send(); assert!( plain_err.is_err() || !plain_err.unwrap().status().is_success(), "a plaintext probe against the TLS listener must not succeed" ); } /// EXIT GATE (rotation under load drops zero requests): hammer the TLS listener /// with concurrent requests while hot-swapping the server cert to a FRESH leaf /// under the same CA. Every request must succeed — the in-flight sessions keep /// their negotiated keys and new handshakes pick up the new cert, so a rotation /// drops zero requests. #[test] fn http_tls_cert_rotation_under_load_drops_zero() { let dir = tempfile::tempdir().unwrap(); let ca = generate_ca("tidal-test-ca"); let tls = write_tls(dir.path(), &ca); let (_rt, addr, http_tls) = serve_tls(&tls); let url = format!("https://127.0.0.1:{}/ping", addr.port()); let failures = Arc::new(AtomicU64::new(0)); let oks = Arc::new(AtomicU64::new(0)); let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); // Several concurrent load threads, each on its own CA-pinned client (new // connections + reused ones), hitting /ping in a tight loop. let mut workers = Vec::new(); for _ in 0..6 { let url = url.clone(); let ca_pem = ca.ca_pem.clone(); let failures = Arc::clone(&failures); let oks = Arc::clone(&oks); let stop = Arc::clone(&stop); workers.push(std::thread::spawn(move || { // A fresh client per worker; `pool_max_idle_per_host(0)` forces a NEW // TLS handshake on (most) requests so the rotation is actually // exercised on the handshake path, not just on warm keep-alive conns. let client = reqwest::blocking::Client::builder() .add_root_certificate(reqwest::Certificate::from_pem(ca_pem.as_bytes()).unwrap()) .pool_max_idle_per_host(0) .timeout(Duration::from_secs(5)) .build() .unwrap(); while !stop.load(Ordering::Relaxed) { match client.get(&url).send() { Ok(r) if r.status().is_success() => { oks.fetch_add(1, Ordering::Relaxed); } _ => { failures.fetch_add(1, Ordering::Relaxed); } } } })); } // Rotate the server cert several times under load: mint a fresh leaf under the // SAME CA and hot-swap the resolver (the m11p7 zero-drop swap). for _ in 0..5 { std::thread::sleep(Duration::from_millis(80)); let (cert_pem, key_pem) = generate_leaf(&ca); std::fs::write(&tls.server_cert, cert_pem.as_bytes()).unwrap(); std::fs::write(&tls.server_key, key_pem.as_bytes()).unwrap(); let fresh = tidal_net::load_certified_key(&tls.server_cert, &tls.server_key) .expect("load rotated cert"); http_tls.resolver.store(fresh); } std::thread::sleep(Duration::from_millis(120)); stop.store(true, Ordering::Relaxed); for w in workers { w.join().unwrap(); } let failed = failures.load(Ordering::Relaxed); let succeeded = oks.load(Ordering::Relaxed); assert!( succeeded > 100, "expected sustained load, got {succeeded} ok" ); assert_eq!( failed, 0, "cert rotation under load dropped {failed} request(s) (of {succeeded} ok) — must be zero" ); }