tidaldb/tidal-server/tests/cluster_security.rs

//! m11p7 security hardening — exit-gate verification (in-process, real TLS).
//!
//! These run in the DEFAULT test build (no OS processes, like `cluster_region.rs`)
//! and exercise the m11p7 inter-node HTTP TLS primitives end to end over REAL
//! rustls: the [`TlsListener`] acceptor, the hot-swappable [`DynamicCertResolver`],
//! and `reqwest`'s CA-pinned client. Together with `tidal-net/tests/mtls.rs` (the
//! gRPC mTLS half: a foreign/absent client cert is rejected at the handshake, so a
//! foreign pod cannot ship segments) and the `cluster::security` unit tests (token
//! mint/verify, foreign-key + tamper rejection, marker-pinning, key rotation),
//! they cover the exit gate:
//!
//! * **zero plaintext inter-node links** — the HTTP listener serves TLS; a
//!   CA-trusting client connects, a plaintext/foreign-CA client cannot.
//! * **rotation under load drops zero requests** — a cert hot-swap mid-load is
//!   served with no dropped request.
//! * **a foreign pod cannot call internal routes** — a client that does not trust
//!   the cluster CA cannot establish the TLS connection at all.
#![allow(clippy::unwrap_used, clippy::missing_panics_doc)]

use std::net::SocketAddr;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;

use axum::{Router, routing::get};
use tidal_net::TlsConfig;
use tidal_server::cluster::http_tls::{HttpTls, TlsListener};

/// A self-signed CA + the leaf-signing material, kept so we can mint fresh leaves
/// under the SAME CA for the rotation test.
struct TestCa {
    ca_pem: String,
    ca: rcgen::Certificate,
    ca_key: rcgen::KeyPair,
}

fn generate_ca(common_name: &str) -> TestCa {
    let mut params = rcgen::CertificateParams::new(vec![common_name.to_string()]).unwrap();
    params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained);
    let ca_key = rcgen::KeyPair::generate().unwrap();
    let ca = params.self_signed(&ca_key).unwrap();
    TestCa {
        ca_pem: ca.pem(),
        ca,
        ca_key,
    }
}

/// Mint a leaf signed by `ca` with loopback SANs (127.0.0.1 + localhost), so a
/// peer dialing `https://127.0.0.1:PORT` verifies the name.
fn generate_leaf(ca: &TestCa) -> (String, String) {
    let params =
        rcgen::CertificateParams::new(vec!["127.0.0.1".to_string(), "localhost".to_string()])
            .unwrap();
    let key = rcgen::KeyPair::generate().unwrap();
    let leaf = params.signed_by(&key, &ca.ca, &ca.ca_key).unwrap();
    (leaf.pem(), key.serialize_pem())
}

/// Write a CA + a leaf into `dir` and return the [`TlsConfig`] pointing at them
/// (the leaf doubles as server + client identity, mirroring the cluster cert).
fn write_tls(dir: &std::path::Path, ca: &TestCa) -> TlsConfig {
    let (cert_pem, key_pem) = generate_leaf(ca);
    let ca_path = dir.join("ca.pem");
    let cert_path = dir.join("node.pem");
    let key_path = dir.join("node-key.pem");
    std::fs::write(&ca_path, ca.ca_pem.as_bytes()).unwrap();
    std::fs::write(&cert_path, cert_pem.as_bytes()).unwrap();
    std::fs::write(&key_path, key_pem.as_bytes()).unwrap();
    TlsConfig {
        ca_cert: ca_path,
        server_cert: cert_path.clone(),
        server_key: key_path.clone(),
        client_cert: Some(cert_path),
        client_key: Some(key_path),
    }
}

/// Build a small async runtime + serve `GET /ping -> "pong"` over the m11p7
/// `TlsListener` fed `tls`. Returns the runtime (keep it alive) and the bound addr.
fn serve_tls(tls: &TlsConfig) -> (tokio::runtime::Runtime, SocketAddr, Arc<HttpTls>) {
    let rt = tokio::runtime::Builder::new_multi_thread()
        .worker_threads(2)
        .enable_all()
        .build()
        .unwrap();
    let http_tls = Arc::new(HttpTls::from_tls_config(tls.clone()).expect("build HttpTls"));
    let addr: SocketAddr = "127.0.0.1:0".parse().unwrap();
    let cfg = Arc::clone(&http_tls.server_config);
    let (addr, listener) = rt.block_on(async move {
        let listener = TlsListener::bind(addr, cfg)
            .await
            .expect("bind TLS listener");
        (listener.local_addr(), listener)
    });
    let router = Router::new().route("/ping", get(|| async { "pong" }));
    rt.spawn(async move {
        let _ = axum::serve(listener, router).await;
    });
    // Give the listener a moment to be ready.
    std::thread::sleep(Duration::from_millis(150));
    (rt, addr, http_tls)
}

/// A blocking reqwest client that trusts ONLY `ca_pem` (the cluster CA).
fn client_trusting(ca_pem: &str) -> reqwest::blocking::Client {
    reqwest::blocking::Client::builder()
        .add_root_certificate(reqwest::Certificate::from_pem(ca_pem.as_bytes()).unwrap())
        .timeout(Duration::from_secs(5))
        .build()
        .unwrap()
}

/// EXIT GATE (encryption + foreign-pod): the HTTP listener serves TLS; a client
/// trusting the cluster CA succeeds, while a client trusting a DIFFERENT CA (a
/// foreign pod) cannot establish the connection at all — it never reaches a route.
#[test]
fn http_tls_serves_ca_trusting_client_and_rejects_foreign() {
    let dir = tempfile::tempdir().unwrap();
    let ca = generate_ca("tidal-test-ca");
    let tls = write_tls(dir.path(), &ca);
    let (_rt, addr, _http_tls) = serve_tls(&tls);

    // (1) A client trusting the cluster CA reaches the route over TLS.
    let ok_client = client_trusting(&ca.ca_pem);
    let resp = ok_client
        .get(format!("https://127.0.0.1:{}/ping", addr.port()))
        .send()
        .expect("CA-trusting client connects over TLS");
    assert!(resp.status().is_success());
    assert_eq!(resp.text().unwrap(), "pong");

    // (2) A foreign client trusting a DIFFERENT CA cannot complete the TLS
    // handshake — the request errors before any route is reached.
    let foreign_ca = generate_ca("rogue-ca");
    let foreign_client = client_trusting(&foreign_ca.ca_pem);
    let err = foreign_client
        .get(format!("https://127.0.0.1:{}/ping", addr.port()))
        .send();
    assert!(
        err.is_err(),
        "a client not trusting the cluster CA must fail the TLS handshake, got {err:?}"
    );

    // (3) A plaintext HTTP probe against the TLS port is also rejected (the
    // handshake never produces an HTTP/1 response).
    let plain = reqwest::blocking::Client::builder()
        .timeout(Duration::from_secs(3))
        .build()
        .unwrap();
    let plain_err = plain
        .get(format!("http://127.0.0.1:{}/ping", addr.port()))
        .send();
    assert!(
        plain_err.is_err() || !plain_err.unwrap().status().is_success(),
        "a plaintext probe against the TLS listener must not succeed"
    );
}

/// EXIT GATE (rotation under load drops zero requests): hammer the TLS listener
/// with concurrent requests while hot-swapping the server cert to a FRESH leaf
/// under the same CA. Every request must succeed — the in-flight sessions keep
/// their negotiated keys and new handshakes pick up the new cert, so a rotation
/// drops zero requests.
#[test]
fn http_tls_cert_rotation_under_load_drops_zero() {
    let dir = tempfile::tempdir().unwrap();
    let ca = generate_ca("tidal-test-ca");
    let tls = write_tls(dir.path(), &ca);
    let (_rt, addr, http_tls) = serve_tls(&tls);
    let url = format!("https://127.0.0.1:{}/ping", addr.port());

    let failures = Arc::new(AtomicU64::new(0));
    let oks = Arc::new(AtomicU64::new(0));
    let stop = Arc::new(std::sync::atomic::AtomicBool::new(false));

    // Several concurrent load threads, each on its own CA-pinned client (new
    // connections + reused ones), hitting /ping in a tight loop.
    let mut workers = Vec::new();
    for _ in 0..6 {
        let url = url.clone();
        let ca_pem = ca.ca_pem.clone();
        let failures = Arc::clone(&failures);
        let oks = Arc::clone(&oks);
        let stop = Arc::clone(&stop);
        workers.push(std::thread::spawn(move || {
            // A fresh client per worker; `pool_max_idle_per_host(0)` forces a NEW
            // TLS handshake on (most) requests so the rotation is actually
            // exercised on the handshake path, not just on warm keep-alive conns.
            let client = reqwest::blocking::Client::builder()
                .add_root_certificate(reqwest::Certificate::from_pem(ca_pem.as_bytes()).unwrap())
                .pool_max_idle_per_host(0)
                .timeout(Duration::from_secs(5))
                .build()
                .unwrap();
            while !stop.load(Ordering::Relaxed) {
                match client.get(&url).send() {
                    Ok(r) if r.status().is_success() => {
                        oks.fetch_add(1, Ordering::Relaxed);
                    }
                    _ => {
                        failures.fetch_add(1, Ordering::Relaxed);
                    }
                }
            }
        }));
    }

    // Rotate the server cert several times under load: mint a fresh leaf under the
    // SAME CA and hot-swap the resolver (the m11p7 zero-drop swap).
    for _ in 0..5 {
        std::thread::sleep(Duration::from_millis(80));
        let (cert_pem, key_pem) = generate_leaf(&ca);
        std::fs::write(&tls.server_cert, cert_pem.as_bytes()).unwrap();
        std::fs::write(&tls.server_key, key_pem.as_bytes()).unwrap();
        let fresh = tidal_net::load_certified_key(&tls.server_cert, &tls.server_key)
            .expect("load rotated cert");
        http_tls.resolver.store(fresh);
    }
    std::thread::sleep(Duration::from_millis(120));
    stop.store(true, Ordering::Relaxed);
    for w in workers {
        w.join().unwrap();
    }

    let failed = failures.load(Ordering::Relaxed);
    let succeeded = oks.load(Ordering::Relaxed);
    assert!(
        succeeded > 100,
        "expected sustained load, got {succeeded} ok"
    );
    assert_eq!(
        failed, 0,
        "cert rotation under load dropped {failed} request(s) (of {succeeded} ok) — must be zero"
    );
}