tidaldb/tidal-server/src/cluster/transport.rs

//! gRPC transport wiring for the single-process cluster (m8p8).
//!
//! Each follower region is wired to a self-loop [`GrpcTransport`]: its gRPC
//! server binds the follower's address and its single peer entry points at that
//! same address, so a leader ship crosses a real gRPC/TCP hop into the
//! follower's own inbound queue. The multi-process region node (m8p10) builds
//! its transport directly in `cluster::node` with sibling peers, so these
//! helpers are scoped to the single-process [`ClusterState`].

use std::{
    collections::HashMap,
    net::{IpAddr, SocketAddr, TcpStream},
    time::{Duration, Instant},
};

use tidal_net::{GrpcTransport, config::GrpcTransportConfig};
use tidaldb::replication::{
    shard::{RegionId, ShardId},
    transport::Transport,
};

use super::topology::TopologySpec;
use crate::error::{Result, ServerError};

/// How long to wait for each follower's gRPC server to bind before serving.
pub(super) const GRPC_READY_TIMEOUT: Duration = Duration::from_secs(5);
/// Poll interval while waiting for a gRPC server to become connectable.
pub(super) const GRPC_READY_POLL: Duration = Duration::from_millis(20);
/// Max attempts to bring a follower's gRPC server up on an auto-allocated port,
/// retrying with a fresh port to self-heal a transient bind race.
pub(super) const GRPC_BUILD_ATTEMPTS: u32 = 3;

/// Build one real [`GrpcTransport`] per follower region.
///
/// Each follower's transport is a **self-loop**: its gRPC server binds the
/// follower's address and its single peer entry points at that same address, so
/// a leader ship (`send_segment(follower_shard, …)` inside
/// `SimulatedCluster::write_signal`) crosses a real gRPC/TCP hop into the
/// follower's own inbound queue, which the follower's segment-receiver thread
/// drains and applies. The leader region receives nothing, so it gets no
/// transport.
///
/// Returns a map keyed by follower [`RegionId`] suitable for
/// `ClusterConfig::transports`.
///
/// # Blocking
///
/// Each [`GrpcTransport::new`] blocks on its own runtime and is then verified
/// ready — must run on a non-async thread.
pub(super) fn build_grpc_transports(
    topology: &TopologySpec,
    name_to_id: &HashMap<String, RegionId>,
    leader_id: RegionId,
) -> Result<HashMap<RegionId, std::sync::Arc<dyn Transport>>> {
    let mut transports: HashMap<RegionId, std::sync::Arc<dyn Transport>> = HashMap::new();

    for region in &topology.regions {
        let region_id = *name_to_id
            .get(&region.name)
            .expect("region name was just inserted into name_to_id");
        if region_id == leader_id {
            continue; // the leader applies writes locally and receives nothing
        }

        let shard = ShardId(region_id.0);
        let transport =
            build_ready_follower_transport(shard, region.grpc_addr.as_deref(), &region.name)?;
        transports.insert(region_id, std::sync::Arc::new(transport));
    }

    Ok(transports)
}

/// Build a follower's self-loop gRPC transport and block until its server is
/// connectable.
///
/// A transport whose server does not come up within [`GRPC_READY_TIMEOUT`] is
/// dropped (freeing its runtime and port) and retried on a freshly-allocated
/// loopback port — this self-heals the rare bind race where the port probed by
/// [`free_loopback_addr`] is taken before tonic rebinds it, which otherwise
/// surfaces only as a silent serve failure and a startup timeout. An explicit
/// operator-provided `grpc_addr` is tried once: reallocating would silently
/// ignore the operator's chosen address.
fn build_ready_follower_transport(
    shard: ShardId,
    addr_spec: Option<&str>,
    region_name: &str,
) -> Result<GrpcTransport> {
    let attempts = if addr_spec.is_none() {
        GRPC_BUILD_ATTEMPTS
    } else {
        1
    };
    let mut last = String::new();

    for attempt in 1..=attempts {
        // Single-process self-loop: the bind address IS the ship target. The
        // bind resolver yields a concrete SocketAddr (literal `grpc_addr`, or an
        // auto-allocated loopback port when unset); the peer entry is that same
        // address stringified (m11p5: `peers` is now `host:port` strings).
        let addr = resolve_grpc_bind_addr(None, addr_spec, region_name)?;
        let mut peers = HashMap::new();
        peers.insert(shard, addr.to_string()); // self-loop: ship target == own server

        let transport = GrpcTransport::new(GrpcTransportConfig {
            local_shard: shard,
            listen_addr: addr,
            peers,
            insecure: true,
            ..GrpcTransportConfig::default()
        })
        .map_err(|e| {
            ServerError::Cluster(format!(
                "build gRPC transport for region '{region_name}' on {addr}: {e}"
            ))
        })?;

        if grpc_server_ready(addr) {
            tracing::info!(region = %region_name, %addr, attempt, "follower gRPC transport ready");
            return Ok(transport);
        }

        last = format!(
            "gRPC server {addr} for region '{region_name}' did not become ready within \
             {GRPC_READY_TIMEOUT:?}"
        );
        tracing::warn!(region = %region_name, %addr, attempt, max = attempts, "{last}; retrying");
        drop(transport); // free the runtime + port before the next attempt
    }

    Err(ServerError::Cluster(last))
}

/// Block until `bind_addr`'s gRPC listener accepts a TCP connection, or
/// [`GRPC_READY_TIMEOUT`] elapses. A successful connect proves the listener is
/// bound, which is enough for the lazily-connected client's first
/// `ship_segment`.
///
/// **Unspecified-bind probe (m11p5):** a `0.0.0.0` / `[::]` bind cannot be
/// *connected to* as a destination on every platform — it is a bind wildcard,
/// not a routable address. When the bind IP is unspecified we probe
/// `127.0.0.1:<port>` (resp. `[::1]:<port>`) instead, which the wildcard bind
/// also accepts. A concrete bind IP (loopback or a specific interface) is
/// probed as-is, preserving the single-process behavior byte-for-byte.
pub(super) fn grpc_server_ready(bind_addr: SocketAddr) -> bool {
    let probe = probe_addr_for_bind(bind_addr);
    let deadline = Instant::now() + GRPC_READY_TIMEOUT;
    loop {
        if TcpStream::connect_timeout(&probe, GRPC_READY_POLL).is_ok() {
            return true;
        }
        if Instant::now() > deadline {
            return false;
        }
        std::thread::sleep(GRPC_READY_POLL);
    }
}

/// Map a bind address to the address a readiness probe should *connect* to:
/// an unspecified wildcard bind (`0.0.0.0` / `[::]`) is substituted with the
/// matching loopback (`127.0.0.1` / `[::1]`); any concrete address is probed
/// unchanged.
const fn probe_addr_for_bind(bind_addr: SocketAddr) -> SocketAddr {
    if bind_addr.ip().is_unspecified() {
        let loopback = match bind_addr.ip() {
            IpAddr::V4(_) => IpAddr::V4(std::net::Ipv4Addr::LOCALHOST),
            IpAddr::V6(_) => IpAddr::V6(std::net::Ipv6Addr::LOCALHOST),
        };
        SocketAddr::new(loopback, bind_addr.port())
    } else {
        bind_addr
    }
}

/// Resolve THIS region's local gRPC **bind** address (a concrete, bindable
/// [`SocketAddr`]) from the bind/advertise split (m11p5 §1). `grpc_addr` is the
/// ADVERTISED address siblings dial — it may be a DNS name the local socket
/// cannot bind — so the bind is derived separately by this table:
///
/// | `grpc_bind`        | `grpc_addr`            | bind result                  |
/// |--------------------|------------------------|------------------------------|
/// | present            | (any)                  | parse `grpc_bind`            |
/// | absent             | literal `SocketAddr`   | parse `grpc_addr` (today's)  |
/// | absent             | hostname `host:port`   | `0.0.0.0:<port-of-grpc_addr>`|
/// | absent             | hostname, no port      | **reject**                   |
/// | absent             | absent                 | auto-allocated loopback port |
///
/// The literal-IP row is byte-for-byte today's behavior: every existing
/// topology keeps binding exactly what it bound before. The hostname rows are
/// new — one shared topology file can name every region by its per-pod DNS
/// name while each pod binds `0.0.0.0`.
///
/// # Errors
///
/// Returns [`ServerError::Cluster`] if `grpc_bind` does not parse as a
/// [`SocketAddr`], if a hostname `grpc_addr` has no parseable `:port`, or if a
/// loopback port cannot be allocated.
pub(super) fn resolve_grpc_bind_addr(
    grpc_bind: Option<&str>,
    grpc_addr: Option<&str>,
    region_name: &str,
) -> Result<SocketAddr> {
    // 1. Explicit bind wins outright — it is a concrete local socket.
    if let Some(bind) = grpc_bind {
        return bind.parse().map_err(|e| {
            ServerError::Cluster(format!(
                "invalid grpc_bind '{bind}' for region '{region_name}': {e}"
            ))
        });
    }

    // 2/5. No advertised address either → single-process auto-allocation.
    let Some(advertised) = grpc_addr else {
        return free_loopback_addr().map_err(|e| {
            ServerError::Cluster(format!(
                "allocate loopback gRPC port for region '{region_name}': {e}"
            ))
        });
    };

    // 2. A literal SocketAddr advertised address binds itself (today's path).
    if let Ok(addr) = advertised.parse::<SocketAddr>() {
        return Ok(addr);
    }

    // 3/4. A hostname advertised address cannot be bound; derive
    // `0.0.0.0:<port>` from its port. No port → reject (an advertise address
    // without a port is meaningless and `validate_multiproc` already rejects
    // it; this is the defense-in-depth boundary at the bind seam).
    let Some((_host, port_str)) = advertised.rsplit_once(':') else {
        return Err(ServerError::Cluster(format!(
            "grpc_addr '{advertised}' for region '{region_name}' is a hostname with no \
             ':port' — cannot derive a local bind (set grpc_bind explicitly or add a port)"
        )));
    };
    let port: u16 = port_str.parse().map_err(|e| {
        ServerError::Cluster(format!(
            "grpc_addr '{advertised}' for region '{region_name}' has an invalid port \
             '{port_str}': {e}"
        ))
    })?;
    Ok(SocketAddr::new(
        IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED),
        port,
    ))
}

/// Reserve a free loopback port by binding `127.0.0.1:0` and reading back the
/// assigned address. The probe listener is dropped immediately so tonic can
/// bind the same port; the rare race where another bind steals the port in
/// between is retried by [`build_ready_follower_transport`].
fn free_loopback_addr() -> std::io::Result<SocketAddr> {
    let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
    listener.local_addr()
}

#[cfg(test)]
mod tests {
    use super::*;

    /// Parse a `SocketAddr` test fixture, panicking with a clear message on a
    /// malformed literal (these are all compile-time-known good inputs).
    fn sa(s: &str) -> SocketAddr {
        s.parse()
            .unwrap_or_else(|e| panic!("test fixture '{s}' must parse as a SocketAddr: {e}"))
    }

    /// The m11p5 §1 bind/advertise derivation table, exhaustively.
    #[test]
    fn bind_derivation_table() {
        // grpc_bind present → it (advertise is ignored for the bind).
        let addr = resolve_grpc_bind_addr(Some("10.0.0.5:9601"), Some("tidaldb-1:9601"), "r")
            .expect("explicit grpc_bind binds itself");
        assert_eq!(addr, sa("10.0.0.5:9601"));

        // grpc_bind absent + grpc_addr is a literal SocketAddr → grpc_addr
        // (today's behavior, byte-for-byte).
        let addr = resolve_grpc_bind_addr(None, Some("127.0.0.1:9601"), "r")
            .expect("literal grpc_addr binds itself");
        assert_eq!(addr, sa("127.0.0.1:9601"));

        // grpc_bind absent + grpc_addr is a hostname → 0.0.0.0:<port>.
        let addr = resolve_grpc_bind_addr(None, Some("tidaldb-1.peers.svc:9601"), "r")
            .expect("hostname grpc_addr binds 0.0.0.0:<port>");
        assert_eq!(addr, sa("0.0.0.0:9601"));

        // grpc_bind absent + grpc_addr is a hostname with NO port → reject.
        let err = resolve_grpc_bind_addr(None, Some("tidaldb-1"), "r")
            .expect_err("a portless hostname grpc_addr cannot derive a bind");
        assert!(
            err.to_string().contains("no") && err.to_string().contains("port"),
            "error must name the missing port: {err}"
        );
    }

    #[test]
    fn invalid_grpc_bind_is_rejected() {
        // A hostname is not a bindable local socket — grpc_bind must parse.
        let err = resolve_grpc_bind_addr(Some("tidaldb-1:9601"), None, "r")
            .expect_err("a hostname grpc_bind must be rejected");
        assert!(err.to_string().contains("grpc_bind"), "got {err}");
    }

    #[test]
    fn no_addresses_auto_allocates_loopback() {
        // Single-process default: neither bind nor advertise declared → an
        // OS-assigned loopback port (the behavior the single-process cluster
        // relies on, byte-for-byte).
        let addr = resolve_grpc_bind_addr(None, None, "r")
            .expect("an unset address auto-allocates a loopback port");
        assert!(
            addr.ip().is_loopback(),
            "auto-allocation is loopback: {addr}"
        );
        assert_ne!(addr.port(), 0, "a concrete port was assigned");
    }

    #[test]
    fn probe_substitutes_loopback_for_unspecified_bind() {
        // A wildcard bind is probed via loopback (it is not itself a routable
        // destination); a concrete bind is probed unchanged.
        assert_eq!(
            probe_addr_for_bind(sa("0.0.0.0:9601")),
            sa("127.0.0.1:9601")
        );
        assert_eq!(probe_addr_for_bind(sa("[::]:9601")), sa("[::1]:9601"));
        assert_eq!(
            probe_addr_for_bind(sa("127.0.0.1:9601")),
            sa("127.0.0.1:9601")
        );
        assert_eq!(
            probe_addr_for_bind(sa("10.0.0.5:9601")),
            sa("10.0.0.5:9601")
        );
    }
}