//! gRPC transport wiring for the single-process cluster (m8p8). //! //! Each follower region is wired to a self-loop [`GrpcTransport`]: its gRPC //! server binds the follower's address and its single peer entry points at that //! same address, so a leader ship crosses a real gRPC/TCP hop into the //! follower's own inbound queue. The multi-process region node (m8p10) builds //! its transport directly in `cluster::node` with sibling peers, so these //! helpers are scoped to the single-process [`ClusterState`]. use std::{ collections::HashMap, net::{IpAddr, SocketAddr, TcpStream}, time::{Duration, Instant}, }; use tidal_net::{GrpcTransport, config::GrpcTransportConfig}; use tidaldb::replication::{ shard::{RegionId, ShardId}, transport::Transport, }; use super::topology::TopologySpec; use crate::error::{Result, ServerError}; /// How long to wait for each follower's gRPC server to bind before serving. pub(super) const GRPC_READY_TIMEOUT: Duration = Duration::from_secs(5); /// Poll interval while waiting for a gRPC server to become connectable. pub(super) const GRPC_READY_POLL: Duration = Duration::from_millis(20); /// Max attempts to bring a follower's gRPC server up on an auto-allocated port, /// retrying with a fresh port to self-heal a transient bind race. pub(super) const GRPC_BUILD_ATTEMPTS: u32 = 3; /// Build one real [`GrpcTransport`] per follower region. /// /// Each follower's transport is a **self-loop**: its gRPC server binds the /// follower's address and its single peer entry points at that same address, so /// a leader ship (`send_segment(follower_shard, …)` inside /// `SimulatedCluster::write_signal`) crosses a real gRPC/TCP hop into the /// follower's own inbound queue, which the follower's segment-receiver thread /// drains and applies. The leader region receives nothing, so it gets no /// transport. /// /// Returns a map keyed by follower [`RegionId`] suitable for /// `ClusterConfig::transports`. /// /// # Blocking /// /// Each [`GrpcTransport::new`] blocks on its own runtime and is then verified /// ready — must run on a non-async thread. pub(super) fn build_grpc_transports( topology: &TopologySpec, name_to_id: &HashMap, leader_id: RegionId, ) -> Result>> { let mut transports: HashMap> = HashMap::new(); for region in &topology.regions { let region_id = *name_to_id .get(®ion.name) .expect("region name was just inserted into name_to_id"); if region_id == leader_id { continue; // the leader applies writes locally and receives nothing } let shard = ShardId(region_id.0); let transport = build_ready_follower_transport(shard, region.grpc_addr.as_deref(), ®ion.name)?; transports.insert(region_id, std::sync::Arc::new(transport)); } Ok(transports) } /// Build a follower's self-loop gRPC transport and block until its server is /// connectable. /// /// A transport whose server does not come up within [`GRPC_READY_TIMEOUT`] is /// dropped (freeing its runtime and port) and retried on a freshly-allocated /// loopback port — this self-heals the rare bind race where the port probed by /// [`free_loopback_addr`] is taken before tonic rebinds it, which otherwise /// surfaces only as a silent serve failure and a startup timeout. An explicit /// operator-provided `grpc_addr` is tried once: reallocating would silently /// ignore the operator's chosen address. fn build_ready_follower_transport( shard: ShardId, addr_spec: Option<&str>, region_name: &str, ) -> Result { let attempts = if addr_spec.is_none() { GRPC_BUILD_ATTEMPTS } else { 1 }; let mut last = String::new(); for attempt in 1..=attempts { // Single-process self-loop: the bind address IS the ship target. The // bind resolver yields a concrete SocketAddr (literal `grpc_addr`, or an // auto-allocated loopback port when unset); the peer entry is that same // address stringified (m11p5: `peers` is now `host:port` strings). let addr = resolve_grpc_bind_addr(None, addr_spec, region_name)?; let mut peers = HashMap::new(); peers.insert(shard, addr.to_string()); // self-loop: ship target == own server let transport = GrpcTransport::new(GrpcTransportConfig { local_shard: shard, listen_addr: addr, peers, insecure: true, ..GrpcTransportConfig::default() }) .map_err(|e| { ServerError::Cluster(format!( "build gRPC transport for region '{region_name}' on {addr}: {e}" )) })?; if grpc_server_ready(addr) { tracing::info!(region = %region_name, %addr, attempt, "follower gRPC transport ready"); return Ok(transport); } last = format!( "gRPC server {addr} for region '{region_name}' did not become ready within \ {GRPC_READY_TIMEOUT:?}" ); tracing::warn!(region = %region_name, %addr, attempt, max = attempts, "{last}; retrying"); drop(transport); // free the runtime + port before the next attempt } Err(ServerError::Cluster(last)) } /// Block until `bind_addr`'s gRPC listener accepts a TCP connection, or /// [`GRPC_READY_TIMEOUT`] elapses. A successful connect proves the listener is /// bound, which is enough for the lazily-connected client's first /// `ship_segment`. /// /// **Unspecified-bind probe (m11p5):** a `0.0.0.0` / `[::]` bind cannot be /// *connected to* as a destination on every platform — it is a bind wildcard, /// not a routable address. When the bind IP is unspecified we probe /// `127.0.0.1:` (resp. `[::1]:`) instead, which the wildcard bind /// also accepts. A concrete bind IP (loopback or a specific interface) is /// probed as-is, preserving the single-process behavior byte-for-byte. pub(super) fn grpc_server_ready(bind_addr: SocketAddr) -> bool { let probe = probe_addr_for_bind(bind_addr); let deadline = Instant::now() + GRPC_READY_TIMEOUT; loop { if TcpStream::connect_timeout(&probe, GRPC_READY_POLL).is_ok() { return true; } if Instant::now() > deadline { return false; } std::thread::sleep(GRPC_READY_POLL); } } /// Map a bind address to the address a readiness probe should *connect* to: /// an unspecified wildcard bind (`0.0.0.0` / `[::]`) is substituted with the /// matching loopback (`127.0.0.1` / `[::1]`); any concrete address is probed /// unchanged. const fn probe_addr_for_bind(bind_addr: SocketAddr) -> SocketAddr { if bind_addr.ip().is_unspecified() { let loopback = match bind_addr.ip() { IpAddr::V4(_) => IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), IpAddr::V6(_) => IpAddr::V6(std::net::Ipv6Addr::LOCALHOST), }; SocketAddr::new(loopback, bind_addr.port()) } else { bind_addr } } /// Resolve THIS region's local gRPC **bind** address (a concrete, bindable /// [`SocketAddr`]) from the bind/advertise split (m11p5 §1). `grpc_addr` is the /// ADVERTISED address siblings dial — it may be a DNS name the local socket /// cannot bind — so the bind is derived separately by this table: /// /// | `grpc_bind` | `grpc_addr` | bind result | /// |--------------------|------------------------|------------------------------| /// | present | (any) | parse `grpc_bind` | /// | absent | literal `SocketAddr` | parse `grpc_addr` (today's) | /// | absent | hostname `host:port` | `0.0.0.0:`| /// | absent | hostname, no port | **reject** | /// | absent | absent | auto-allocated loopback port | /// /// The literal-IP row is byte-for-byte today's behavior: every existing /// topology keeps binding exactly what it bound before. The hostname rows are /// new — one shared topology file can name every region by its per-pod DNS /// name while each pod binds `0.0.0.0`. /// /// # Errors /// /// Returns [`ServerError::Cluster`] if `grpc_bind` does not parse as a /// [`SocketAddr`], if a hostname `grpc_addr` has no parseable `:port`, or if a /// loopback port cannot be allocated. pub(super) fn resolve_grpc_bind_addr( grpc_bind: Option<&str>, grpc_addr: Option<&str>, region_name: &str, ) -> Result { // 1. Explicit bind wins outright — it is a concrete local socket. if let Some(bind) = grpc_bind { return bind.parse().map_err(|e| { ServerError::Cluster(format!( "invalid grpc_bind '{bind}' for region '{region_name}': {e}" )) }); } // 2/5. No advertised address either → single-process auto-allocation. let Some(advertised) = grpc_addr else { return free_loopback_addr().map_err(|e| { ServerError::Cluster(format!( "allocate loopback gRPC port for region '{region_name}': {e}" )) }); }; // 2. A literal SocketAddr advertised address binds itself (today's path). if let Ok(addr) = advertised.parse::() { return Ok(addr); } // 3/4. A hostname advertised address cannot be bound; derive // `0.0.0.0:` from its port. No port → reject (an advertise address // without a port is meaningless and `validate_multiproc` already rejects // it; this is the defense-in-depth boundary at the bind seam). let Some((_host, port_str)) = advertised.rsplit_once(':') else { return Err(ServerError::Cluster(format!( "grpc_addr '{advertised}' for region '{region_name}' is a hostname with no \ ':port' — cannot derive a local bind (set grpc_bind explicitly or add a port)" ))); }; let port: u16 = port_str.parse().map_err(|e| { ServerError::Cluster(format!( "grpc_addr '{advertised}' for region '{region_name}' has an invalid port \ '{port_str}': {e}" )) })?; Ok(SocketAddr::new( IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED), port, )) } /// Reserve a free loopback port by binding `127.0.0.1:0` and reading back the /// assigned address. The probe listener is dropped immediately so tonic can /// bind the same port; the rare race where another bind steals the port in /// between is retried by [`build_ready_follower_transport`]. fn free_loopback_addr() -> std::io::Result { let listener = std::net::TcpListener::bind("127.0.0.1:0")?; listener.local_addr() } #[cfg(test)] mod tests { use super::*; /// Parse a `SocketAddr` test fixture, panicking with a clear message on a /// malformed literal (these are all compile-time-known good inputs). fn sa(s: &str) -> SocketAddr { s.parse() .unwrap_or_else(|e| panic!("test fixture '{s}' must parse as a SocketAddr: {e}")) } /// The m11p5 §1 bind/advertise derivation table, exhaustively. #[test] fn bind_derivation_table() { // grpc_bind present → it (advertise is ignored for the bind). let addr = resolve_grpc_bind_addr(Some("10.0.0.5:9601"), Some("tidaldb-1:9601"), "r") .expect("explicit grpc_bind binds itself"); assert_eq!(addr, sa("10.0.0.5:9601")); // grpc_bind absent + grpc_addr is a literal SocketAddr → grpc_addr // (today's behavior, byte-for-byte). let addr = resolve_grpc_bind_addr(None, Some("127.0.0.1:9601"), "r") .expect("literal grpc_addr binds itself"); assert_eq!(addr, sa("127.0.0.1:9601")); // grpc_bind absent + grpc_addr is a hostname → 0.0.0.0:. let addr = resolve_grpc_bind_addr(None, Some("tidaldb-1.peers.svc:9601"), "r") .expect("hostname grpc_addr binds 0.0.0.0:"); assert_eq!(addr, sa("0.0.0.0:9601")); // grpc_bind absent + grpc_addr is a hostname with NO port → reject. let err = resolve_grpc_bind_addr(None, Some("tidaldb-1"), "r") .expect_err("a portless hostname grpc_addr cannot derive a bind"); assert!( err.to_string().contains("no") && err.to_string().contains("port"), "error must name the missing port: {err}" ); } #[test] fn invalid_grpc_bind_is_rejected() { // A hostname is not a bindable local socket — grpc_bind must parse. let err = resolve_grpc_bind_addr(Some("tidaldb-1:9601"), None, "r") .expect_err("a hostname grpc_bind must be rejected"); assert!(err.to_string().contains("grpc_bind"), "got {err}"); } #[test] fn no_addresses_auto_allocates_loopback() { // Single-process default: neither bind nor advertise declared → an // OS-assigned loopback port (the behavior the single-process cluster // relies on, byte-for-byte). let addr = resolve_grpc_bind_addr(None, None, "r") .expect("an unset address auto-allocates a loopback port"); assert!( addr.ip().is_loopback(), "auto-allocation is loopback: {addr}" ); assert_ne!(addr.port(), 0, "a concrete port was assigned"); } #[test] fn probe_substitutes_loopback_for_unspecified_bind() { // A wildcard bind is probed via loopback (it is not itself a routable // destination); a concrete bind is probed unchanged. assert_eq!( probe_addr_for_bind(sa("0.0.0.0:9601")), sa("127.0.0.1:9601") ); assert_eq!(probe_addr_for_bind(sa("[::]:9601")), sa("[::1]:9601")); assert_eq!( probe_addr_for_bind(sa("127.0.0.1:9601")), sa("127.0.0.1:9601") ); assert_eq!( probe_addr_for_bind(sa("10.0.0.5:9601")), sa("10.0.0.5:9601") ); } }