//! Configuration for the gRPC transport. use std::{collections::HashMap, net::SocketAddr, path::PathBuf, time::Duration}; use tidaldb::replication::shard::ShardId; use crate::error::GrpcTransportError; /// Maximum payload size in bytes (64 MiB). /// /// **Source of truth:** the engine crate's `InProcessTransport::MAX_PAYLOAD_BYTES` /// (`tidaldb`, `replication/in_process.rs`). Both transports must reject the same /// over-size payloads so behavior does not diverge by which transport a deployment /// uses. That engine constant is currently private (module-local in `in_process.rs`), /// so it cannot be `use`d here directly; the value is mirrored and pinned by the /// compile-time cross-check below. If the engine ever exposes it (e.g. as /// `tidaldb::replication::MAX_PAYLOAD_BYTES`), replace this literal with a re-export /// of that constant and delete the cross-check. pub const MAX_PAYLOAD_BYTES: usize = 64 * 1024 * 1024; /// Compile-time guard pinning [`MAX_PAYLOAD_BYTES`] to the engine's 64 MiB limit. /// /// This catches accidental drift in the mirrored literal at build time (it is the /// best available cross-check while the engine constant stays private). If this /// fails to compile, the two crates' payload limits have diverged: reconcile them, /// preferring to make the engine's constant the single referenced source of truth. const _: () = assert!( MAX_PAYLOAD_BYTES == 64 * 1024 * 1024, "tidal-net MAX_PAYLOAD_BYTES must match the engine's InProcessTransport limit (64 MiB)" ); /// Configuration for a single [`GrpcTransport`](crate::GrpcTransport) instance. #[derive(Debug, Clone)] pub struct GrpcTransportConfig { /// This node's shard identity. pub local_shard: ShardId, /// Address to bind the gRPC server on. pub listen_addr: SocketAddr, /// Peer shard advertised addresses (`shard_id` -> `host:port`). /// /// A bare `host:port` string, NOT a pre-resolved [`SocketAddr`]. This is /// load-bearing for DNS peers (m11p5): [`PeerPool::new`] builds the tonic /// URI from the string, so `Channel::from_shared("http://hostname:port")` /// makes hyper re-resolve DNS on every (re)connect. A pre-resolved IP /// literal would pin the peer to whatever address it had at boot — the /// k8s pod-rescheduled-onto-a-new-IP case becomes structurally /// unreachable until process restart. A literal IP `host:port` is still /// accepted (and re-resolves trivially to itself), so every pre-m11p5 /// topology keeps working byte-for-byte. /// /// [`PeerPool::new`]: crate::client::PeerPool::new pub peers: HashMap, /// TLS configuration. `None` requires `insecure = true`. pub tls: Option, /// Allow plaintext connections (no TLS). pub insecure: bool, /// Capacity of the inbound segment channel. pub channel_capacity: usize, /// Maximum payload size in bytes. pub max_payload_bytes: usize, /// Number of consecutive failures before the circuit breaker opens. pub circuit_breaker_threshold: u32, /// Duration the circuit breaker stays open before allowing a probe. pub circuit_breaker_reset: Duration, /// Maximum time to establish a TCP/TLS connection to a peer before the /// attempt fails fast (and the failure trips the circuit breaker). Without /// this, a peer whose host is up but whose port silently blackholes the /// SYN would stall the single-threaded shipper indefinitely. pub connect_timeout: Duration, /// Maximum time for a single `ship_segment` RPC (connect + send + response) /// before it is aborted as a transient failure. Bounds the per-call latency /// the shipper can ever incur on a stalled peer. pub request_timeout: Duration, /// HTTP/2 PING keep-alive interval on the client channel. Idle and active /// connections send a PING this often; a peer that stops responding is /// detected within `keep_alive_interval + keep_alive_timeout` instead of /// hanging on a half-open connection. pub keep_alive_interval: Duration, /// How long to wait for a keep-alive PING ACK before declaring the /// connection dead and tearing it down. pub keep_alive_timeout: Duration, /// Delay before a FAILED catch-up pull is retried by the transport's /// timer (m11p4). The event path alone (re-trigger on the next pushed /// segment) deadlocks an IDLE cluster: a follower whose pull failed — /// e.g. the leader's gRPC server was not yet ready during a rolling /// restart — waited for a push that never came and stayed lagged /// indefinitely (the 2026-06-11 p3 rollout incident). The timer is the /// proactive wake-up alongside that event path; pulls stay single-flight /// and rate-limited regardless of which path triggers them. pub catchup_retry_interval: Duration, /// How often the cert-rotation reloader polls the TLS files for a content /// change (m11p7). Only spawned when TLS is configured; ignored on the /// plaintext path. Content-hash polling (not inotify) is the robust signal /// for Kubernetes secret rotation, which swaps a `..data` symlink atomically /// — a mode inotify watchers routinely miss. Default 30s; tests set it low /// to exercise rotation-under-load fast. pub rotation_poll_interval: Duration, /// Maximum wall-clock time a single INBOUND TLS handshake may take before it /// is abandoned (m11p7 hardening). The TCP accept happens before any client /// cert is verified, so a peer that completes the connect but stalls the TLS /// `ClientHello` (a slowloris) would otherwise pin a task + socket /// indefinitely, pre-auth. Default 10s; only used on the mTLS path. pub handshake_timeout: Duration, /// Maximum number of inbound TLS handshakes allowed in flight at once /// (m11p7 hardening). Bounds the task/fd cost of a connection flood against /// the replication port: when the limit is reached, a freshly-accepted /// connection is dropped (load-shed) before its handshake is attempted, /// rather than each spawning an unbounded task. Default 256; only the mTLS /// path enforces it. pub max_concurrent_handshakes: usize, } impl GrpcTransportConfig { /// Reject a config whose numeric invariants would silently break the /// transport rather than fail loudly. /// /// # Errors /// /// Returns [`GrpcTransportError::Internal`] if any of the following holds. /// Each is a misconfiguration that, left unchecked, degrades into a silent /// fault the operator cannot see at runtime: /// /// - `channel_capacity == 0`: [`tokio::sync::mpsc::channel`] panics on a /// zero capacity, so the transport would not even start — surface it as a /// typed error here instead of a panic deep in [`GrpcTransport::new`]. /// - any timeout is `Duration::ZERO` (`connect_timeout`, `request_timeout`, /// `keep_alive_interval`, `keep_alive_timeout`): a zero deadline aborts /// every attempt instantly, turning the fail-fast design into a transport /// that can never ship a single segment. /// - `max_payload_bytes == 0` or `> MAX_PAYLOAD_BYTES`: a zero ceiling /// rejects every segment; a ceiling above the engine's wire limit lets the /// client/server codec accept a payload the peer will refuse, so both ends /// must agree on the same in-range bound. /// - `circuit_breaker_reset == Duration::ZERO`: a breaker that reopens with /// no cool-down hammers a down peer with no backoff. /// - any peer address is not a syntactic `host:port` (empty/malformed host /// or a port outside `1..=65535`): a bad peer string surfaces only as /// "invalid URI" deep in [`PeerPool::new`] otherwise — catch the config /// typo here, naming the offending peer. The check is hostname-tolerant /// (it mirrors the server crate's `validate_host_port`): a DNS name is a /// valid peer address, an IP literal is too, but reachability is NOT /// probed (siblings boot in any order). /// /// `circuit_breaker_threshold == 0` is intentionally allowed: the breaker /// treats a zero threshold as "open on the first failure" (fail-fast), a /// legitimate—if aggressive—policy rather than a degenerate one. /// /// [`GrpcTransport::new`]: crate::transport::GrpcTransport::new pub fn validate(&self) -> Result<(), GrpcTransportError> { if self.channel_capacity == 0 { return Err(GrpcTransportError::Internal( "channel_capacity must be > 0 (a zero-capacity mpsc channel panics)".into(), )); } for (name, d) in [ ("connect_timeout", self.connect_timeout), ("request_timeout", self.request_timeout), ("keep_alive_interval", self.keep_alive_interval), ("keep_alive_timeout", self.keep_alive_timeout), ] { if d.is_zero() { return Err(GrpcTransportError::Internal(format!( "{name} must be > 0 (a zero deadline aborts every attempt instantly)" ))); } } if self.max_payload_bytes == 0 || self.max_payload_bytes > MAX_PAYLOAD_BYTES { return Err(GrpcTransportError::Internal(format!( "max_payload_bytes must be in 1..={MAX_PAYLOAD_BYTES}, got {}", self.max_payload_bytes ))); } if self.circuit_breaker_reset.is_zero() { return Err(GrpcTransportError::Internal( "circuit_breaker_reset must be > 0 (a zero cool-down hammers a down peer)".into(), )); } if self.catchup_retry_interval.is_zero() { return Err(GrpcTransportError::Internal( "catchup_retry_interval must be > 0 (a zero delay turns the \ catch-up retry timer into a hot loop against the source)" .into(), )); } if self.rotation_poll_interval.is_zero() { return Err(GrpcTransportError::Internal( "rotation_poll_interval must be > 0 (a zero delay turns the \ cert-rotation reloader into a hot loop re-reading the TLS files)" .into(), )); } if self.handshake_timeout.is_zero() { return Err(GrpcTransportError::Internal( "handshake_timeout must be > 0 (a zero deadline rejects every \ inbound TLS handshake instantly)" .into(), )); } if self.max_concurrent_handshakes == 0 { return Err(GrpcTransportError::Internal( "max_concurrent_handshakes must be > 0 (a zero bound load-sheds \ every inbound connection, refusing all peers)" .into(), )); } for (shard, addr) in &self.peers { validate_peer_addr(*shard, addr)?; } Ok(()) } } /// Syntactic check that a peer's advertised address is a `host:port` the tonic /// URI builder ([`PeerPool::new`](crate::client::PeerPool::new)) can turn into /// a valid endpoint, mirroring the server crate's `validate_host_port`: /// /// - an optional `http://` / `https://` prefix is tolerated (the scheme the /// pool prepends is derived from TLS, but a prefixed value must not be /// silently doubled into `http://http://…`), /// - the host must be non-empty and contain no whitespace or `/`, /// - the port must parse as a non-zero `u16` (port 0 is never a peer-reachable /// address). /// /// A DNS name passes (the m11p5 point): reachability is NOT probed here — /// siblings boot in any order, so an unresolvable-at-startup peer is normal. fn validate_peer_addr(shard: ShardId, addr: &str) -> Result<(), GrpcTransportError> { let bare = addr .strip_prefix("http://") .or_else(|| addr.strip_prefix("https://")) .unwrap_or(addr) .trim_end_matches('/'); let bad = |why: &str| { Err(GrpcTransportError::Internal(format!( "peer {shard:?} address '{addr}' is not a valid host:port ({why})" ))) }; let Some((host, port)) = bare.rsplit_once(':') else { return bad("missing ':port'"); }; if host.is_empty() || host.contains([' ', '/']) { return bad("empty or malformed host"); } match port.parse::() { Ok(0) => bad("port 0 is not reachable by peers"), Ok(_) => Ok(()), Err(_) => bad("port must be 1-65535"), } } impl Default for GrpcTransportConfig { fn default() -> Self { Self { local_shard: ShardId::SINGLE, // Default replication listener. Inside tidalDB's reserved dev band // (59520–59529); 59529 is the slot claimed for the gRPC transport. listen_addr: "127.0.0.1:59529".parse().expect("valid default addr"), peers: HashMap::new(), tls: None, insecure: true, channel_capacity: 1024, max_payload_bytes: MAX_PAYLOAD_BYTES, circuit_breaker_threshold: 5, circuit_breaker_reset: Duration::from_secs(30), connect_timeout: Duration::from_secs(5), request_timeout: Duration::from_secs(10), keep_alive_interval: Duration::from_secs(10), keep_alive_timeout: Duration::from_secs(5), catchup_retry_interval: Duration::from_secs(30), rotation_poll_interval: Duration::from_secs(30), handshake_timeout: Duration::from_secs(10), max_concurrent_handshakes: 256, } } } /// TLS certificate paths for mutual TLS. #[derive(Debug, Clone)] pub struct TlsConfig { /// Path to the CA certificate PEM file. pub ca_cert: PathBuf, /// Path to the server certificate PEM file. pub server_cert: PathBuf, /// Path to the server private key PEM file. pub server_key: PathBuf, /// Path to the client certificate PEM file (for mTLS). pub client_cert: Option, /// Path to the client private key PEM file (for mTLS). pub client_key: Option, } #[cfg(test)] mod tests { use super::*; #[test] fn default_config_is_valid() { // The shipped defaults must pass their own invariant check, otherwise // every default-built transport would error at construction. GrpcTransportConfig::default() .validate() .expect("default config must satisfy its own invariants"); } #[test] fn zero_channel_capacity_is_rejected() { // A zero-capacity mpsc channel panics inside GrpcTransport::new; the // typed error must catch it before that panic. let cfg = GrpcTransportConfig { channel_capacity: 0, ..GrpcTransportConfig::default() }; let err = cfg .validate() .expect_err("zero channel_capacity must be rejected"); assert!(matches!(err, GrpcTransportError::Internal(_))); } #[test] fn zero_timeouts_are_rejected() { let mutators: [fn(&mut GrpcTransportConfig); 4] = [ |c| c.connect_timeout = Duration::ZERO, |c| c.request_timeout = Duration::ZERO, |c| c.keep_alive_interval = Duration::ZERO, |c| c.keep_alive_timeout = Duration::ZERO, ]; for mutate in mutators { let mut cfg = GrpcTransportConfig::default(); mutate(&mut cfg); assert!( cfg.validate().is_err(), "a zero deadline must be rejected (it aborts every attempt instantly)" ); } } #[test] fn zero_handshake_bounds_are_rejected() { // A zero handshake timeout rejects every inbound TLS handshake instantly; // a zero concurrency bound load-sheds every connection — both refuse all // peers, so the typed config check must catch them. let mutators: [fn(&mut GrpcTransportConfig); 2] = [ |c| c.handshake_timeout = Duration::ZERO, |c| c.max_concurrent_handshakes = 0, ]; for mutate in mutators { let mut cfg = GrpcTransportConfig::default(); mutate(&mut cfg); assert!( cfg.validate().is_err(), "a zero handshake bound must be rejected" ); } } #[test] fn zero_circuit_breaker_reset_is_rejected() { let cfg = GrpcTransportConfig { circuit_breaker_reset: Duration::ZERO, ..GrpcTransportConfig::default() }; assert!(cfg.validate().is_err()); } #[test] fn zero_catchup_retry_interval_is_rejected() { // A zero retry delay turns the m11p4 catch-up timer into a hot loop // that hammers the stream source. let cfg = GrpcTransportConfig { catchup_retry_interval: Duration::ZERO, ..GrpcTransportConfig::default() }; assert!(cfg.validate().is_err()); } #[test] fn payload_ceiling_out_of_range_is_rejected() { // Zero rejects every segment; above the engine wire limit lets the codec // accept a payload the peer will refuse. for bytes in [0, MAX_PAYLOAD_BYTES + 1] { let cfg = GrpcTransportConfig { max_payload_bytes: bytes, ..GrpcTransportConfig::default() }; assert!( cfg.validate().is_err(), "max_payload_bytes {bytes} is out of the 1..={MAX_PAYLOAD_BYTES} range" ); } } #[test] fn payload_ceiling_at_engine_limit_is_allowed() { let cfg = GrpcTransportConfig { max_payload_bytes: MAX_PAYLOAD_BYTES, ..GrpcTransportConfig::default() }; assert!(cfg.validate().is_ok()); } #[test] fn zero_circuit_breaker_threshold_is_allowed() { // A zero threshold means "open on the first failure" (fail-fast), a // legitimate aggressive policy — not a degenerate config. let cfg = GrpcTransportConfig { circuit_breaker_threshold: 0, ..GrpcTransportConfig::default() }; assert!(cfg.validate().is_ok()); } #[test] fn valid_peer_addresses_are_accepted() { // IP literal, DNS name, fully-qualified k8s pod DNS, and a // scheme-prefixed value must all pass — the m11p5 DNS-peer point is // that hostnames are first-class peer addresses. for addr in [ "127.0.0.1:9601", "tidal-eu:9601", "tidaldb-1.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9601", "http://127.0.0.1:9601", "https://tidal-eu:9601", ] { let cfg = GrpcTransportConfig { peers: HashMap::from([(ShardId(1), addr.to_string())]), ..GrpcTransportConfig::default() }; cfg.validate() .unwrap_or_else(|e| panic!("peer addr '{addr}' must be accepted: {e:?}")); } } #[test] fn malformed_peer_addresses_are_rejected() { // A bad peer string would otherwise surface only as an opaque // "invalid URI" deep in PeerPool::new — the config check names the // offending peer instead. for (addr, why) in [ ("127.0.0.1", "missing port"), ("127.0.0.1:0", "port 0"), ("127.0.0.1:99999", "port out of range"), (":9601", "empty host"), ("10.0.0.1:port", "non-numeric port"), ] { let cfg = GrpcTransportConfig { peers: HashMap::from([(ShardId(1), addr.to_string())]), ..GrpcTransportConfig::default() }; let Err(err) = cfg.validate() else { panic!("peer addr '{addr}' ({why}) must be rejected"); }; assert!( matches!(err, GrpcTransportError::Internal(_)), "expected an Internal config error for '{addr}', got {err:?}" ); } } }