//! `ClusterState`: the single-process multi-region fabric behind the HTTP //! surface, plus the experimental-mode opt-in gate shared by both cluster modes. use std::{ collections::HashMap, sync::{ Arc, atomic::{AtomicBool, Ordering}, }, }; use tidaldb::{ replication::shard::RegionId, testing::{ClusterConfig, SimulatedCluster}, }; use super::{topology::TopologySpec, transport::build_grpc_transports}; use crate::{ error::{Result, ServerError}, offload::ClusterWritePool, }; /// HTTP-facing cluster state wrapping the simulated cluster fabric. /// /// Maps human-readable region names (e.g. "us-east") to [`RegionId`] values /// used by the distributed fabric. All cluster operations go through this /// layer. /// /// # Experimental /// /// Replication between regions runs over the **real `tidal-net` gRPC /// transport** (see the module-level docs), but every region still lives inside /// this single process: there is no process or host isolation, so a crash takes /// the whole "cluster" down. It provides faithful multi-region replication /// semantics, not production high-availability. Cluster mode is therefore gated /// behind an explicit operator opt-in — see [`ensure_experimental_enabled`]. pub struct ClusterState { /// `Some` for the lifetime of the server; consumed by [`shutdown`] / /// [`Drop`] so every node's `TidalDb` is dropped (checkpoint + WAL fsync + /// thread join) deterministically on shutdown. /// /// [`shutdown`]: ClusterState::shutdown /// /// Held behind an [`Arc`] so scatter-gather can hand each shard's blocking /// query to a detached `'static` worker thread (see /// [`crate::scatter_gather`]) without borrowing `&self` for the worker's /// lifetime. Cloning the `Arc` is cheap and the underlying /// [`SimulatedCluster`] is `Sync`, so concurrent reads are sound. cluster: Option>, name_to_id: HashMap, id_to_name: HashMap, shutting_down: AtomicBool, /// Fixed-size, runtime-free OS-thread pool for cluster write/heal work /// (gRPC segment ship). Created once at startup and shared by every /// `/signals` and `/cluster/heal` request, replacing the old per-request /// `std::thread` spawn (unbounded growth on the hottest cluster path). See /// [`crate::offload::ClusterWritePool`]. write_pool: ClusterWritePool, } /// Environment variable that opts in to the experimental cluster mode. pub const EXPERIMENTAL_CLUSTER_ENV: &str = "TIDAL_ALLOW_EXPERIMENTAL_CLUSTER"; /// Which cluster mode the experimental gate is being checked for, so the loud /// WARN can describe exactly what each mode does and does NOT provide. #[derive(Debug, Clone, Copy)] pub enum ClusterMode { /// Every region runs inside ONE process (m8p8). No host/process isolation. SingleProcess, /// Exactly ONE region runs in this process and peers with sibling processes /// over real gRPC (m8p10). Real process isolation, but still no quorum-ack /// writes and no automatic failure detection. MultiProcess, } /// Gate cluster mode behind an explicit operator opt-in. /// /// Both cluster modes are experimental and must not be started by accident: /// single-process has no host/process isolation, and multi-process — while it /// gives real process isolation — still lacks quorum-ack writes and automatic /// failure detection. Starting is permitted only when either: /// /// * the `--experimental-cluster` CLI flag is passed (`flag_set == true`), or /// * the [`EXPERIMENTAL_CLUSTER_ENV`] env var is set to a truthy value /// (`1`, `true`, `yes`). /// /// On success a loud `WARN` is emitted describing exactly what the selected /// `mode` is and is not. /// /// # Errors /// /// Returns [`ServerError::ExperimentalDisabled`] when neither opt-in is present. pub fn ensure_experimental_enabled(flag_set: bool, mode: ClusterMode) -> Result<()> { let env_set = std::env::var(EXPERIMENTAL_CLUSTER_ENV) .map(|v| matches!(v.trim().to_ascii_lowercase().as_str(), "1" | "true" | "yes")) .unwrap_or(false); if !(flag_set || env_set) { let reason = match mode { ClusterMode::SingleProcess => { "cluster mode replicates over real gRPC but runs all regions in ONE process \ (no host/process isolation) — it is not production HA." } ClusterMode::MultiProcess => { "multi-process cluster mode (--region) gives real process isolation but does \ NOT yet provide quorum-ack writes or automatic failure detection — it is not \ production HA." } }; return Err(ServerError::ExperimentalDisabled(format!( "{reason} To run it anyway, pass --experimental-cluster or set \ {EXPERIMENTAL_CLUSTER_ENV}=1." ))); } match mode { ClusterMode::SingleProcess => tracing::warn!( "EXPERIMENTAL cluster mode enabled. Regions replicate over the real tidal-net gRPC \ transport (loopback), but all regions run in a SINGLE process — there is no host or \ process isolation, so this provides NO production high-availability. True \ multi-process deployment is the --region mode. Do NOT use this for production traffic." ), ClusterMode::MultiProcess => tracing::warn!( "EXPERIMENTAL multi-process cluster mode enabled (--region). This process owns ONE \ region and peers with siblings over real gRPC, so process isolation IS real — but \ quorum-ack writes and automatic failure detection are NOT yet provided (a 204 means \ leader durability only, and failover is operator-driven). Do NOT use this for \ production traffic." ), } Ok(()) } impl ClusterState { /// Build from topology, schema, and ranking profiles. /// /// Wires each follower region to a real [`GrpcTransport`](tidal_net::GrpcTransport) /// (m8p8) and waits for every follower's gRPC server to bind before /// returning, so the first HTTP write always finds a connectable peer. /// /// # Blocking / threading /// /// MUST be called from a non-async thread. `GrpcTransport::new` starts a /// gRPC server by blocking on its own tokio runtime and asserts it is not /// inside another runtime; calling this from the axum/main reactor would /// panic. The caller (`main::run_cluster`) hops to a dedicated `std::thread`. /// /// # Errors /// /// Returns [`ServerError::SchemaConfig`] when the topology has no regions, /// contains duplicate region names, or names a leader that is not one of the /// declared regions; or [`ServerError::Cluster`] when a follower's gRPC /// transport cannot be built or its server does not become ready in time. pub fn new( topology: &TopologySpec, schema: tidaldb::schema::Schema, profiles: Vec, ) -> Result { if topology.regions.is_empty() { return Err(ServerError::SchemaConfig( "topology must declare at least one region".into(), )); } let mut name_to_id = HashMap::new(); let mut id_to_name = HashMap::new(); let mut region_ids = Vec::new(); for (i, region) in topology.regions.iter().enumerate() { let id = RegionId(u16::try_from(i).map_err(|_| { ServerError::SchemaConfig("topology declares more than 65535 regions".into()) })?); if name_to_id.insert(region.name.clone(), id).is_some() { return Err(ServerError::SchemaConfig(format!( "duplicate region name '{}' in topology", region.name ))); } id_to_name.insert(id, region.name.clone()); region_ids.push(id); } let leader_id = *name_to_id.get(&topology.leader).ok_or_else(|| { ServerError::SchemaConfig(format!("leader '{}' not found in regions", topology.leader)) })?; // m8p8: wire a real gRPC transport per follower region. let transports = build_grpc_transports(topology, &name_to_id, leader_id)?; tracing::info!( followers = transports.len(), "cluster replication wired over gRPC transport" ); let config = ClusterConfig { regions: region_ids, leader_region: leader_id, schema, profiles, transports: Some(transports), }; let cluster = SimulatedCluster::build(config); // Build the shared cluster-write worker pool once, sized from the // topology's `write_workers` (or the default derived from available // parallelism). Every `/signals` and `/cluster/heal` request shares it, // so write concurrency is bounded and threads are reused. let pool_config = topology.write_pool_config(); tracing::info!( workers = pool_config.workers, queue_depth = pool_config.queue_depth, "cluster write worker pool started" ); let write_pool = ClusterWritePool::new(pool_config); Ok(Self { cluster: Some(Arc::new(cluster)), name_to_id, id_to_name, shutting_down: AtomicBool::new(false), write_pool, }) } pub fn set_shutting_down(&self) { self.shutting_down.store(true, Ordering::Release); } pub fn is_shutting_down(&self) -> bool { self.shutting_down.load(Ordering::Acquire) } /// Cleanly shut down every node in the cluster. /// /// Drops the wrapped [`SimulatedCluster`], which drops each node's /// `TidalDb`. `TidalDb::Drop` runs the full shutdown path (checkpoint /// in-memory signal state → flush storage → write WAL checkpoint marker + /// fsync → join the WAL writer, sweeper, checkpoint, and text-syncer /// threads). The drop is idempotent, so a later [`Drop`] of `ClusterState` /// is a no-op. pub fn shutdown(&mut self) { self.set_shutting_down(); if self.cluster.take().is_some() { tracing::info!("cluster shutdown: closing all nodes (checkpoint + WAL fsync)"); } } /// Access the underlying cluster, or a 503 error if it has been shut down. /// /// The cluster is present for the entire request-serving lifetime; it is /// only taken during [`shutdown`](Self::shutdown), after axum has stopped /// accepting requests, so in practice no live handler observes `None`. That /// ordering invariant lives in `serve_cluster`, not in the type, so rather /// than `expect`-panic (which corrupts a database's reactor thread, per /// `CODING_GUIDELINES` §7) a post-shutdown access returns /// [`ServerError::Unavailable`] → 503, handled by the caller's existing /// error mapping. pub(super) fn cluster_ref(&self) -> Result<&SimulatedCluster> { self.cluster .as_ref() .map(AsRef::as_ref) .ok_or_else(|| ServerError::Unavailable("server shutting down".into())) } /// Clone the cluster `Arc` for scatter-gather fan-out, or a 503 if the /// server is shutting down. pub(super) fn cluster_arc(&self) -> Result> { self.cluster .as_ref() .map(Arc::clone) .ok_or_else(|| ServerError::Unavailable("server shutting down".into())) } pub(super) fn resolve_region(&self, name: &str) -> Result { self.name_to_id .get(name) .copied() .ok_or_else(|| ServerError::BadRequest(format!("unknown region '{name}'"))) } pub(super) fn region_name(&self, id: RegionId) -> &str { self.id_to_name.get(&id).map_or_else( || { tracing::warn!(region_id = id.0, "unknown region ID in name lookup"); "unknown" }, String::as_str, ) } /// Default region for reads when no `?region=` is specified: the leader. fn default_read_region(&self) -> Result { Ok(self.cluster_ref()?.leader_region()) } pub(super) fn read_region(&self, region_name: Option<&str>) -> Result { region_name.map_or_else( || self.default_read_region(), |name| self.resolve_region(name), ) } /// All region IDs (shard list for scatter-gather), or a 503 if shutting down. /// /// # Errors /// /// Returns [`ServerError::Unavailable`] if the cluster fabric has been taken /// during shutdown. pub fn shard_ids(&self) -> Result> { Ok(self.cluster_ref()?.regions()) } /// Access the underlying cluster, or a 503 if shutting down. /// /// # Errors /// /// Returns [`ServerError::Unavailable`] if the cluster fabric has been taken /// during shutdown. pub fn cluster(&self) -> Result<&SimulatedCluster> { self.cluster_ref() } /// Region name mapping for scatter-gather metadata. #[must_use] pub const fn id_to_name_map(&self) -> &HashMap { &self.id_to_name } /// Shared cluster write pool (gRPC segment ship), for the write/heal routes. pub(super) const fn write_pool(&self) -> &ClusterWritePool { &self.write_pool } } impl Drop for ClusterState { /// Backstop for the explicit [`shutdown`](ClusterState::shutdown): if the /// server exited without calling it, dropping the cluster here still runs /// each node's `TidalDb::Drop` (checkpoint + WAL fsync + thread join). fn drop(&mut self) { self.shutdown(); } }