8a0950260f
Splits monolithic cluster.rs into tidal-server/src/cluster/ modules. Adds redeliver-missed relay, bounded HLC drift, lag tracking, and reconcile idempotence. Five new tier-3 test suites (chaos, lifecycle, multiproc, region, routes, runbook) all green. Docs, CHANGELOG, and ROADMAP updated with G4/G5/G6 known gaps.
343 lines
14 KiB
Rust
343 lines
14 KiB
Rust
//! `ClusterState`: the single-process multi-region fabric behind the HTTP
|
|
//! surface, plus the experimental-mode opt-in gate shared by both cluster modes.
|
|
|
|
use std::{
|
|
collections::HashMap,
|
|
sync::{
|
|
Arc,
|
|
atomic::{AtomicBool, Ordering},
|
|
},
|
|
};
|
|
|
|
use tidaldb::{
|
|
replication::shard::RegionId,
|
|
testing::{ClusterConfig, SimulatedCluster},
|
|
};
|
|
|
|
use super::{topology::TopologySpec, transport::build_grpc_transports};
|
|
use crate::{
|
|
error::{Result, ServerError},
|
|
offload::ClusterWritePool,
|
|
};
|
|
|
|
/// HTTP-facing cluster state wrapping the simulated cluster fabric.
|
|
///
|
|
/// Maps human-readable region names (e.g. "us-east") to [`RegionId`] values
|
|
/// used by the distributed fabric. All cluster operations go through this
|
|
/// layer.
|
|
///
|
|
/// # Experimental
|
|
///
|
|
/// Replication between regions runs over the **real `tidal-net` gRPC
|
|
/// transport** (see the module-level docs), but every region still lives inside
|
|
/// this single process: there is no process or host isolation, so a crash takes
|
|
/// the whole "cluster" down. It provides faithful multi-region replication
|
|
/// semantics, not production high-availability. Cluster mode is therefore gated
|
|
/// behind an explicit operator opt-in — see [`ensure_experimental_enabled`].
|
|
pub struct ClusterState {
|
|
/// `Some` for the lifetime of the server; consumed by [`shutdown`] /
|
|
/// [`Drop`] so every node's `TidalDb` is dropped (checkpoint + WAL fsync +
|
|
/// thread join) deterministically on shutdown.
|
|
///
|
|
/// [`shutdown`]: ClusterState::shutdown
|
|
///
|
|
/// Held behind an [`Arc`] so scatter-gather can hand each shard's blocking
|
|
/// query to a detached `'static` worker thread (see
|
|
/// [`crate::scatter_gather`]) without borrowing `&self` for the worker's
|
|
/// lifetime. Cloning the `Arc` is cheap and the underlying
|
|
/// [`SimulatedCluster`] is `Sync`, so concurrent reads are sound.
|
|
cluster: Option<Arc<SimulatedCluster>>,
|
|
name_to_id: HashMap<String, RegionId>,
|
|
id_to_name: HashMap<RegionId, String>,
|
|
shutting_down: AtomicBool,
|
|
/// Fixed-size, runtime-free OS-thread pool for cluster write/heal work
|
|
/// (gRPC segment ship). Created once at startup and shared by every
|
|
/// `/signals` and `/cluster/heal` request, replacing the old per-request
|
|
/// `std::thread` spawn (unbounded growth on the hottest cluster path). See
|
|
/// [`crate::offload::ClusterWritePool`].
|
|
write_pool: ClusterWritePool,
|
|
}
|
|
|
|
/// Environment variable that opts in to the experimental cluster mode.
|
|
pub const EXPERIMENTAL_CLUSTER_ENV: &str = "TIDAL_ALLOW_EXPERIMENTAL_CLUSTER";
|
|
|
|
/// Which cluster mode the experimental gate is being checked for, so the loud
|
|
/// WARN can describe exactly what each mode does and does NOT provide.
|
|
#[derive(Debug, Clone, Copy)]
|
|
pub enum ClusterMode {
|
|
/// Every region runs inside ONE process (m8p8). No host/process isolation.
|
|
SingleProcess,
|
|
/// Exactly ONE region runs in this process and peers with sibling processes
|
|
/// over real gRPC (m8p10). Real process isolation, but still no quorum-ack
|
|
/// writes and no automatic failure detection.
|
|
MultiProcess,
|
|
}
|
|
|
|
/// Gate cluster mode behind an explicit operator opt-in.
|
|
///
|
|
/// Both cluster modes are experimental and must not be started by accident:
|
|
/// single-process has no host/process isolation, and multi-process — while it
|
|
/// gives real process isolation — still lacks quorum-ack writes and automatic
|
|
/// failure detection. Starting is permitted only when either:
|
|
///
|
|
/// * the `--experimental-cluster` CLI flag is passed (`flag_set == true`), or
|
|
/// * the [`EXPERIMENTAL_CLUSTER_ENV`] env var is set to a truthy value
|
|
/// (`1`, `true`, `yes`).
|
|
///
|
|
/// On success a loud `WARN` is emitted describing exactly what the selected
|
|
/// `mode` is and is not.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns [`ServerError::ExperimentalDisabled`] when neither opt-in is present.
|
|
pub fn ensure_experimental_enabled(flag_set: bool, mode: ClusterMode) -> Result<()> {
|
|
let env_set = std::env::var(EXPERIMENTAL_CLUSTER_ENV)
|
|
.map(|v| matches!(v.trim().to_ascii_lowercase().as_str(), "1" | "true" | "yes"))
|
|
.unwrap_or(false);
|
|
|
|
if !(flag_set || env_set) {
|
|
let reason = match mode {
|
|
ClusterMode::SingleProcess => {
|
|
"cluster mode replicates over real gRPC but runs all regions in ONE process \
|
|
(no host/process isolation) — it is not production HA."
|
|
}
|
|
ClusterMode::MultiProcess => {
|
|
"multi-process cluster mode (--region) gives real process isolation but does \
|
|
NOT yet provide quorum-ack writes or automatic failure detection — it is not \
|
|
production HA."
|
|
}
|
|
};
|
|
return Err(ServerError::ExperimentalDisabled(format!(
|
|
"{reason} To run it anyway, pass --experimental-cluster or set \
|
|
{EXPERIMENTAL_CLUSTER_ENV}=1."
|
|
)));
|
|
}
|
|
|
|
match mode {
|
|
ClusterMode::SingleProcess => tracing::warn!(
|
|
"EXPERIMENTAL cluster mode enabled. Regions replicate over the real tidal-net gRPC \
|
|
transport (loopback), but all regions run in a SINGLE process — there is no host or \
|
|
process isolation, so this provides NO production high-availability. True \
|
|
multi-process deployment is the --region mode. Do NOT use this for production traffic."
|
|
),
|
|
ClusterMode::MultiProcess => tracing::warn!(
|
|
"EXPERIMENTAL multi-process cluster mode enabled (--region). This process owns ONE \
|
|
region and peers with siblings over real gRPC, so process isolation IS real — but \
|
|
quorum-ack writes and automatic failure detection are NOT yet provided (a 204 means \
|
|
leader durability only, and failover is operator-driven). Do NOT use this for \
|
|
production traffic."
|
|
),
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
impl ClusterState {
|
|
/// Build from topology, schema, and ranking profiles.
|
|
///
|
|
/// Wires each follower region to a real [`GrpcTransport`](tidal_net::GrpcTransport)
|
|
/// (m8p8) and waits for every follower's gRPC server to bind before
|
|
/// returning, so the first HTTP write always finds a connectable peer.
|
|
///
|
|
/// # Blocking / threading
|
|
///
|
|
/// MUST be called from a non-async thread. `GrpcTransport::new` starts a
|
|
/// gRPC server by blocking on its own tokio runtime and asserts it is not
|
|
/// inside another runtime; calling this from the axum/main reactor would
|
|
/// panic. The caller (`main::run_cluster`) hops to a dedicated `std::thread`.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns [`ServerError::SchemaConfig`] when the topology has no regions,
|
|
/// contains duplicate region names, or names a leader that is not one of the
|
|
/// declared regions; or [`ServerError::Cluster`] when a follower's gRPC
|
|
/// transport cannot be built or its server does not become ready in time.
|
|
pub fn new(
|
|
topology: &TopologySpec,
|
|
schema: tidaldb::schema::Schema,
|
|
profiles: Vec<tidaldb::ranking::profile::RankingProfile>,
|
|
) -> Result<Self> {
|
|
if topology.regions.is_empty() {
|
|
return Err(ServerError::SchemaConfig(
|
|
"topology must declare at least one region".into(),
|
|
));
|
|
}
|
|
|
|
let mut name_to_id = HashMap::new();
|
|
let mut id_to_name = HashMap::new();
|
|
let mut region_ids = Vec::new();
|
|
|
|
for (i, region) in topology.regions.iter().enumerate() {
|
|
let id = RegionId(u16::try_from(i).map_err(|_| {
|
|
ServerError::SchemaConfig("topology declares more than 65535 regions".into())
|
|
})?);
|
|
if name_to_id.insert(region.name.clone(), id).is_some() {
|
|
return Err(ServerError::SchemaConfig(format!(
|
|
"duplicate region name '{}' in topology",
|
|
region.name
|
|
)));
|
|
}
|
|
id_to_name.insert(id, region.name.clone());
|
|
region_ids.push(id);
|
|
}
|
|
|
|
let leader_id = *name_to_id.get(&topology.leader).ok_or_else(|| {
|
|
ServerError::SchemaConfig(format!("leader '{}' not found in regions", topology.leader))
|
|
})?;
|
|
|
|
// m8p8: wire a real gRPC transport per follower region.
|
|
let transports = build_grpc_transports(topology, &name_to_id, leader_id)?;
|
|
tracing::info!(
|
|
followers = transports.len(),
|
|
"cluster replication wired over gRPC transport"
|
|
);
|
|
|
|
let config = ClusterConfig {
|
|
regions: region_ids,
|
|
leader_region: leader_id,
|
|
schema,
|
|
profiles,
|
|
transports: Some(transports),
|
|
};
|
|
|
|
let cluster = SimulatedCluster::build(config);
|
|
|
|
// Build the shared cluster-write worker pool once, sized from the
|
|
// topology's `write_workers` (or the default derived from available
|
|
// parallelism). Every `/signals` and `/cluster/heal` request shares it,
|
|
// so write concurrency is bounded and threads are reused.
|
|
let pool_config = topology.write_pool_config();
|
|
tracing::info!(
|
|
workers = pool_config.workers,
|
|
queue_depth = pool_config.queue_depth,
|
|
"cluster write worker pool started"
|
|
);
|
|
let write_pool = ClusterWritePool::new(pool_config);
|
|
|
|
Ok(Self {
|
|
cluster: Some(Arc::new(cluster)),
|
|
name_to_id,
|
|
id_to_name,
|
|
shutting_down: AtomicBool::new(false),
|
|
write_pool,
|
|
})
|
|
}
|
|
|
|
pub fn set_shutting_down(&self) {
|
|
self.shutting_down.store(true, Ordering::Release);
|
|
}
|
|
|
|
pub fn is_shutting_down(&self) -> bool {
|
|
self.shutting_down.load(Ordering::Acquire)
|
|
}
|
|
|
|
/// Cleanly shut down every node in the cluster.
|
|
///
|
|
/// Drops the wrapped [`SimulatedCluster`], which drops each node's
|
|
/// `TidalDb`. `TidalDb::Drop` runs the full shutdown path (checkpoint
|
|
/// in-memory signal state → flush storage → write WAL checkpoint marker +
|
|
/// fsync → join the WAL writer, sweeper, checkpoint, and text-syncer
|
|
/// threads). The drop is idempotent, so a later [`Drop`] of `ClusterState`
|
|
/// is a no-op.
|
|
pub fn shutdown(&mut self) {
|
|
self.set_shutting_down();
|
|
if self.cluster.take().is_some() {
|
|
tracing::info!("cluster shutdown: closing all nodes (checkpoint + WAL fsync)");
|
|
}
|
|
}
|
|
|
|
/// Access the underlying cluster, or a 503 error if it has been shut down.
|
|
///
|
|
/// The cluster is present for the entire request-serving lifetime; it is
|
|
/// only taken during [`shutdown`](Self::shutdown), after axum has stopped
|
|
/// accepting requests, so in practice no live handler observes `None`. That
|
|
/// ordering invariant lives in `serve_cluster`, not in the type, so rather
|
|
/// than `expect`-panic (which corrupts a database's reactor thread, per
|
|
/// `CODING_GUIDELINES` §7) a post-shutdown access returns
|
|
/// [`ServerError::Unavailable`] → 503, handled by the caller's existing
|
|
/// error mapping.
|
|
pub(super) fn cluster_ref(&self) -> Result<&SimulatedCluster> {
|
|
self.cluster
|
|
.as_ref()
|
|
.map(AsRef::as_ref)
|
|
.ok_or_else(|| ServerError::Unavailable("server shutting down".into()))
|
|
}
|
|
|
|
/// Clone the cluster `Arc` for scatter-gather fan-out, or a 503 if the
|
|
/// server is shutting down.
|
|
pub(super) fn cluster_arc(&self) -> Result<Arc<SimulatedCluster>> {
|
|
self.cluster
|
|
.as_ref()
|
|
.map(Arc::clone)
|
|
.ok_or_else(|| ServerError::Unavailable("server shutting down".into()))
|
|
}
|
|
|
|
pub(super) fn resolve_region(&self, name: &str) -> Result<RegionId> {
|
|
self.name_to_id
|
|
.get(name)
|
|
.copied()
|
|
.ok_or_else(|| ServerError::BadRequest(format!("unknown region '{name}'")))
|
|
}
|
|
|
|
pub(super) fn region_name(&self, id: RegionId) -> &str {
|
|
self.id_to_name.get(&id).map_or_else(
|
|
|| {
|
|
tracing::warn!(region_id = id.0, "unknown region ID in name lookup");
|
|
"unknown"
|
|
},
|
|
String::as_str,
|
|
)
|
|
}
|
|
|
|
/// Default region for reads when no `?region=` is specified: the leader.
|
|
fn default_read_region(&self) -> Result<RegionId> {
|
|
Ok(self.cluster_ref()?.leader_region())
|
|
}
|
|
|
|
pub(super) fn read_region(&self, region_name: Option<&str>) -> Result<RegionId> {
|
|
region_name.map_or_else(
|
|
|| self.default_read_region(),
|
|
|name| self.resolve_region(name),
|
|
)
|
|
}
|
|
|
|
/// All region IDs (shard list for scatter-gather), or a 503 if shutting down.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns [`ServerError::Unavailable`] if the cluster fabric has been taken
|
|
/// during shutdown.
|
|
pub fn shard_ids(&self) -> Result<Vec<RegionId>> {
|
|
Ok(self.cluster_ref()?.regions())
|
|
}
|
|
|
|
/// Access the underlying cluster, or a 503 if shutting down.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns [`ServerError::Unavailable`] if the cluster fabric has been taken
|
|
/// during shutdown.
|
|
pub fn cluster(&self) -> Result<&SimulatedCluster> {
|
|
self.cluster_ref()
|
|
}
|
|
|
|
/// Region name mapping for scatter-gather metadata.
|
|
#[must_use]
|
|
pub const fn id_to_name_map(&self) -> &HashMap<RegionId, String> {
|
|
&self.id_to_name
|
|
}
|
|
|
|
/// Shared cluster write pool (gRPC segment ship), for the write/heal routes.
|
|
pub(super) const fn write_pool(&self) -> &ClusterWritePool {
|
|
&self.write_pool
|
|
}
|
|
}
|
|
|
|
impl Drop for ClusterState {
|
|
/// Backstop for the explicit [`shutdown`](ClusterState::shutdown): if the
|
|
/// server exited without calling it, dropping the cluster here still runs
|
|
/// each node's `TidalDb::Drop` (checkpoint + WAL fsync + thread join).
|
|
fn drop(&mut self) {
|
|
self.shutdown();
|
|
}
|
|
}
|