8a0950260f
Splits monolithic cluster.rs into tidal-server/src/cluster/ modules. Adds redeliver-missed relay, bounded HLC drift, lag tracking, and reconcile idempotence. Five new tier-3 test suites (chaos, lifecycle, multiproc, region, routes, runbook) all green. Docs, CHANGELOG, and ROADMAP updated with G4/G5/G6 known gaps.
470 lines
17 KiB
Rust
470 lines
17 KiB
Rust
//! Tier-3 SINGLE-PROCESS-cluster smoke suite over the real HTTP binary.
|
||
//!
|
||
//! # What these tests verify
|
||
//!
|
||
//! Each test spawns real `tidal-server cluster` OS processes (the **single-process**
|
||
//! mode — no `--region`), connects over HTTP, and exercises the full binary boot
|
||
//! path: config/topology/schema parsing, the experimental-cluster opt-in gate,
|
||
//! router wiring, the gRPC transport bring-up, and the data/cluster-management
|
||
//! routes end-to-end against a real process (not an in-test `ClusterState`).
|
||
//!
|
||
//! # Scope: single-process mode, by design
|
||
//!
|
||
//! These tests drive the **single-process** cluster fabric: each spawned
|
||
//! `tidal-server cluster` process holds ALL regions, and replication between
|
||
//! regions traverses real gRPC on the loopback interface *within that one
|
||
//! process*. They therefore drive only `nodes[0]` and prove the single-process
|
||
//! binary boots, serves, and replicates intra-process. They do not (and are not
|
||
//! meant to) cross a process boundary.
|
||
//!
|
||
//! # The multi-process surface is covered elsewhere (m8p10, RESOLVED)
|
||
//!
|
||
//! True multi-process HA — one `tidal-server cluster --region` process per region,
|
||
//! peering over real gRPC to sibling addresses, with cross-process HTTP routing and
|
||
//! real network-partition injection — shipped in m8p10 and is verified by the
|
||
//! sibling tier-3 suites (all `--features cluster-e2e`):
|
||
//!
|
||
//! * [`cluster_multiproc`](cluster_multiproc) — cross-process replication, leader
|
||
//! crash + failover, region-routing flip, write forwarding, status aggregation;
|
||
//! * [`cluster_chaos`](cluster_chaos) — REAL network partitions via an in-harness
|
||
//! TCP relay proxy (degraded query, heal/reconcile, follower↔follower partition);
|
||
//! * [`cluster_lifecycle`](cluster_lifecycle) — clock skew + rolling upgrade;
|
||
//! * [`cluster_runbook`](cluster_runbook) — every `docs/runbooks/cluster.md` §5–§11
|
||
//! operation against a real 3-process cluster.
|
||
//!
|
||
//! This file is the single-process smoke complement to those — the former
|
||
//! "KNOWN GAP" (ROADMAP.md G1 / m8p10) is RESOLVED.
|
||
//!
|
||
//! Gated behind `--features cluster-e2e` (disabled by default; these tests
|
||
//! are slow and require a built binary).
|
||
//!
|
||
//! ```bash
|
||
//! cargo test -p tidal-server --features cluster-e2e --test cluster_e2e -- --nocapture
|
||
//! ```
|
||
#![cfg(feature = "cluster-e2e")]
|
||
// Tier-3 E2E harness: unwrap on known-good fixtures is idiomatic test noise.
|
||
#![allow(clippy::unwrap_used)]
|
||
|
||
use std::{
|
||
io::Write,
|
||
net::{SocketAddr, TcpListener},
|
||
path::{Path, PathBuf},
|
||
process::{Child, Command},
|
||
time::{Duration, Instant},
|
||
};
|
||
|
||
/// A running tidal-server cluster node.
|
||
struct NodeHandle {
|
||
#[allow(dead_code)]
|
||
name: String,
|
||
http_addr: SocketAddr,
|
||
process: Child,
|
||
}
|
||
|
||
impl Drop for NodeHandle {
|
||
fn drop(&mut self) {
|
||
// Best-effort graceful shutdown.
|
||
#[cfg(unix)]
|
||
{
|
||
// Send SIGTERM via the `kill` binary so this crate stays
|
||
// `unsafe_code = forbid` — no `libc::kill` FFI needed.
|
||
let _ = Command::new("kill")
|
||
.args(["-TERM", &self.process.id().to_string()])
|
||
.status();
|
||
// Wait up to 2s for graceful exit.
|
||
let deadline = Instant::now() + Duration::from_secs(2);
|
||
loop {
|
||
match self.process.try_wait() {
|
||
Ok(Some(_)) => break,
|
||
_ if Instant::now() > deadline => {
|
||
let _ = self.process.kill();
|
||
break;
|
||
}
|
||
_ => std::thread::sleep(Duration::from_millis(50)),
|
||
}
|
||
}
|
||
}
|
||
#[cfg(not(unix))]
|
||
{
|
||
let _ = self.process.kill();
|
||
}
|
||
let _ = self.process.wait();
|
||
}
|
||
}
|
||
|
||
/// Multi-process cluster test harness.
|
||
struct ClusterHarness {
|
||
nodes: Vec<NodeHandle>,
|
||
_config_dir: tempfile::TempDir,
|
||
}
|
||
|
||
impl ClusterHarness {
|
||
/// Spawn `n` `tidal-server cluster` processes, each a FULL in-process
|
||
/// cluster.
|
||
///
|
||
/// Allocates dynamic HTTP ports, generates topology + schema YAML, spawns
|
||
/// the processes, and waits for all to become healthy. Note (see module
|
||
/// docs): every spawned process internally holds ALL `n` regions, so the
|
||
/// tests only meaningfully exercise `nodes[0]`. The extra processes prove
|
||
/// the binary boots independently on its own port, not cross-process
|
||
/// replication.
|
||
fn start(n: usize) -> Self {
|
||
assert!(n >= 2, "need at least 2 nodes for a cluster");
|
||
|
||
let region_names: Vec<String> = (0..n).map(|i| format!("region-{i}")).collect();
|
||
let http_addrs: Vec<SocketAddr> = (0..n).map(|_| free_addr()).collect();
|
||
|
||
// Write config files.
|
||
let config_dir = tempfile::tempdir().expect("create temp dir");
|
||
|
||
let topology = Self::write_topology(config_dir.path(), ®ion_names);
|
||
let schema = Self::write_schema(config_dir.path());
|
||
|
||
// Find the tidal-server binary.
|
||
let bin = tidal_server_bin();
|
||
|
||
// Spawn processes.
|
||
let mut nodes = Vec::with_capacity(n);
|
||
for i in 0..n {
|
||
let child = Command::new(&bin)
|
||
.arg("cluster")
|
||
.arg("--listen")
|
||
.arg(http_addrs[i].to_string())
|
||
.arg("--schema")
|
||
.arg(&schema)
|
||
.arg("--topology")
|
||
.arg(&topology)
|
||
// Cluster mode is experimental (regions replicate over real
|
||
// gRPC but share one process — no host/process isolation) and
|
||
// refuses to start without this opt-in.
|
||
.env("TIDAL_ALLOW_EXPERIMENTAL_CLUSTER", "1")
|
||
.env("TIDAL_SERVER_LOG", "warn")
|
||
.stdout(std::process::Stdio::piped())
|
||
.stderr(std::process::Stdio::piped())
|
||
.spawn()
|
||
.unwrap_or_else(|e| panic!("failed to spawn tidal-server: {e}"));
|
||
|
||
nodes.push(NodeHandle {
|
||
name: region_names[i].clone(),
|
||
http_addr: http_addrs[i],
|
||
process: child,
|
||
});
|
||
}
|
||
|
||
let harness = Self {
|
||
nodes,
|
||
_config_dir: config_dir,
|
||
};
|
||
|
||
// Wait for all nodes to become healthy. Each node boots a real gRPC
|
||
// transport (its own tokio runtime + server) per follower region on top
|
||
// of opening its tidalDB instances, and the suite may start several
|
||
// such processes concurrently, so allow a generous tier-3 budget.
|
||
harness.wait_all_healthy(Duration::from_secs(30));
|
||
harness
|
||
}
|
||
|
||
fn write_topology(dir: &Path, regions: &[String]) -> PathBuf {
|
||
let path = dir.join("topology.yaml");
|
||
let mut f = std::fs::File::create(&path).expect("create topology.yaml");
|
||
writeln!(f, "regions:").unwrap();
|
||
for name in regions {
|
||
writeln!(f, " - name: {name}").unwrap();
|
||
}
|
||
writeln!(f, "leader: {}", regions[0]).unwrap();
|
||
path
|
||
}
|
||
|
||
fn write_schema(dir: &Path) -> PathBuf {
|
||
let path = dir.join("schema.yaml");
|
||
std::fs::write(
|
||
&path,
|
||
r"signals:
|
||
- name: view
|
||
entity: item
|
||
decay:
|
||
exponential:
|
||
half_life_seconds: 604800
|
||
windows: [one_hour]
|
||
velocity: false
|
||
- name: like
|
||
entity: item
|
||
decay:
|
||
exponential:
|
||
half_life_seconds: 86400
|
||
windows: [one_hour]
|
||
velocity: false
|
||
text_fields:
|
||
- name: title
|
||
kind: text
|
||
embedding_slots:
|
||
- name: content_vector
|
||
entity: item
|
||
dimensions: 4
|
||
",
|
||
)
|
||
.expect("write schema.yaml");
|
||
path
|
||
}
|
||
|
||
fn wait_all_healthy(&self, timeout: Duration) {
|
||
let deadline = Instant::now() + timeout;
|
||
for node in &self.nodes {
|
||
wait_for_health(&node.http_addr, deadline);
|
||
}
|
||
}
|
||
|
||
fn http_get(&self, node_idx: usize, path: &str) -> reqwest::blocking::Response {
|
||
let url = format!("http://{}{path}", self.nodes[node_idx].http_addr);
|
||
reqwest::blocking::get(&url).unwrap_or_else(|e| panic!("GET {url} failed: {e}"))
|
||
}
|
||
|
||
fn http_post_json(
|
||
&self,
|
||
node_idx: usize,
|
||
path: &str,
|
||
body: &serde_json::Value,
|
||
) -> reqwest::blocking::Response {
|
||
let url = format!("http://{}{path}", self.nodes[node_idx].http_addr);
|
||
let client = reqwest::blocking::Client::new();
|
||
client
|
||
.post(&url)
|
||
.json(body)
|
||
.send()
|
||
.unwrap_or_else(|e| panic!("POST {url} failed: {e}"))
|
||
}
|
||
|
||
/// Poll `/cluster/status` until all follower regions show `lag_events == 0`, or timeout.
|
||
///
|
||
/// The leader is excluded because it writes directly (not via replication)
|
||
/// and its `applied_events` counter tracks replication from other shards.
|
||
fn wait_converged(&self, timeout: Duration) {
|
||
let deadline = Instant::now() + timeout;
|
||
loop {
|
||
assert!(
|
||
Instant::now() <= deadline,
|
||
"convergence timeout after {timeout:?}; cluster did not converge"
|
||
);
|
||
let resp = self.http_get(0, "/cluster/status");
|
||
if resp.status().is_success() {
|
||
let body: serde_json::Value = resp.json().unwrap();
|
||
let leader = body["leader"].as_str().unwrap_or("");
|
||
if let Some(regions) = body["regions"].as_array() {
|
||
let followers_converged = regions
|
||
.iter()
|
||
.filter(|r| r["name"].as_str().unwrap_or("") != leader)
|
||
.all(|r| r["lag_events"].as_u64().unwrap_or(u64::MAX) == 0);
|
||
if followers_converged {
|
||
return;
|
||
}
|
||
}
|
||
}
|
||
std::thread::sleep(Duration::from_millis(200));
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Allocate a free port by binding to :0 and recording the assigned port.
|
||
fn free_addr() -> SocketAddr {
|
||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||
listener.local_addr().unwrap()
|
||
}
|
||
|
||
/// Find the tidal-server binary. Build it first if needed.
|
||
fn tidal_server_bin() -> PathBuf {
|
||
// Resolve the workspace root that owns the cargo `target/` dir. tidal-server
|
||
// is a workspace member at `<workspace>/tidal-server`, so the workspace root
|
||
// is the parent of `CARGO_MANIFEST_DIR`. Probe the parent first, then the
|
||
// grandparent, so the harness keeps working regardless of how the crate is
|
||
// nested in a workspace.
|
||
let manifest_dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR"));
|
||
let candidates = [
|
||
manifest_dir.parent(),
|
||
manifest_dir.parent().and_then(std::path::Path::parent),
|
||
];
|
||
let workspace_root = candidates
|
||
.into_iter()
|
||
.flatten()
|
||
.find(|root| root.join("target").is_dir() || root.join("Cargo.toml").is_file())
|
||
.or_else(|| manifest_dir.parent())
|
||
.expect("workspace root");
|
||
|
||
// Build the binary to make sure it's up to date.
|
||
let status = Command::new("cargo")
|
||
.arg("build")
|
||
.arg("-p")
|
||
.arg("tidal-server")
|
||
.current_dir(workspace_root)
|
||
.stdout(std::process::Stdio::null())
|
||
.stderr(std::process::Stdio::null())
|
||
.status()
|
||
.expect("cargo build");
|
||
assert!(status.success(), "cargo build -p tidal-server failed");
|
||
|
||
let bin = workspace_root.join("target/debug/tidal-server");
|
||
assert!(
|
||
bin.exists(),
|
||
"tidal-server binary not found at {}",
|
||
bin.display()
|
||
);
|
||
bin
|
||
}
|
||
|
||
/// Poll a node's `/health/startup` until it returns 200.
|
||
fn wait_for_health(addr: &SocketAddr, deadline: Instant) {
|
||
let url = format!("http://{addr}/health/startup");
|
||
loop {
|
||
assert!(
|
||
Instant::now() <= deadline,
|
||
"health check timeout for {addr}"
|
||
);
|
||
match reqwest::blocking::get(&url) {
|
||
Ok(resp) if resp.status().is_success() => return,
|
||
_ => std::thread::sleep(Duration::from_millis(100)),
|
||
}
|
||
}
|
||
}
|
||
|
||
// ── E2E Tests ─────────────────────────────────────────────────────────────
|
||
|
||
/// Single-process-cluster smoke test: boot a real `tidal-server cluster`
|
||
/// process, write data over HTTP, and verify it converges within its OWN
|
||
/// in-process replication fabric.
|
||
///
|
||
/// This is NOT a cross-process convergence proof: node 0 holds all regions
|
||
/// internally, so `wait_converged` polls node 0's own `/cluster/status`. The
|
||
/// cross-process proof lives in `cluster_multiproc`/`cluster_chaos`/
|
||
/// `cluster_lifecycle`/`cluster_runbook` (see the module docs).
|
||
#[test]
|
||
fn single_process_cluster_smoke() {
|
||
let harness = ClusterHarness::start(3);
|
||
|
||
// 1. Health check.
|
||
let resp = harness.http_get(0, "/health");
|
||
assert!(resp.status().is_success());
|
||
let body: serde_json::Value = resp.json().unwrap();
|
||
assert_eq!(body["mode"], "cluster");
|
||
|
||
// 2. Cluster status — all regions healthy, leader is region-0.
|
||
let resp = harness.http_get(0, "/cluster/status");
|
||
assert!(resp.status().is_success());
|
||
let status: serde_json::Value = resp.json().unwrap();
|
||
assert_eq!(status["leader"], "region-0");
|
||
assert_eq!(status["regions"].as_array().unwrap().len(), 3);
|
||
|
||
// 3. Write items.
|
||
for i in 1..=5u64 {
|
||
let resp = harness.http_post_json(
|
||
0,
|
||
"/items",
|
||
&serde_json::json!({
|
||
"entity_id": i,
|
||
"metadata": { "title": format!("item {i}") }
|
||
}),
|
||
);
|
||
assert!(
|
||
resp.status().is_success(),
|
||
"POST /items failed: {}",
|
||
resp.status()
|
||
);
|
||
}
|
||
|
||
// 4. Write signals.
|
||
for i in 1..=5u64 {
|
||
let resp = harness.http_post_json(
|
||
0,
|
||
"/signals",
|
||
&serde_json::json!({
|
||
"entity_id": i,
|
||
"signal": "view",
|
||
"weight": 1.0
|
||
}),
|
||
);
|
||
assert!(
|
||
resp.status().is_success(),
|
||
"POST /signals failed: {}",
|
||
resp.status()
|
||
);
|
||
}
|
||
|
||
// 5. Wait for convergence.
|
||
harness.wait_converged(Duration::from_secs(5));
|
||
|
||
// 6. Verify follower regions show zero lag.
|
||
let resp = harness.http_get(0, "/cluster/status");
|
||
let status: serde_json::Value = resp.json().unwrap();
|
||
let leader = status["leader"].as_str().unwrap();
|
||
for region in status["regions"].as_array().unwrap() {
|
||
if region["name"].as_str().unwrap() == leader {
|
||
continue; // Leader writes directly, not via replication.
|
||
}
|
||
assert_eq!(
|
||
region["lag_events"].as_u64().unwrap(),
|
||
0,
|
||
"follower {} has lag",
|
||
region["name"]
|
||
);
|
||
}
|
||
|
||
// 7. Retrieve from a follower region.
|
||
let resp = harness.http_get(0, "/feed?profile=trending&limit=5®ion=region-1");
|
||
assert!(resp.status().is_success());
|
||
let feed: serde_json::Value = resp.json().unwrap();
|
||
assert!(
|
||
!feed["items"].as_array().unwrap().is_empty(),
|
||
"follower region should have items"
|
||
);
|
||
}
|
||
|
||
/// Promote a follower within a single process's in-process cluster and verify
|
||
/// subsequent writes route to the new leader.
|
||
///
|
||
/// Like [`single_process_cluster_smoke`], this exercises one process's own
|
||
/// fabric (node 0 holds all regions); cross-process promotion is proven by
|
||
/// `cluster_multiproc::mp_uat_step2_leader_crash_failover_under_10s`.
|
||
#[test]
|
||
fn single_process_cluster_promote_leader() {
|
||
let harness = ClusterHarness::start(3);
|
||
|
||
// Write some initial data.
|
||
harness.http_post_json(
|
||
0,
|
||
"/items",
|
||
&serde_json::json!({ "entity_id": 1, "metadata": { "title": "before promote" } }),
|
||
);
|
||
harness.http_post_json(
|
||
0,
|
||
"/signals",
|
||
&serde_json::json!({ "entity_id": 1, "signal": "view", "weight": 1.0 }),
|
||
);
|
||
harness.wait_converged(Duration::from_secs(5));
|
||
|
||
// Promote region-1 to leader.
|
||
let resp = harness.http_post_json(
|
||
0,
|
||
"/cluster/promote",
|
||
&serde_json::json!({ "region": "region-1" }),
|
||
);
|
||
assert!(resp.status().is_success());
|
||
|
||
// Verify status shows new leader.
|
||
let resp = harness.http_get(0, "/cluster/status");
|
||
let status: serde_json::Value = resp.json().unwrap();
|
||
assert_eq!(status["leader"], "region-1");
|
||
|
||
// Write more signals — should route to new leader.
|
||
let resp = harness.http_post_json(
|
||
0,
|
||
"/signals",
|
||
&serde_json::json!({ "entity_id": 1, "signal": "like", "weight": 2.0 }),
|
||
);
|
||
assert!(
|
||
resp.status().is_success(),
|
||
"POST /signals after promote failed: {}",
|
||
resp.status()
|
||
);
|
||
// Note: this single-process smoke does not assert post-promotion convergence.
|
||
// Per-source-shard lag tracking across a promotion is verified over real
|
||
// processes by `cluster_lifecycle`/`cluster_chaos` (the m8p10 lag-gauge fix).
|
||
}
|