tidaldb/tidal-server/tests/cluster_e2e.rs

//! Tier-3 multi-process cluster E2E tests.
//!
//! Spawns real `tidal-server cluster` OS processes, connects via HTTP,
//! and verifies distributed behavior across process boundaries.
//!
//! Gated behind `--features cluster-e2e` (disabled by default; these tests
//! are slow and require a built binary).
//!
//! ```bash
//! cargo test -p tidal-server --features cluster-e2e --test cluster_e2e -- --nocapture
//! ```
#![cfg(feature = "cluster-e2e")]

use std::io::Write;
use std::net::{SocketAddr, TcpListener};
use std::path::PathBuf;
use std::process::{Child, Command};
use std::time::{Duration, Instant};

/// A running tidal-server cluster node.
struct NodeHandle {
    #[allow(dead_code)]
    name: String,
    http_addr: SocketAddr,
    process: Child,
}

impl Drop for NodeHandle {
    fn drop(&mut self) {
        // Best-effort graceful shutdown.
        #[cfg(unix)]
        {
            unsafe {
                libc::kill(self.process.id() as i32, libc::SIGTERM);
            }
            // Wait up to 2s for graceful exit.
            let deadline = Instant::now() + Duration::from_secs(2);
            loop {
                match self.process.try_wait() {
                    Ok(Some(_)) => break,
                    _ if Instant::now() > deadline => {
                        let _ = self.process.kill();
                        break;
                    }
                    _ => std::thread::sleep(Duration::from_millis(50)),
                }
            }
        }
        #[cfg(not(unix))]
        {
            let _ = self.process.kill();
        }
        let _ = self.process.wait();
    }
}

/// Multi-process cluster test harness.
struct ClusterHarness {
    nodes: Vec<NodeHandle>,
    _config_dir: tempfile::TempDir,
}

impl ClusterHarness {
    /// Spawn a multi-region cluster with `n` nodes.
    ///
    /// Allocates dynamic ports, generates topology + schema YAML, spawns
    /// `tidal-server cluster` processes, and waits for all to become healthy.
    fn start(n: usize) -> Self {
        assert!(n >= 2, "need at least 2 nodes for a cluster");

        let region_names: Vec<String> = (0..n).map(|i| format!("region-{i}")).collect();
        let http_addrs: Vec<SocketAddr> = (0..n).map(|_| free_addr()).collect();

        // Write config files.
        let config_dir = tempfile::tempdir().expect("create temp dir");

        let topology = Self::write_topology(&config_dir.path().to_path_buf(), &region_names);
        let schema = Self::write_schema(&config_dir.path().to_path_buf());

        // Find the tidal-server binary.
        let bin = tidal_server_bin();

        // Spawn processes.
        let mut nodes = Vec::with_capacity(n);
        for i in 0..n {
            let child = Command::new(&bin)
                .arg("cluster")
                .arg("--listen")
                .arg(http_addrs[i].to_string())
                .arg("--schema")
                .arg(&schema)
                .arg("--topology")
                .arg(&topology)
                .env("TIDAL_SERVER_LOG", "warn")
                .stdout(std::process::Stdio::piped())
                .stderr(std::process::Stdio::piped())
                .spawn()
                .unwrap_or_else(|e| panic!("failed to spawn tidal-server: {e}"));

            nodes.push(NodeHandle {
                name: region_names[i].clone(),
                http_addr: http_addrs[i],
                process: child,
            });
        }

        let harness = Self {
            nodes,
            _config_dir: config_dir,
        };

        // Wait for all nodes to become healthy.
        harness.wait_all_healthy(Duration::from_secs(15));
        harness
    }

    fn write_topology(dir: &PathBuf, regions: &[String]) -> PathBuf {
        let path = dir.join("topology.yaml");
        let mut f = std::fs::File::create(&path).expect("create topology.yaml");
        writeln!(f, "regions:").unwrap();
        for name in regions {
            writeln!(f, "  - name: {name}").unwrap();
        }
        writeln!(f, "leader: {}", regions[0]).unwrap();
        path
    }

    fn write_schema(dir: &PathBuf) -> PathBuf {
        let path = dir.join("schema.yaml");
        std::fs::write(
            &path,
            r#"signals:
  - name: view
    entity: item
    decay:
      exponential:
        half_life_seconds: 604800
    windows: [one_hour]
    velocity: false
  - name: like
    entity: item
    decay:
      exponential:
        half_life_seconds: 86400
    windows: [one_hour]
    velocity: false
text_fields:
  - name: title
    kind: text
embedding_slots:
  - name: content_vector
    entity: item
    dimensions: 4
"#,
        )
        .expect("write schema.yaml");
        path
    }

    fn wait_all_healthy(&self, timeout: Duration) {
        let deadline = Instant::now() + timeout;
        for node in &self.nodes {
            wait_for_health(&node.http_addr, deadline);
        }
    }

    fn http_get(&self, node_idx: usize, path: &str) -> reqwest::blocking::Response {
        let url = format!("http://{}{path}", self.nodes[node_idx].http_addr);
        reqwest::blocking::get(&url).unwrap_or_else(|e| panic!("GET {url} failed: {e}"))
    }

    fn http_post_json(
        &self,
        node_idx: usize,
        path: &str,
        body: &serde_json::Value,
    ) -> reqwest::blocking::Response {
        let url = format!("http://{}{path}", self.nodes[node_idx].http_addr);
        let client = reqwest::blocking::Client::new();
        client
            .post(&url)
            .json(body)
            .send()
            .unwrap_or_else(|e| panic!("POST {url} failed: {e}"))
    }

    /// Poll `/cluster/status` until all follower regions show lag_events == 0, or timeout.
    ///
    /// The leader is excluded because it writes directly (not via replication)
    /// and its `applied_events` counter tracks replication from other shards.
    fn wait_converged(&self, timeout: Duration) {
        let deadline = Instant::now() + timeout;
        loop {
            if Instant::now() > deadline {
                panic!("convergence timeout after {timeout:?}; cluster did not converge");
            }
            let resp = self.http_get(0, "/cluster/status");
            if resp.status().is_success() {
                let body: serde_json::Value = resp.json().unwrap();
                let leader = body["leader"].as_str().unwrap_or("");
                if let Some(regions) = body["regions"].as_array() {
                    let followers_converged = regions
                        .iter()
                        .filter(|r| r["name"].as_str().unwrap_or("") != leader)
                        .all(|r| r["lag_events"].as_u64().unwrap_or(u64::MAX) == 0);
                    if followers_converged {
                        return;
                    }
                }
            }
            std::thread::sleep(Duration::from_millis(200));
        }
    }
}

/// Allocate a free port by binding to :0 and recording the assigned port.
fn free_addr() -> SocketAddr {
    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
    listener.local_addr().unwrap()
}

/// Find the tidal-server binary. Build it first if needed.
fn tidal_server_bin() -> PathBuf {
    // Try the standard cargo target directory.
    let manifest_dir = env!("CARGO_MANIFEST_DIR");
    let workspace_root = std::path::Path::new(manifest_dir)
        .parent()
        .expect("workspace root");

    // Build the binary to make sure it's up to date.
    let status = Command::new("cargo")
        .arg("build")
        .arg("-p")
        .arg("tidal-server")
        .current_dir(workspace_root)
        .stdout(std::process::Stdio::null())
        .stderr(std::process::Stdio::null())
        .status()
        .expect("cargo build");
    assert!(status.success(), "cargo build -p tidal-server failed");

    let bin = workspace_root.join("target/debug/tidal-server");
    assert!(bin.exists(), "tidal-server binary not found at {bin:?}");
    bin
}

/// Poll a node's `/health/startup` until it returns 200.
fn wait_for_health(addr: &SocketAddr, deadline: Instant) {
    let url = format!("http://{addr}/health/startup");
    loop {
        if Instant::now() > deadline {
            panic!("health check timeout for {addr}");
        }
        match reqwest::blocking::get(&url) {
            Ok(resp) if resp.status().is_success() => return,
            _ => std::thread::sleep(Duration::from_millis(100)),
        }
    }
}

// ── E2E Tests ─────────────────────────────────────────────────────────────

/// Smoke test: start 3-node cluster, write data, verify convergence.
#[test]
fn cluster_smoke_test() {
    let harness = ClusterHarness::start(3);

    // 1. Health check.
    let resp = harness.http_get(0, "/health");
    assert!(resp.status().is_success());
    let body: serde_json::Value = resp.json().unwrap();
    assert_eq!(body["mode"], "cluster");

    // 2. Cluster status — all regions healthy, leader is region-0.
    let resp = harness.http_get(0, "/cluster/status");
    assert!(resp.status().is_success());
    let status: serde_json::Value = resp.json().unwrap();
    assert_eq!(status["leader"], "region-0");
    assert_eq!(status["regions"].as_array().unwrap().len(), 3);

    // 3. Write items.
    for i in 1..=5u64 {
        let resp = harness.http_post_json(
            0,
            "/items",
            &serde_json::json!({
                "entity_id": i,
                "metadata": { "title": format!("item {i}") }
            }),
        );
        assert!(
            resp.status().is_success(),
            "POST /items failed: {}",
            resp.status()
        );
    }

    // 4. Write signals.
    for i in 1..=5u64 {
        let resp = harness.http_post_json(
            0,
            "/signals",
            &serde_json::json!({
                "entity_id": i,
                "signal": "view",
                "weight": 1.0
            }),
        );
        assert!(
            resp.status().is_success(),
            "POST /signals failed: {}",
            resp.status()
        );
    }

    // 5. Wait for convergence.
    harness.wait_converged(Duration::from_secs(5));

    // 6. Verify follower regions show zero lag.
    let resp = harness.http_get(0, "/cluster/status");
    let status: serde_json::Value = resp.json().unwrap();
    let leader = status["leader"].as_str().unwrap();
    for region in status["regions"].as_array().unwrap() {
        if region["name"].as_str().unwrap() == leader {
            continue; // Leader writes directly, not via replication.
        }
        assert_eq!(
            region["lag_events"].as_u64().unwrap(),
            0,
            "follower {} has lag",
            region["name"]
        );
    }

    // 7. Retrieve from a follower region.
    let resp = harness.http_get(0, "/feed?profile=trending&limit=5&region=region-1");
    assert!(resp.status().is_success());
    let feed: serde_json::Value = resp.json().unwrap();
    assert!(
        !feed["items"].as_array().unwrap().is_empty(),
        "follower region should have items"
    );
}

/// Promote a follower and verify writes route to new leader.
#[test]
fn cluster_promote_leader() {
    let harness = ClusterHarness::start(3);

    // Write some initial data.
    harness.http_post_json(
        0,
        "/items",
        &serde_json::json!({ "entity_id": 1, "metadata": { "title": "before promote" } }),
    );
    harness.http_post_json(
        0,
        "/signals",
        &serde_json::json!({ "entity_id": 1, "signal": "view", "weight": 1.0 }),
    );
    harness.wait_converged(Duration::from_secs(5));

    // Promote region-1 to leader.
    let resp = harness.http_post_json(
        0,
        "/cluster/promote",
        &serde_json::json!({ "region": "region-1" }),
    );
    assert!(resp.status().is_success());

    // Verify status shows new leader.
    let resp = harness.http_get(0, "/cluster/status");
    let status: serde_json::Value = resp.json().unwrap();
    assert_eq!(status["leader"], "region-1");

    // Write more signals — should route to new leader.
    let resp = harness.http_post_json(
        0,
        "/signals",
        &serde_json::json!({ "entity_id": 1, "signal": "like", "weight": 2.0 }),
    );
    assert!(
        resp.status().is_success(),
        "POST /signals after promote failed: {}",
        resp.status()
    );
    // Note: convergence after leader promotion is tracked per-source-shard in
    // ReplicationState, but /cluster/status reports lag against ShardId(0) only.
    // Full multi-shard lag tracking is a known gap (see ROADMAP.md G1).
}