tidaldb/tidal-server/tests/cluster_grpc.rs

//! m8p8 in-process gRPC replication tests.
//!
//! Unlike `cluster_e2e.rs` (tier-3, feature-gated, spawns OS processes), these
//! run in the default test build. They prove that `ClusterState` replicates
//! between regions over the **real `tidal-net` gRPC transport** — closing gap
//! G1 (`ClusterState` previously wrapped crossbeam channels) — and that the
//! async HTTP write path offloads its blocking gRPC ship correctly.
#![allow(clippy::unwrap_used, clippy::missing_panics_doc)]

use std::{
    sync::Arc,
    time::{Duration, Instant},
};

use tidal_server::cluster::{ClusterState, RegionSpec, TopologySpec, build_cluster_router};
use tidaldb::schema::{DecaySpec, EntityId, EntityKind, Schema, SchemaBuilder, Window};

/// Three-region topology (leader us-east) with auto-allocated loopback gRPC
/// ports — the production default path.
fn three_region_topology() -> TopologySpec {
    TopologySpec {
        regions: vec![
            RegionSpec {
                name: "us-east".into(),
                grpc_addr: None,
            },
            RegionSpec {
                name: "eu-west".into(),
                grpc_addr: None,
            },
            RegionSpec {
                name: "ap-south".into(),
                grpc_addr: None,
            },
        ],
        leader: "us-east".into(),
        write_workers: None,
    }
}

fn view_schema() -> Schema {
    let mut builder = SchemaBuilder::new();
    let _ = builder
        .signal(
            "view",
            EntityKind::Item,
            DecaySpec::Exponential {
                half_life: Duration::from_secs(7 * 24 * 3600),
            },
        )
        .windows(&[Window::OneHour])
        .velocity(false)
        .add();
    builder.build().unwrap()
}

/// Direct proof: writes to the leader replicate to every follower over gRPC,
/// and the followers' decay scores converge to the leader's.
#[test]
fn grpc_replication_converges_across_followers() {
    // Plain (non-async) test thread: ClusterState::new starts the follower gRPC
    // servers via GrpcTransport::new, which blocks on its own runtime.
    let state = ClusterState::new(&three_region_topology(), view_schema(), Vec::new())
        .expect("cluster builds with gRPC transports");
    let cluster = state.cluster().expect("cluster is live before shutdown");

    let leader = cluster.leader_region();
    let followers: Vec<_> = cluster
        .regions()
        .into_iter()
        .filter(|&r| r != leader)
        .collect();
    assert_eq!(followers.len(), 2, "expected two follower regions");

    // Write 10 signals to the leader; each ships over gRPC to both followers.
    for i in 1..=10u64 {
        cluster
            .write_signal("view", EntityId::new(i), 1.0)
            .expect("leader signal write + gRPC ship");
    }

    // Poll until both followers have applied all 10 leader segments.
    let target = cluster.leader_seqno();
    assert_eq!(target, 10, "leader should have sequenced 10 writes");
    let deadline = Instant::now() + Duration::from_secs(5);
    loop {
        let converged = followers
            .iter()
            .all(|&r| cluster.applied_count(r) >= target);
        if converged {
            break;
        }
        assert!(
            Instant::now() <= deadline,
            "followers did not converge over gRPC within 5s: applied = {:?}, target = {target}",
            followers
                .iter()
                .map(|&r| cluster.applied_count(r))
                .collect::<Vec<_>>()
        );
        std::thread::sleep(Duration::from_millis(20));
    }

    // Decay scores must match the leader's to 6 decimals on every follower.
    for i in 1..=10u64 {
        let eid = EntityId::new(i);
        let l = cluster.read_decay_score(leader, eid, "view").unwrap_or(0.0);
        for &f in &followers {
            let got = cluster.read_decay_score(f, eid, "view").unwrap_or(0.0);
            assert!(
                (l - got).abs() < 1e-6,
                "entity {i}: leader={l} follower={got}"
            );
        }
    }
}

/// Full HTTP path: POST /signals (which offloads its blocking gRPC ship off the
/// reactor) replicates to followers, and a `?region=` read serves the follower.
#[test]
fn http_signal_replicates_and_region_read_serves_follower() {
    let rt = tokio::runtime::Builder::new_multi_thread()
        .worker_threads(2)
        .enable_all()
        .build()
        .unwrap();

    // Build the cluster off the reactor (we are on a plain test thread).
    let state = ClusterState::new(&three_region_topology(), view_schema(), Vec::new())
        .expect("cluster builds with gRPC transports");
    let router = build_cluster_router(Arc::new(state), None);

    // Bind + serve on the runtime; keep the test thread free for blocking HTTP.
    let listener = rt
        .block_on(tokio::net::TcpListener::bind("127.0.0.1:0"))
        .unwrap();
    let addr = listener.local_addr().unwrap();
    rt.spawn(async move {
        let _ = axum::serve(listener, router).await;
    });

    let base = format!("http://{addr}");
    let client = reqwest::blocking::Client::new();

    // Broadcast items to all regions, then write signals (replicated to followers).
    for i in 1..=5u64 {
        let resp = client
            .post(format!("{base}/items"))
            .json(&serde_json::json!({
                "entity_id": i,
                "metadata": { "title": format!("item {i}") }
            }))
            .send()
            .unwrap();
        assert!(resp.status().is_success(), "POST /items: {}", resp.status());

        let resp = client
            .post(format!("{base}/signals"))
            .json(&serde_json::json!({ "entity_id": i, "signal": "view", "weight": 1.0 }))
            .send()
            .unwrap();
        assert!(
            resp.status().is_success(),
            "POST /signals: {}",
            resp.status()
        );
    }

    // Poll /cluster/status until every follower reports zero lag.
    let deadline = Instant::now() + Duration::from_secs(5);
    loop {
        let status: serde_json::Value = client
            .get(format!("{base}/cluster/status"))
            .send()
            .unwrap()
            .json()
            .unwrap();
        let leader = status["leader"].as_str().unwrap_or("");
        let converged = status["regions"]
            .as_array()
            .unwrap()
            .iter()
            .filter(|r| r["name"].as_str().unwrap_or("") != leader)
            .all(|r| r["lag_events"].as_u64().unwrap_or(u64::MAX) == 0);
        if converged {
            break;
        }
        assert!(
            Instant::now() <= deadline,
            "followers did not converge over gRPC within 5s: {status}"
        );
        std::thread::sleep(Duration::from_millis(50));
    }

    // A region-pinned read on a follower must return the replicated items.
    let feed: serde_json::Value = client
        .get(format!(
            "{base}/feed?profile=trending&limit=5&region=eu-west"
        ))
        .send()
        .unwrap()
        .json()
        .unwrap();
    let items = feed["items"].as_array().unwrap();
    assert!(
        !items.is_empty(),
        "follower eu-west should serve replicated items, got: {feed}"
    );

    rt.shutdown_timeout(Duration::from_secs(2));
}