//! m8p8 in-process gRPC replication tests. //! //! Unlike `cluster_e2e.rs` (tier-3, feature-gated, spawns OS processes), these //! run in the default test build. They prove that `ClusterState` replicates //! between regions over the **real `tidal-net` gRPC transport** — closing gap //! G1 (`ClusterState` previously wrapped crossbeam channels) — and that the //! async HTTP write path offloads its blocking gRPC ship correctly. #![allow(clippy::unwrap_used, clippy::missing_panics_doc)] use std::{ sync::Arc, time::{Duration, Instant}, }; use tidal_server::cluster::{ClusterState, RegionSpec, TopologySpec, build_cluster_router}; use tidaldb::schema::{DecaySpec, EntityId, EntityKind, Schema, SchemaBuilder, Window}; /// Three-region topology (leader us-east) with auto-allocated loopback gRPC /// ports — the production default path. fn three_region_topology() -> TopologySpec { TopologySpec { regions: vec![ RegionSpec { name: "us-east".into(), grpc_addr: None, }, RegionSpec { name: "eu-west".into(), grpc_addr: None, }, RegionSpec { name: "ap-south".into(), grpc_addr: None, }, ], leader: "us-east".into(), write_workers: None, } } fn view_schema() -> Schema { let mut builder = SchemaBuilder::new(); let _ = builder .signal( "view", EntityKind::Item, DecaySpec::Exponential { half_life: Duration::from_secs(7 * 24 * 3600), }, ) .windows(&[Window::OneHour]) .velocity(false) .add(); builder.build().unwrap() } /// Direct proof: writes to the leader replicate to every follower over gRPC, /// and the followers' decay scores converge to the leader's. #[test] fn grpc_replication_converges_across_followers() { // Plain (non-async) test thread: ClusterState::new starts the follower gRPC // servers via GrpcTransport::new, which blocks on its own runtime. let state = ClusterState::new(&three_region_topology(), view_schema(), Vec::new()) .expect("cluster builds with gRPC transports"); let cluster = state.cluster().expect("cluster is live before shutdown"); let leader = cluster.leader_region(); let followers: Vec<_> = cluster .regions() .into_iter() .filter(|&r| r != leader) .collect(); assert_eq!(followers.len(), 2, "expected two follower regions"); // Write 10 signals to the leader; each ships over gRPC to both followers. for i in 1..=10u64 { cluster .write_signal("view", EntityId::new(i), 1.0) .expect("leader signal write + gRPC ship"); } // Poll until both followers have applied all 10 leader segments. let target = cluster.leader_seqno(); assert_eq!(target, 10, "leader should have sequenced 10 writes"); let deadline = Instant::now() + Duration::from_secs(5); loop { let converged = followers .iter() .all(|&r| cluster.applied_count(r) >= target); if converged { break; } assert!( Instant::now() <= deadline, "followers did not converge over gRPC within 5s: applied = {:?}, target = {target}", followers .iter() .map(|&r| cluster.applied_count(r)) .collect::>() ); std::thread::sleep(Duration::from_millis(20)); } // Decay scores must match the leader's to 6 decimals on every follower. for i in 1..=10u64 { let eid = EntityId::new(i); let l = cluster.read_decay_score(leader, eid, "view").unwrap_or(0.0); for &f in &followers { let got = cluster.read_decay_score(f, eid, "view").unwrap_or(0.0); assert!( (l - got).abs() < 1e-6, "entity {i}: leader={l} follower={got}" ); } } } /// Full HTTP path: POST /signals (which offloads its blocking gRPC ship off the /// reactor) replicates to followers, and a `?region=` read serves the follower. #[test] fn http_signal_replicates_and_region_read_serves_follower() { let rt = tokio::runtime::Builder::new_multi_thread() .worker_threads(2) .enable_all() .build() .unwrap(); // Build the cluster off the reactor (we are on a plain test thread). let state = ClusterState::new(&three_region_topology(), view_schema(), Vec::new()) .expect("cluster builds with gRPC transports"); let router = build_cluster_router(Arc::new(state), None); // Bind + serve on the runtime; keep the test thread free for blocking HTTP. let listener = rt .block_on(tokio::net::TcpListener::bind("127.0.0.1:0")) .unwrap(); let addr = listener.local_addr().unwrap(); rt.spawn(async move { let _ = axum::serve(listener, router).await; }); let base = format!("http://{addr}"); let client = reqwest::blocking::Client::new(); // Broadcast items to all regions, then write signals (replicated to followers). for i in 1..=5u64 { let resp = client .post(format!("{base}/items")) .json(&serde_json::json!({ "entity_id": i, "metadata": { "title": format!("item {i}") } })) .send() .unwrap(); assert!(resp.status().is_success(), "POST /items: {}", resp.status()); let resp = client .post(format!("{base}/signals")) .json(&serde_json::json!({ "entity_id": i, "signal": "view", "weight": 1.0 })) .send() .unwrap(); assert!( resp.status().is_success(), "POST /signals: {}", resp.status() ); } // Poll /cluster/status until every follower reports zero lag. let deadline = Instant::now() + Duration::from_secs(5); loop { let status: serde_json::Value = client .get(format!("{base}/cluster/status")) .send() .unwrap() .json() .unwrap(); let leader = status["leader"].as_str().unwrap_or(""); let converged = status["regions"] .as_array() .unwrap() .iter() .filter(|r| r["name"].as_str().unwrap_or("") != leader) .all(|r| r["lag_events"].as_u64().unwrap_or(u64::MAX) == 0); if converged { break; } assert!( Instant::now() <= deadline, "followers did not converge over gRPC within 5s: {status}" ); std::thread::sleep(Duration::from_millis(50)); } // A region-pinned read on a follower must return the replicated items. let feed: serde_json::Value = client .get(format!( "{base}/feed?profile=trending&limit=5®ion=eu-west" )) .send() .unwrap() .json() .unwrap(); let items = feed["items"].as_array().unwrap(); assert!( !items.is_empty(), "follower eu-west should serve replicated items, got: {feed}" ); rt.shutdown_timeout(Duration::from_secs(2)); }