tidaldb/tidal-server/tests/cluster_quorum.rs

//! Tier-3 quorum-ack suite (m11p3, REAL multi-process cluster).
//!
//! Two pillars:
//!
//! 1. **Quorum semantics under partition** — `ack=quorum` writes succeed
//!    while a majority is reachable, fail fast (retryable 503 naming the
//!    laggards) when it is not, never disturb `ack=leader` traffic, and
//!    recover after heal.
//! 2. **The ledger checker (the m11p3 exit gate)** — SIGKILL the leader
//!    under concurrent `ack=quorum` load, across many distinct kill points,
//!    and prove ZERO acknowledged-write loss: every write the client saw a
//!    2xx + `x-tidal-seq` for is present on the promoted survivor.
//!
//!    The proof is two-layered per kill point:
//!    - **Frontier**: a quorum ack for seqno S means some follower's
//!      CONTIGUOUS applied frontier reached S (durably — m11p3 acks are
//!      post-apply). The operator rule "promote the max-applied survivor"
//!      therefore guarantees the promoted node holds EVERY acked seqno:
//!      `max(acked seq) <= max(survivor applied_events)` is asserted before
//!      the promote.
//!    - **Content**: every acked ITEM is found via `/search` on the new
//!      leader (the frontier can't lie about data it doesn't have, but this
//!      catches a frontier that lies about data it has).
//!
//!    Kill-point count: `TIDAL_QUORUM_KILLPOINTS` (default 8 for CI; the
//!    exit-gate run is 100 — see docs/planning/milestone-11/phase-3.md for
//!    the recorded run).
//!
//! Run: `cargo test -p tidal-server --features cluster-e2e --test cluster_quorum -- --nocapture`

#![cfg(feature = "cluster-e2e")]
#![allow(
    clippy::unwrap_used,
    clippy::expect_used,
    clippy::panic,
    clippy::cast_possible_truncation,
    clippy::cast_precision_loss,
    clippy::too_many_lines
)]

mod support;

use std::sync::{
    Arc,
    atomic::{AtomicBool, Ordering},
};
use std::time::{Duration, Instant};

use support::{
    multiproc::{BREAKER_RESET, ClusterOptions, MultiProcCluster, convergence_budget},
    partition::proxied_rewrite,
};

const LEADER: usize = 0;

/// CI-default kill points; the exit-gate run sets `TIDAL_QUORUM_KILLPOINTS=100`.
fn killpoints() -> usize {
    std::env::var("TIDAL_QUORUM_KILLPOINTS")
        .ok()
        .and_then(|v| v.parse().ok())
        .filter(|&n| n > 0)
        .unwrap_or(8)
}

/// A unique all-alpha search token for an entity id (digits 0-9 → letters
/// a-j), so `/search?query=<token>` is an exact item-presence probe under
/// the default tokenizer.
fn item_token(entity_id: u64) -> String {
    let mut token = String::from("kpq");
    for d in entity_id.to_string().bytes() {
        token.push(char::from(b'a' + (d - b'0')));
    }
    token
}

/// POST with the `x-tidal-ack` header through a dedicated client. Returns
/// `Some(seq)` only for a 2xx carrying `x-tidal-seq` — the ledger's
/// definition of "acknowledged".
fn post_acked(
    client: &reqwest::blocking::Client,
    base: &str,
    path: &str,
    ack: &str,
    body: &serde_json::Value,
) -> Option<u64> {
    let resp = client
        .post(format!("{base}{path}"))
        .header("x-tidal-ack", ack)
        .json(body)
        .send()
        .ok()?;
    if !resp.status().is_success() {
        return None;
    }
    resp.headers()
        .get("x-tidal-seq")?
        .to_str()
        .ok()?
        .parse()
        .ok()
}

/// m11p3 quorum semantics over a REAL 3-process cluster with real TCP
/// partitions:
///
/// - healthy: `ack=quorum` 204/201 + seq header; the leader's `commit_index`
///   tracks the writes;
/// - ONE follower severed: quorum (2 of 3) still commits via the other;
/// - BOTH followers severed: quorum 503s fast naming the laggards while
///   `ack=leader` writes keep succeeding (the knob's cost is the caller's
///   choice, never the deployment's);
/// - healed: quorum commits again.
#[test]
fn mp_quorum_writes_gate_and_recover_under_partition() {
    let (rewrite, proxies) = proxied_rewrite(&["eu-west", "ap-south"]);
    let cluster = MultiProcCluster::start_with(ClusterOptions::new(3).with_rewrite(rewrite));
    let client = reqwest::blocking::Client::builder()
        .timeout(Duration::from_secs(8))
        .build()
        .unwrap();
    let leader_base = cluster.node(LEADER);

    // ── Healthy majority: quorum writes commit ──────────────────────────────
    let seq = post_acked(
        &client,
        &leader_base,
        "/items",
        "quorum",
        &serde_json::json!({ "entity_id": 1, "metadata": { "title": "quorum one" } }),
    )
    .expect("healthy-cluster quorum item write must ack");
    let view_seq = post_acked(
        &client,
        &leader_base,
        "/signals",
        "quorum",
        &serde_json::json!({ "entity_id": 1, "signal": "view", "weight": 1.0 }),
    )
    .expect("healthy-cluster quorum signal write must ack");
    assert!(view_seq > seq, "one log: signal follows the item");
    let status = cluster.local_status(LEADER).unwrap();
    assert!(
        status["commit_index"].as_u64().unwrap() >= view_seq,
        "a quorum ack is at or below the commit index: {status}"
    );
    println!("[quorum] healthy: item seq={seq}, view seq={view_seq} committed");

    // ── One follower down: 2-of-3 majority still commits ────────────────────
    proxies.region("ap-south").sever_grpc();
    let seq = post_acked(
        &client,
        &leader_base,
        "/signals",
        "quorum",
        &serde_json::json!({ "entity_id": 1, "signal": "view", "weight": 2.0 }),
    )
    .expect("quorum must survive a single-follower outage (majority intact)");
    println!("[quorum] ap-south severed: quorum still commits (seq={seq})");

    // ── Both followers down: quorum 503s naming the laggards ────────────────
    proxies.region("eu-west").sever_grpc();
    let resp = client
        .post(format!("{leader_base}/signals"))
        .header("x-tidal-ack", "quorum")
        .json(&serde_json::json!({ "entity_id": 1, "signal": "view", "weight": 3.0 }))
        .send()
        .unwrap();
    assert_eq!(
        resp.status().as_u16(),
        503,
        "no majority reachable: the quorum write must fail fast"
    );
    let body: serde_json::Value = resp.json().unwrap();
    assert_eq!(body["retryable"].as_bool(), Some(true));
    let laggards: Vec<&str> = body["laggards"]
        .as_array()
        .unwrap()
        .iter()
        .map(|v| v.as_str().unwrap())
        .collect();
    assert!(
        laggards.contains(&"eu-west") || laggards.contains(&"ap-south"),
        "the 503 names the lagging followers: {body}"
    );
    // The leader-ack contract is untouched by the followers' outage.
    let resp = client
        .post(format!("{leader_base}/signals"))
        .header("x-tidal-ack", "leader")
        .json(&serde_json::json!({ "entity_id": 1, "signal": "view", "weight": 4.0 }))
        .send()
        .unwrap();
    assert_eq!(
        resp.status().as_u16(),
        204,
        "ack=leader writes keep succeeding through a follower outage"
    );
    println!("[quorum] both severed: quorum 503 named {laggards:?}; leader-ack still 204");

    // ── Heal: quorum recovers (drive through the breaker window) ────────────
    proxies.region("eu-west").heal_all();
    proxies.region("ap-south").heal_all();
    for region in ["eu-west", "ap-south"] {
        let resp = client
            .post(format!("{leader_base}/cluster/heal"))
            .json(&serde_json::json!({ "region": region }))
            .send()
            .unwrap();
        assert_eq!(resp.status().as_u16(), 200);
    }
    let deadline = Instant::now() + BREAKER_RESET + convergence_budget();
    let mut healed_seq = None;
    while healed_seq.is_none() {
        assert!(
            Instant::now() <= deadline,
            "healed quorum writes must commit within the breaker+convergence budget"
        );
        healed_seq = post_acked(
            &client,
            &leader_base,
            "/signals",
            "quorum",
            &serde_json::json!({ "entity_id": 1, "signal": "view", "weight": 5.0 }),
        );
        if healed_seq.is_none() {
            // Re-issue the heal: the breaker can swallow the first post-heal
            // ships (the documented runbook loop).
            for region in ["eu-west", "ap-south"] {
                let _ = client
                    .post(format!("{leader_base}/cluster/heal"))
                    .json(&serde_json::json!({ "region": region }))
                    .send();
            }
            std::thread::sleep(Duration::from_millis(250));
        }
    }
    println!(
        "[quorum] healed: quorum commits again (seq={})",
        healed_seq.unwrap()
    );
}

/// THE m11p3 EXIT GATE: SIGKILL the leader under `ack=quorum` load at many
/// distinct kill points; the ledger checker proves zero acknowledged loss on
/// the promoted (max-applied) survivor every time. See the module docs for
/// the two-layer proof.
#[test]
fn mp_quorum_ledger_zero_acked_loss_across_killpoints() {
    let rounds = killpoints();
    println!("[ledger] running {rounds} leader-kill points (TIDAL_QUORUM_KILLPOINTS to widen)");

    for round in 0..rounds {
        let mut cluster = MultiProcCluster::start(3);
        let leader_base = cluster.node(LEADER);
        let gateway_bases = [
            cluster.node(LEADER),
            cluster.node(1), // forwarded quorum writes through a follower
        ];

        // ── Concurrent quorum writers, ledger = client-observed acks ───────
        let stop = Arc::new(AtomicBool::new(false));
        let id_base = 1_000 * (round as u64 + 1);
        let mut writers = Vec::new();
        for (w, base) in gateway_bases.iter().enumerate() {
            let base = base.clone();
            let stop = Arc::clone(&stop);
            writers.push(std::thread::spawn(move || {
                let client = reqwest::blocking::Client::builder()
                    .timeout(Duration::from_secs(4))
                    .build()
                    .unwrap();
                // (entity_id, item_seq, view_seq-if-acked)
                let mut acked: Vec<(u64, u64, Option<u64>)> = Vec::new();
                let mut n = 0u64;
                while !stop.load(Ordering::Acquire) {
                    let entity_id = id_base + (w as u64) * 500 + n;
                    n += 1;
                    let Some(item_seq) = post_acked(
                        &client,
                        &base,
                        "/items",
                        "quorum",
                        &serde_json::json!({
                            "entity_id": entity_id,
                            "metadata": { "title": item_token(entity_id) }
                        }),
                    ) else {
                        // Not acknowledged (leader dying/dead, forward failed,
                        // or quorum timeout): by contract it owes us nothing.
                        continue;
                    };
                    let view_seq = post_acked(
                        &client,
                        &base,
                        "/signals",
                        "quorum",
                        &serde_json::json!({
                            "entity_id": entity_id, "signal": "view", "weight": 1.0
                        }),
                    );
                    acked.push((entity_id, item_seq, view_seq));
                }
                acked
            }));
        }

        // Pseudo-random kill point: spread across boot-warm, mid-burst, and
        // saturated states deterministically (reproducible per round).
        let kill_after = Duration::from_millis(120 + (round as u64 * 97) % 480);
        std::thread::sleep(kill_after);
        cluster.kill_hard(LEADER);
        stop.store(true, Ordering::Release);
        let _ = client_drain(&leader_base); // flush any half-open socket
        let mut ledger: Vec<(u64, u64, Option<u64>)> = Vec::new();
        for w in writers {
            ledger.extend(w.join().expect("writer thread"));
        }
        let max_acked_seq = ledger
            .iter()
            .map(|(_, item_seq, view_seq)| view_seq.unwrap_or(*item_seq).max(*item_seq))
            .max()
            .unwrap_or(0);

        // ── Operator rule: promote the max-applied survivor ────────────────
        let survivors = [1usize, 2usize];
        let applied: Vec<(usize, u64)> = survivors
            .iter()
            .map(|&idx| {
                let status = cluster
                    .local_status(idx)
                    .expect("survivor must serve status");
                (idx, status["applied_events"].as_u64().unwrap())
            })
            .collect();
        let (chosen, chosen_applied) = applied
            .iter()
            .copied()
            .max_by_key(|&(_, a)| a)
            .expect("two survivors");

        // ── INVARIANT A (frontier): no acked seqno above the chosen
        // survivor's contiguous durable frontier ───────────────────────────
        assert!(
            max_acked_seq <= chosen_applied,
            "round {round}: ACKNOWLEDGED LOSS — max acked seq {max_acked_seq} exceeds the \
             max-applied survivor's frontier {chosen_applied} (applied: {applied:?}, \
             {} acked writes)",
            ledger.len()
        );

        let new_leader = cluster.region_name(chosen).to_string();
        let resp = cluster.post(
            chosen,
            "/cluster/promote",
            &serde_json::json!({ "region": new_leader }),
        );
        assert_eq!(resp.status().as_u16(), 200, "round {round}: promote");
        cluster.wait_leader_agreed(&new_leader, Duration::from_secs(10));

        // ── INVARIANT B (content): every acked item is on the new leader ───
        let client = reqwest::blocking::Client::builder()
            .timeout(Duration::from_secs(4))
            .build()
            .unwrap();
        // The text index auto-commits every 2s (engine default), so the FIRST
        // probe polls past the commit interval; data presence is what is
        // asserted, not commit timing.
        let new_leader_base = cluster.node(chosen);
        let search_deadline = Instant::now() + Duration::from_secs(10);
        for (entity_id, item_seq, _) in &ledger {
            let token = item_token(*entity_id);
            // Definitely assigned: the loop body's first statement writes it
            // before any break can be reached.
            let mut last: serde_json::Value;
            let present = loop {
                last = client
                    .get(format!("{new_leader_base}/search?query={token}&limit=5"))
                    .send()
                    .unwrap()
                    .json()
                    .unwrap();
                let hit = last["items"]
                    .as_array()
                    .unwrap_or(&Vec::new())
                    .iter()
                    .any(|it| it["entity_id"].as_u64() == Some(*entity_id));
                if hit {
                    break true;
                }
                if Instant::now() > search_deadline {
                    break false;
                }
                std::thread::sleep(Duration::from_millis(200));
            };
            assert!(
                present,
                "round {round}: ACKNOWLEDGED LOSS — item {entity_id} (seq {item_seq}, \
                 acked at quorum) is missing on promoted leader {new_leader}: {last}"
            );
        }
        println!(
            "[ledger] round {round}: kill@{kill_after:?} → {} acked writes (max seq \
             {max_acked_seq}) all present on {new_leader} (applied {chosen_applied})",
            ledger.len()
        );
        drop(cluster);
    }
}

/// Issue one throwaway GET so a dead leader's half-open client sockets are
/// observed closed before the ledger math (keeps the teardown deterministic
/// on macOS, where a killed process's sockets can linger in the client pool).
fn client_drain(base: &str) -> Option<()> {
    let client = reqwest::blocking::Client::builder()
        .timeout(Duration::from_millis(300))
        .build()
        .ok()?;
    let _ = client.get(format!("{base}/health/live")).send();
    Some(())
}