tidaldb/tidal-server/tests/cluster_faults.rs

//! Tier-3 fault-injection suite (m11p9 continuous correctness).
//!
//! The chaos gate (`cluster_chaos.rs`) covers partition / crash / clock-skew.
//! This suite adds the fault classes the roadmap (§4/m11p9) names as missing:
//!
//! 1. **Disk-full on a follower** (`mp_disk_full_follower_degrades_no_acked_loss`)
//!    — a follower whose WAL hits `ENOSPC` mid-replication halts its receiver
//!    (degraded, alive), the healthy majority keeps acking quorum writes, NO
//!    acknowledged write is lost, and a restart with space recovers it to full
//!    parity.
//! 2. **Slow-fsync on a follower** (`mp_slow_fsync_follower_lags_but_quorum_holds`)
//!    — a slow disk lags one follower; the fast follower supplies quorum so
//!    every write still commits; the slow node converges once the burst ends; no
//!    loss.
//! 3. **Slow-fsync on BOTH followers** (`mp_slow_fsync_both_followers_force_honest_quorum_timeout`)
//!    — when no follower can confirm inside the budget, `ack=quorum` returns a
//!    retryable 503 naming the laggards (never a false success) while
//!    `ack=leader` is unaffected; the followers recover.
//! 4. **Asymmetric partition** (`mp_asymmetric_partition_no_split_brain_no_loss`)
//!    — a follower that loses INBOUND links (can still send) cannot disrupt the
//!    cluster: pre-vote + check-quorum keep the standing leader, there is never a
//!    second leader at the same term, and no acked write is lost.
//!
//! These are REAL faults exercising REAL recovery paths: the WAL hooks return a
//! genuine `ENOSPC` / sleep the real fsync (behind the `fault-injection` feature,
//! compiled out of production — see `tidal/src/fault.rs`); partitions sever real
//! loopback TCP. Every test asserts through the first-class invariant checkers
//! (`support::invariants`): the no-acked-loss ledger, cross-replica feed parity,
//! single-leader-per-term, and per-node monotonic frontiers.
//!
//! Run: `cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_faults -- --nocapture --test-threads 1`

#![cfg(feature = "cluster-e2e")]
#![allow(
    clippy::unwrap_used,
    clippy::expect_used,
    clippy::panic,
    clippy::cast_possible_truncation,
    clippy::cast_precision_loss,
    clippy::too_many_lines
)]

mod support;

use std::time::{Duration, Instant};

use support::{
    invariants::{AckLedger, MonotonicCounters, assert_single_leader_now, feed_item_ids},
    multiproc::{BREAKER_RESET, ClusterOptions, MultiProcCluster, convergence_budget},
    partition::proxied_rewrite,
};

const LEADER: usize = 0;
const FSYNC_DELAY: &str = "TIDAL_FAULT_FSYNC_DELAY_MS";
const DISK_FULL: &str = "TIDAL_FAULT_DISK_FULL_AFTER_BYTES";

/// Stable single-leader posture (no auto-election) + quorum default, for the
/// fault tests whose subject is the data plane, not the election. A faulted
/// node must not trigger a spurious failover that muddies the assertion.
const STABLE_QUORUM_YAML: &str =
    "election:\n  auto_election: false\nreplication:\n  ack: quorum\n  quorum_timeout_ms: 2000";

/// Drive `count` quorum item+view writes for ids `id_base..id_base+count`
/// through `base`, recording the client-observed acks into `ledger`. Returns
/// how many items the client saw acknowledged.
fn drive_quorum_writes(base: &str, id_base: u64, count: u64, ledger: &mut AckLedger) -> u64 {
    let client = reqwest::blocking::Client::builder()
        .timeout(Duration::from_secs(4))
        .build()
        .unwrap();
    let mut acked = 0;
    for i in 0..count {
        if ledger.write_item_and_view(&client, base, "quorum", id_base + i) {
            acked += 1;
        }
    }
    acked
}

/// **Disk-full on a follower.** A follower whose WAL fills mid-replication must
/// degrade gracefully (halt apply, stay alive — never corrupt, never crash),
/// the healthy majority must keep acking `ack=quorum` writes, NO acknowledged
/// write may be lost, and a restart with space must recover the follower to full
/// content + score parity.
#[test]
fn mp_disk_full_follower_degrades_no_acked_loss() {
    // ap-south (node 2) hits ENOSPC after ~8 KiB of post-boot WAL writes — far
    // past the empty-cluster boot (it writes nothing until it applies), but
    // crossed after a few dozen replicated records.
    let opts = ClusterOptions::new(3)
        .with_env(2, DISK_FULL, "8192")
        .with_topology_extra(STABLE_QUORUM_YAML);
    let mut cluster = MultiProcCluster::start_with(opts);
    cluster.wait_converged_all(convergence_budget());

    // Write a burst of quorum item+view writes through the leader. node 1
    // (healthy) supplies the one-of-two follower quorum, so every write commits
    // even after node 2's disk fills and its receiver halts.
    let mut ledger = AckLedger::new();
    let mut mono = MonotonicCounters::new("disk-full");
    let mut acked = 0;
    for round in 0..6 {
        acked += drive_quorum_writes(&cluster.node(LEADER), 10_000 + round * 20, 20, &mut ledger);
        mono.observe_cluster_frontiers(&cluster);
    }
    assert!(acked >= 100, "expected ~120 acked writes, got {acked}");

    // node 2 is FROZEN behind: its WAL is full, its receiver halted, so its
    // applied frontier stalled while node 1 stayed current. Poll briefly to let
    // node 1 finish converging and confirm the asymmetry.
    let leader_seq = cluster.leader_last_seq().expect("leader serves status");
    let deadline = Instant::now() + convergence_budget();
    loop {
        let n1 = cluster.local_status(1).expect("node1 status");
        let n2 = cluster.local_status(2).expect("node2 status");
        let n1_applied = n1["applied_events"].as_u64().unwrap_or(0);
        let n2_applied = n2["applied_events"].as_u64().unwrap_or(0);
        if n1_applied >= leader_seq && n2_applied < leader_seq {
            println!(
                "[disk-full] node1 caught up (applied {n1_applied}/{leader_seq}); \
                 node2 frozen at applied {n2_applied} (disk full)"
            );
            break;
        }
        assert!(
            Instant::now() <= deadline,
            "node1 must converge while node2 stays frozen: \
             n1_applied={n1_applied} n2_applied={n2_applied} leader_seq={leader_seq}"
        );
        std::thread::sleep(Duration::from_millis(100));
    }

    // No acked loss: every acked write is durable on the HEALTHY follower's
    // contiguous frontier (the real quorum-durability claim under a disk-full
    // minority), and every acked item is content-present on it.
    let (_keep, _) = ledger.assert_frontier_covers_acks(&cluster, &[1usize], "disk-full");
    ledger.assert_items_present(&cluster, 1, Duration::from_secs(10), "disk-full");

    // HEAL: restart node 2 with the fault disarmed (0 = off). A fresh receiver
    // catches the full log up via the stream — no operator verb beyond restart.
    cluster.restart(2, &[(DISK_FULL, "0")]);
    cluster.wait_converged_all(convergence_budget() + BREAKER_RESET);

    // The recovered disk-full node now holds every acked write. Convergence is
    // proven three ways: the contiguous frontier matched the leader
    // (wait_converged_all above), the materialized item SET matches the leader,
    // and every acked item is content-present via /search. We assert feed item-SET
    // parity, NOT score parity: `trending` scores are a windowed velocity
    // (count / window), and a node rebuilt from a burst WAL-replay aligns its time
    // buckets differently than one that accumulated continuously — so a velocity
    // score legitimately differs across a restart even with identical durable data.
    // Decay/score parity to 1e-6 between CONTINUOUS replicas is asserted by the
    // slow-fsync and asymmetric tests (and the chaos gate).
    let leader_ids = feed_item_ids(&cluster, LEADER, "trending", 200);
    let recovered_ids = feed_item_ids(&cluster, 2, "trending", 200);
    assert_eq!(
        leader_ids, recovered_ids,
        "recovered disk-full node must materialize the same item set as the leader"
    );
    ledger.assert_items_present(
        &cluster,
        2,
        Duration::from_secs(10),
        "disk-full (post-recovery)",
    );
    println!(
        "[disk-full] node2 recovered after restart: {acked} acked writes all present, \
         item set parity with leader ({} items)",
        leader_ids.len()
    );
}

/// **Slow-fsync on one follower.** A slow disk lags one follower; the fast
/// follower supplies quorum so every `ack=quorum` write still commits; the slow
/// node is visibly behind during the burst, converges once it ends, and loses
/// nothing.
#[test]
fn mp_slow_fsync_follower_lags_but_quorum_holds() {
    // ap-south (node 2) sleeps 250ms before every durable fsync.
    let opts = ClusterOptions::new(3)
        .with_env(2, FSYNC_DELAY, "250")
        .with_topology_extra(STABLE_QUORUM_YAML);
    let cluster = MultiProcCluster::start_with(opts);
    cluster.wait_converged_all(convergence_budget());

    let mut ledger = AckLedger::new();
    let mut mono = MonotonicCounters::new("slow-fsync-follower");
    let acked = drive_quorum_writes(&cluster.node(LEADER), 20_000, 60, &mut ledger);
    mono.observe_cluster_frontiers(&cluster);
    assert!(
        acked >= 55,
        "fast follower must supply quorum: only {acked} acked"
    );

    // Right after the burst the slow follower trails the fast one (250ms/fsync
    // ≫ the fast nodes' ~2ms group-commit), proving the slow disk is visible.
    let n1 = cluster.local_status(1).expect("node1 status");
    let n2 = cluster.local_status(2).expect("node2 status");
    let n1_lag = n1["lag_events"].as_u64().unwrap_or(u64::MAX);
    let n2_lag = n2["lag_events"].as_u64().unwrap_or(0);
    println!("[slow-fsync] after burst: node1 lag={n1_lag} node2(slow) lag={n2_lag}");
    assert!(
        n2_lag > n1_lag,
        "the slow follower must trail the fast one: node1 lag={n1_lag} node2 lag={n2_lag}"
    );

    // A slow disk is not a broken one: the burst over, the slow follower drains
    // and converges; no acked write is lost; frontiers never regressed.
    cluster.wait_converged_all(convergence_budget() * 2);
    mono.observe_cluster_frontiers(&cluster);
    let (_keep, _) =
        ledger.assert_frontier_covers_acks(&cluster, &[1usize, 2usize], "slow-fsync-follower");
    // Item-SET parity, not score: the slow follower drains its backlog as a burst
    // once the write burst ends, which reconstructs its time-bucketed velocity
    // differently than the leader's continuous accumulation — so a `trending`
    // (velocity) score can differ across the catch-up even with identical durable
    // data. The frontier + content + set checks prove data convergence robustly.
    let leader_ids = feed_item_ids(&cluster, LEADER, "trending", 200);
    let slow_ids = feed_item_ids(&cluster, 2, "trending", 200);
    assert_eq!(
        leader_ids, slow_ids,
        "converged slow follower must materialize the same item set as the leader"
    );
    ledger.assert_items_present(&cluster, 2, Duration::from_secs(10), "slow-fsync-follower");
    println!("[slow-fsync] {acked} acked writes, slow follower converged to parity, no loss");
}

/// **Slow-fsync on BOTH followers + tight budget → honest quorum timeout.** When
/// neither follower can confirm a write inside `quorum_timeout_ms`, `ack=quorum`
/// must return a retryable 503 naming the laggards — never a false 2xx — while
/// `ack=leader` on the same cluster keeps succeeding (the leader's disk is fast).
/// The slow followers are not broken, so the cluster converges once load eases.
#[test]
fn mp_slow_fsync_both_followers_force_honest_quorum_timeout() {
    // Both followers sleep 1500ms per fsync; the quorum budget is 300ms — so the
    // commit index physically cannot advance to a fresh write inside the budget.
    let yaml =
        "election:\n  auto_election: false\nreplication:\n  ack: quorum\n  quorum_timeout_ms: 300";
    let opts = ClusterOptions::new(3)
        .with_env(1, FSYNC_DELAY, "1500")
        .with_env(2, FSYNC_DELAY, "1500")
        .with_topology_extra(yaml);
    let cluster = MultiProcCluster::start_with(opts);
    cluster.wait_converged_all(convergence_budget());

    let client = cluster.client();
    let leader = cluster.node(LEADER);

    // ack=quorum (the topology default): the budget expires before either slow
    // follower fsyncs the apply → retryable 503 naming the laggards.
    let q = client
        .post(format!("{leader}/signals"))
        .json(&serde_json::json!({ "entity_id": 30_001, "signal": "view", "weight": 1.0 }))
        .send()
        .expect("quorum write sends");
    assert_eq!(
        q.status().as_u16(),
        503,
        "ack=quorum must time out honestly against two slow followers"
    );
    let body: serde_json::Value = q.json().expect("503 carries a JSON body");
    assert_eq!(
        body["retryable"].as_bool(),
        Some(true),
        "503 must be retryable: {body}"
    );
    assert!(
        body["laggards"].as_array().is_some_and(|l| !l.is_empty()),
        "503 must name the laggards: {body}"
    );
    println!(
        "[both-slow] ack=quorum → honest 503 retryable, laggards={}",
        body["laggards"]
    );

    // ack=leader on the same cluster is unaffected (the leader's disk is fast):
    // the caller's durability choice is the caller's, never the deployment's.
    let l = client
        .post(format!("{leader}/signals"))
        .header("x-tidal-ack", "leader")
        .json(&serde_json::json!({ "entity_id": 30_002, "signal": "view", "weight": 1.0 }))
        .send()
        .expect("leader write sends");
    assert_eq!(
        l.status().as_u16(),
        204,
        "ack=leader must keep succeeding while ack=quorum times out: {}",
        l.status()
    );
    println!("[both-slow] ack=leader → 204 (unaffected)");

    // The slow followers are not broken: they DO apply, just slowly. Once the
    // single in-flight write drains, the cluster converges (the timed-out write
    // is in the leader's log and commits late — at-least-once, never lost).
    cluster.wait_converged_all(convergence_budget() * 3);
    println!("[both-slow] slow followers drained and converged");
}

/// **Asymmetric partition.** A follower that loses its INBOUND links (peers
/// cannot reach it; it can still send) must not disrupt the cluster: pre-vote +
/// check-quorum keep the standing leader, there is never a second leader at the
/// same term, the reachable majority keeps acking quorum writes with zero loss,
/// and the follower rejoins cleanly on heal.
#[test]
fn mp_asymmetric_partition_no_split_brain_no_loss() {
    // Proxy only the edges INTO ap-south; ap-south's own outbound dials are
    // unproxied (identity), so severing the inbound edges is a true ASYMMETRIC
    // cut: ap-south stops RECEIVING heartbeats/ships but can still SEND the
    // disruptive RequestVotes pre-vote is designed to neutralize.
    let (rewrite, proxies) = proxied_rewrite(&["ap-south"]);
    let fast_election = "election:\n  heartbeat_interval_ms: 100\n  election_timeout_min_ms: 500\n  \
         election_timeout_max_ms: 1000\n  leader_lease_ms: 350\nreplication:\n  ack: quorum";
    let opts = ClusterOptions::new(3)
        .with_rewrite(rewrite)
        .with_topology_extra(fast_election);
    let cluster = MultiProcCluster::start_with(opts);
    cluster.wait_leader_agreed("us-east", convergence_budget());
    cluster.wait_converged_all(convergence_budget());

    // Baseline writes; capture the standing leader's term.
    let mut ledger = AckLedger::new();
    drive_quorum_writes(&cluster.node(LEADER), 40_000, 30, &mut ledger);
    let base_term = cluster.local_status(LEADER).expect("leader status")["term"]
        .as_u64()
        .unwrap_or(0);

    // ── Asymmetric sever: cut every inbound edge to ap-south ──────────────────
    proxies.region("ap-south").sever_all();

    // The reachable majority keeps acking quorum writes (us-east + eu-west).
    let acked = drive_quorum_writes(&cluster.node(LEADER), 41_000, 30, &mut ledger);
    assert!(
        acked >= 25,
        "reachable majority must keep acking: only {acked}"
    );

    // Through several election timeouts: the standing leader is unchanged, there
    // is never a second leader at any term, and ap-south's term does not explode
    // (pre-vote blocks a node that can't receive grants from bumping its term).
    let watch_until = Instant::now() + Duration::from_secs(12);
    let mut max_apsouth_term = base_term;
    while Instant::now() < watch_until {
        // The shared checker snapshots every live node and asserts
        // single-leader-per-term; reuse its snapshots for the rest of the checks
        // (no second fetch, no hardcoded node count).
        let statuses = assert_single_leader_now(&cluster, "asymmetric-partition");
        // The two reachable nodes still agree us-east leads.
        for st in &statuses {
            let region = st["region"].as_str().unwrap_or("?");
            if region == "us-east" || region == "eu-west" {
                assert_eq!(
                    st["leader"].as_str(),
                    Some("us-east"),
                    "reachable node {region} must still see us-east as leader: {st}"
                );
            }
            if region == "ap-south" {
                max_apsouth_term = max_apsouth_term.max(st["term"].as_u64().unwrap_or(0));
            }
        }
        std::thread::sleep(Duration::from_millis(200));
    }
    assert!(
        max_apsouth_term <= base_term + 2,
        "ap-south term exploded under asymmetric partition ({base_term} → {max_apsouth_term}); \
         pre-vote should have neutralized the disruptive node"
    );
    println!(
        "[asymmetric] leader held us-east, single-leader-per-term, ap-south term bounded \
         ({base_term} → {max_apsouth_term})"
    );

    // No acked loss: every acked write is durable on the reachable follower.
    let (_keep, _) =
        ledger.assert_frontier_covers_acks(&cluster, &[1usize], "asymmetric-partition");
    ledger.assert_items_present(
        &cluster,
        LEADER,
        Duration::from_secs(10),
        "asymmetric-partition",
    );

    // HEAL: ap-south's inbound returns; it catches up and rejoins as a follower.
    proxies.region("ap-south").heal_all();
    cluster.wait_converged_all(convergence_budget() + BREAKER_RESET);
    // Item-SET parity, not score: ap-south applies the partition backlog as a
    // burst on heal, reconstructing its velocity buckets differently than the
    // leader's continuous accumulation — so trending (velocity) scores can differ
    // across the catch-up even with identical durable data. Set + content +
    // frontier prove data convergence robustly.
    let leader_ids = feed_item_ids(&cluster, LEADER, "trending", 200);
    let rejoined_ids = feed_item_ids(&cluster, 2, "trending", 200);
    assert_eq!(
        leader_ids, rejoined_ids,
        "rejoined ap-south must materialize the same item set as the leader"
    );
    ledger.assert_items_present(
        &cluster,
        2,
        Duration::from_secs(10),
        "asymmetric-partition (rejoined)",
    );
    println!("[asymmetric] ap-south rejoined to parity, no acked loss");
}