tidaldb/tidal/tests/m12_reseed_term_marker.rs

//! m12 reseed-loop-fix — the durable term marker the reseed seed synthesizes.
//!
//! The boot-time snapshot install restores a checkpoint-based artifact whose
//! engine WAL is EMPTY, so `wal_term_mark()` would report `tail_term = 0`. The
//! post-open seed (`ShardReplica::new`) then durably synthesizes the artifact's
//! term marker via [`TidalDb::append_term_marker`] so the WAL-tail term is
//! truthful — on this boot AND every reboot — which is what stops `decide_join`
//! from looping a checkpoint-restored node through `ReseedRequired` forever.
//!
//! This test proves the DURABILITY MECHANISM that fix relies on at the engine
//! layer (the `decide_join` classification itself is unit-tested in
//! `cluster::election_driver`):
//!
//! - `append_term_marker` folds the WAL-tail term cell immediately (this boot);
//! - the marker survives a replication checkpoint whose WAL marker advances
//!   PAST the marker's seqno (the exact reseed-seed ordering: marker, then
//!   `persist_replication_checkpoint`) AND a hard crash-reopen — because blob
//!   records are deliberately NOT checkpoint-filtered on recovery
//!   (`wal::reader`), so recovery re-derives the truthful term on every restart;
//! - the append path refuses term 0 (the topology era) and non-cluster mode.

#![allow(clippy::unwrap_used)]

use tidaldb::{
    TempTidalHome, TidalDb,
    db::config::{NodeConfig, NodeRole},
    replication::shard::ShardId,
    schema::{DecaySpec, EntityKind, SchemaBuilder, Window},
    testing::{CrashInjector, CrashPoint, crash_injector::run_with_crash},
    wal::format::{MemberEntry, MemberRole, MembershipRecord},
};

use std::time::Duration;

fn make_schema() -> tidaldb::schema::Schema {
    let mut builder = SchemaBuilder::new();
    let _ = builder
        .signal(
            "view",
            EntityKind::Item,
            DecaySpec::Exponential {
                half_life: Duration::from_secs(7 * 24 * 3600),
            },
        )
        .windows(&[Window::AllTime])
        .velocity(false)
        .add();
    builder.build().unwrap()
}

/// Open a PERSISTENT cluster node (non-empty `peer_shards` → `replicate_blobs`,
/// the WAL becomes the one replicated log so kind-3 markers can ride it).
fn open_cluster_node(home: &TempTidalHome, schema: tidaldb::schema::Schema) -> TidalDb {
    TidalDb::builder()
        .with_data_dir(home.path())
        .with_schema(schema)
        .with_cluster(NodeConfig {
            role: NodeRole::Single,
            shard_id: ShardId(0),
            peer_shards: vec![ShardId(1)],
            ..NodeConfig::default()
        })
        .open()
        .expect("persistent cluster node opens")
}

/// A kind-4 record used only to push the WAL frontier (and thus the checkpoint
/// seq) PAST the term marker, so the durability assertion is "the marker
/// survives a checkpoint advanced beyond it", not merely "at it".
fn roster() -> MembershipRecord {
    MembershipRecord {
        version: 1,
        term: 7,
        members: vec![
            MemberEntry {
                id: 0,
                name: "us-east".to_string(),
                grpc_addr: "tidaldb-0:9500".to_string(),
                http_addr: "http://tidaldb-0:9501".to_string(),
                role: MemberRole::Voter,
            },
            MemberEntry {
                id: 1,
                name: "eu-west".to_string(),
                grpc_addr: "tidaldb-1:9500".to_string(),
                http_addr: "http://tidaldb-1:9501".to_string(),
                role: MemberRole::Voter,
            },
        ],
    }
}

/// The reseed seed's `append_term_marker` folds the WAL-tail term cell
/// immediately: `wal_term_mark()` reports the artifact's (term, region) the
/// moment the seed runs, on the very boot that installed the snapshot.
#[test]
fn append_term_marker_folds_the_cell_this_boot() {
    let home = TempTidalHome::new().unwrap();
    let db = open_cluster_node(&home, make_schema());

    // A checkpoint-restored artifact's WAL is empty → topology era, term 0.
    assert_eq!(db.wal_term_mark(), (0, 0, 0));

    // The seed synthesizes the discovered leader's (term 7, region 1).
    let seq = db.append_term_marker(7, 1).unwrap();
    assert!(seq >= 1, "the marker consumes a real seqno, got {seq}");

    let (term, marker_seq, region) = db.wal_term_mark();
    assert_eq!(term, 7, "the WAL-tail term is the artifact's term, not 0");
    assert_eq!(marker_seq, seq);
    assert_eq!(region, 1, "the marker names the reseed-leader stream");
}

/// THE durability property the reseed-loop-fix hinges on: a synthesized term
/// marker survives a replication checkpoint whose WAL marker lands PAST the
/// marker's seqno AND a hard crash-reopen. If it did not, the next boot would
/// re-read `tail_term = 0` and re-latch `ReseedRequired` — the loop.
///
/// A crash (not a clean `close()`) is the faithful test: it exercises the
/// re-derive-from-the-surviving-log path and sidesteps the shutdown-time WAL
/// compaction that could reclaim a control-only segment.
#[test]
fn term_marker_survives_checkpoint_past_it_and_crash_reopen() {
    let home = TempTidalHome::new().unwrap();
    let schema = make_schema();

    // The first CheckpointPreFlush passes (our explicit force_replication_checkpoint
    // lands the WAL marker durably PAST the term marker's seq); the second (the
    // shutdown ledger checkpoint) fires the crash, so no compaction runs.
    let injector = CrashInjector::new(CrashPoint::CheckpointPreFlush, 1);
    let outcome = run_with_crash(&injector, || {
        let db = open_cluster_node(&home, schema.clone());

        // The reseed seed: synthesize the artifact's term marker (lands at seq 1).
        let marker_seq = db.append_term_marker(7, 1).unwrap();
        // Push the frontier PAST the marker so the checkpoint seq exceeds it —
        // this is what proves the marker survives a checkpoint advanced BEYOND
        // it (blobs are not checkpoint-filtered), not merely a checkpoint at it.
        let later_seq = db.append_membership_record(roster()).unwrap();
        assert!(
            later_seq > marker_seq,
            "the membership record must sit above the term marker (got {later_seq} > {marker_seq})"
        );
        // The live cell already reports the truthful term before the crash.
        assert_eq!(db.wal_term_mark().0, 7);

        // Land a crash-consistent checkpoint: the WAL marker advances to
        // `later_seq` — strictly past the term marker at `marker_seq`. This is
        // the exact reseed-seed ordering (append marker, then persist checkpoint),
        // via the SAME public API the seed calls.
        db.persist_replication_checkpoint().unwrap();

        // CRASH during the shutdown ledger checkpoint — nothing past here, and
        // no compaction, reaches disk.
        let _ = db.close();
    });
    assert!(
        matches!(outcome, Err(CrashPoint::CheckpointPreFlush)),
        "the crash injector must have fired during shutdown, got {outcome:?}"
    );

    // Reopen the crashed node: recovery scans the surviving log. The term marker
    // sits BELOW the checkpoint boundary, but blob records are not checkpoint-
    // filtered, so recovery re-folds it → the WAL-tail term is truthful again.
    let db = open_cluster_node(&home, schema);
    let (term, _seq, region) = db.wal_term_mark();
    assert_eq!(
        term, 7,
        "recovery re-derives the artifact's term across the checkpoint boundary — \
         not 0, so decide_join will NOT re-latch ReseedRequired"
    );
    assert_eq!(
        region, 1,
        "the recovered marker still names the reseed-leader stream"
    );
}

/// The append path refuses term 0 (the topology era never journals a marker, so
/// the seed correctly SKIPS the synthesis for a term-0 artifact).
#[test]
fn append_term_marker_refuses_term_zero() {
    let home = TempTidalHome::new().unwrap();
    let db = open_cluster_node(&home, make_schema());

    let err = db.append_term_marker(0, 1).unwrap_err();
    assert!(
        err.to_string().contains("elected terms"),
        "term 0 is refused, got: {err}"
    );
    assert_eq!(db.wal_term_mark(), (0, 0, 0), "no marker entered the log");
}

/// A non-cluster (standalone) node refuses the append: the WAL is not a
/// replicated log there, so the reseed seed (which only runs in cluster mode)
/// could never reach this.
#[test]
fn standalone_node_refuses_term_marker_append() {
    let home = TempTidalHome::new().unwrap();
    let db = TidalDb::builder()
        .with_data_dir(home.path())
        .with_schema(make_schema())
        .open()
        .expect("standalone node opens");

    let err = db.append_term_marker(7, 1).unwrap_err();
    assert!(err.to_string().contains("cluster mode"), "got: {err}");
}