//! m12 reseed-loop-fix — the durable term marker the reseed seed synthesizes. //! //! The boot-time snapshot install restores a checkpoint-based artifact whose //! engine WAL is EMPTY, so `wal_term_mark()` would report `tail_term = 0`. The //! post-open seed (`ShardReplica::new`) then durably synthesizes the artifact's //! term marker via [`TidalDb::append_term_marker`] so the WAL-tail term is //! truthful — on this boot AND every reboot — which is what stops `decide_join` //! from looping a checkpoint-restored node through `ReseedRequired` forever. //! //! This test proves the DURABILITY MECHANISM that fix relies on at the engine //! layer (the `decide_join` classification itself is unit-tested in //! `cluster::election_driver`): //! //! - `append_term_marker` folds the WAL-tail term cell immediately (this boot); //! - the marker survives a replication checkpoint whose WAL marker advances //! PAST the marker's seqno (the exact reseed-seed ordering: marker, then //! `persist_replication_checkpoint`) AND a hard crash-reopen — because blob //! records are deliberately NOT checkpoint-filtered on recovery //! (`wal::reader`), so recovery re-derives the truthful term on every restart; //! - the append path refuses term 0 (the topology era) and non-cluster mode. #![allow(clippy::unwrap_used)] use tidaldb::{ TempTidalHome, TidalDb, db::config::{NodeConfig, NodeRole}, replication::shard::ShardId, schema::{DecaySpec, EntityKind, SchemaBuilder, Window}, testing::{CrashInjector, CrashPoint, crash_injector::run_with_crash}, wal::format::{MemberEntry, MemberRole, MembershipRecord}, }; use std::time::Duration; fn make_schema() -> tidaldb::schema::Schema { let mut builder = SchemaBuilder::new(); let _ = builder .signal( "view", EntityKind::Item, DecaySpec::Exponential { half_life: Duration::from_secs(7 * 24 * 3600), }, ) .windows(&[Window::AllTime]) .velocity(false) .add(); builder.build().unwrap() } /// Open a PERSISTENT cluster node (non-empty `peer_shards` → `replicate_blobs`, /// the WAL becomes the one replicated log so kind-3 markers can ride it). fn open_cluster_node(home: &TempTidalHome, schema: tidaldb::schema::Schema) -> TidalDb { TidalDb::builder() .with_data_dir(home.path()) .with_schema(schema) .with_cluster(NodeConfig { role: NodeRole::Single, shard_id: ShardId(0), peer_shards: vec![ShardId(1)], ..NodeConfig::default() }) .open() .expect("persistent cluster node opens") } /// A kind-4 record used only to push the WAL frontier (and thus the checkpoint /// seq) PAST the term marker, so the durability assertion is "the marker /// survives a checkpoint advanced beyond it", not merely "at it". fn roster() -> MembershipRecord { MembershipRecord { version: 1, term: 7, members: vec![ MemberEntry { id: 0, name: "us-east".to_string(), grpc_addr: "tidaldb-0:9500".to_string(), http_addr: "http://tidaldb-0:9501".to_string(), role: MemberRole::Voter, }, MemberEntry { id: 1, name: "eu-west".to_string(), grpc_addr: "tidaldb-1:9500".to_string(), http_addr: "http://tidaldb-1:9501".to_string(), role: MemberRole::Voter, }, ], } } /// The reseed seed's `append_term_marker` folds the WAL-tail term cell /// immediately: `wal_term_mark()` reports the artifact's (term, region) the /// moment the seed runs, on the very boot that installed the snapshot. #[test] fn append_term_marker_folds_the_cell_this_boot() { let home = TempTidalHome::new().unwrap(); let db = open_cluster_node(&home, make_schema()); // A checkpoint-restored artifact's WAL is empty → topology era, term 0. assert_eq!(db.wal_term_mark(), (0, 0, 0)); // The seed synthesizes the discovered leader's (term 7, region 1). let seq = db.append_term_marker(7, 1).unwrap(); assert!(seq >= 1, "the marker consumes a real seqno, got {seq}"); let (term, marker_seq, region) = db.wal_term_mark(); assert_eq!(term, 7, "the WAL-tail term is the artifact's term, not 0"); assert_eq!(marker_seq, seq); assert_eq!(region, 1, "the marker names the reseed-leader stream"); } /// THE durability property the reseed-loop-fix hinges on: a synthesized term /// marker survives a replication checkpoint whose WAL marker lands PAST the /// marker's seqno AND a hard crash-reopen. If it did not, the next boot would /// re-read `tail_term = 0` and re-latch `ReseedRequired` — the loop. /// /// A crash (not a clean `close()`) is the faithful test: it exercises the /// re-derive-from-the-surviving-log path and sidesteps the shutdown-time WAL /// compaction that could reclaim a control-only segment. #[test] fn term_marker_survives_checkpoint_past_it_and_crash_reopen() { let home = TempTidalHome::new().unwrap(); let schema = make_schema(); // The first CheckpointPreFlush passes (our explicit force_replication_checkpoint // lands the WAL marker durably PAST the term marker's seq); the second (the // shutdown ledger checkpoint) fires the crash, so no compaction runs. let injector = CrashInjector::new(CrashPoint::CheckpointPreFlush, 1); let outcome = run_with_crash(&injector, || { let db = open_cluster_node(&home, schema.clone()); // The reseed seed: synthesize the artifact's term marker (lands at seq 1). let marker_seq = db.append_term_marker(7, 1).unwrap(); // Push the frontier PAST the marker so the checkpoint seq exceeds it — // this is what proves the marker survives a checkpoint advanced BEYOND // it (blobs are not checkpoint-filtered), not merely a checkpoint at it. let later_seq = db.append_membership_record(roster()).unwrap(); assert!( later_seq > marker_seq, "the membership record must sit above the term marker (got {later_seq} > {marker_seq})" ); // The live cell already reports the truthful term before the crash. assert_eq!(db.wal_term_mark().0, 7); // Land a crash-consistent checkpoint: the WAL marker advances to // `later_seq` — strictly past the term marker at `marker_seq`. This is // the exact reseed-seed ordering (append marker, then persist checkpoint), // via the SAME public API the seed calls. db.persist_replication_checkpoint().unwrap(); // CRASH during the shutdown ledger checkpoint — nothing past here, and // no compaction, reaches disk. let _ = db.close(); }); assert!( matches!(outcome, Err(CrashPoint::CheckpointPreFlush)), "the crash injector must have fired during shutdown, got {outcome:?}" ); // Reopen the crashed node: recovery scans the surviving log. The term marker // sits BELOW the checkpoint boundary, but blob records are not checkpoint- // filtered, so recovery re-folds it → the WAL-tail term is truthful again. let db = open_cluster_node(&home, schema); let (term, _seq, region) = db.wal_term_mark(); assert_eq!( term, 7, "recovery re-derives the artifact's term across the checkpoint boundary — \ not 0, so decide_join will NOT re-latch ReseedRequired" ); assert_eq!( region, 1, "the recovered marker still names the reseed-leader stream" ); } /// The append path refuses term 0 (the topology era never journals a marker, so /// the seed correctly SKIPS the synthesis for a term-0 artifact). #[test] fn append_term_marker_refuses_term_zero() { let home = TempTidalHome::new().unwrap(); let db = open_cluster_node(&home, make_schema()); let err = db.append_term_marker(0, 1).unwrap_err(); assert!( err.to_string().contains("elected terms"), "term 0 is refused, got: {err}" ); assert_eq!(db.wal_term_mark(), (0, 0, 0), "no marker entered the log"); } /// A non-cluster (standalone) node refuses the append: the WAL is not a /// replicated log there, so the reseed seed (which only runs in cluster mode) /// could never reach this. #[test] fn standalone_node_refuses_term_marker_append() { let home = TempTidalHome::new().unwrap(); let db = TidalDb::builder() .with_data_dir(home.path()) .with_schema(make_schema()) .open() .expect("standalone node opens"); let err = db.append_term_marker(7, 1).unwrap_err(); assert!(err.to_string().contains("cluster mode"), "got: {err}"); }