236 lines
8.5 KiB
Rust
236 lines
8.5 KiB
Rust
//! m11p5 — kind-4 `MembershipRecord` on the one replicated log (data layer).
|
|
//!
|
|
//! Proves the WAL-level membership machinery end to end through a real
|
|
//! persistent `TidalDb` in cluster mode:
|
|
//!
|
|
//! - a leader appends a kind-4 record, the `ClusterMembership` cell folds it,
|
|
//! and the seqno is durable;
|
|
//! - reopening the data dir re-derives the LATEST roster from the surviving log
|
|
//! (recovery fold — the highest version wins, no merge logic);
|
|
//! - the append path refuses version regressions and non-cluster mode.
|
|
//!
|
|
//! Consumer logic (quorum / election reconfigure) is NOT exercised here — this
|
|
//! is the data layer only, exactly the m11p5 A3 task slice.
|
|
|
|
#![allow(clippy::unwrap_used)]
|
|
|
|
use tidaldb::{
|
|
TempTidalHome, TidalDb,
|
|
db::config::{NodeConfig, NodeRole},
|
|
replication::shard::ShardId,
|
|
schema::{DecaySpec, EntityKind, SchemaBuilder, Window},
|
|
testing::{CrashInjector, CrashPoint, crash_injector::run_with_crash},
|
|
wal::format::{MemberEntry, MemberRole, MembershipRecord},
|
|
};
|
|
|
|
use std::time::Duration;
|
|
|
|
fn make_schema() -> tidaldb::schema::Schema {
|
|
let mut builder = SchemaBuilder::new();
|
|
let _ = builder
|
|
.signal(
|
|
"view",
|
|
EntityKind::Item,
|
|
DecaySpec::Exponential {
|
|
half_life: Duration::from_secs(7 * 24 * 3600),
|
|
},
|
|
)
|
|
.windows(&[Window::AllTime])
|
|
.velocity(false)
|
|
.add();
|
|
builder.build().unwrap()
|
|
}
|
|
|
|
/// Open a PERSISTENT cluster node (non-empty `peer_shards` → `replicate_blobs`,
|
|
/// the WAL becomes the one replicated log so kind-4 records can ride it).
|
|
fn open_cluster_node(home: &TempTidalHome, schema: tidaldb::schema::Schema) -> TidalDb {
|
|
TidalDb::builder()
|
|
.with_data_dir(home.path())
|
|
.with_schema(schema)
|
|
.with_cluster(NodeConfig {
|
|
role: NodeRole::Single,
|
|
shard_id: ShardId(0),
|
|
peer_shards: vec![ShardId(1)],
|
|
..NodeConfig::default()
|
|
})
|
|
.open()
|
|
.expect("persistent cluster node opens")
|
|
}
|
|
|
|
fn member(id: u16, name: &str, role: MemberRole) -> MemberEntry {
|
|
MemberEntry {
|
|
id,
|
|
name: name.to_string(),
|
|
grpc_addr: format!("tidaldb-{id}.peers.svc.cluster.local:9500"),
|
|
http_addr: format!("http://tidaldb-{id}.peers.svc.cluster.local:9501"),
|
|
role,
|
|
}
|
|
}
|
|
|
|
fn roster_v1() -> MembershipRecord {
|
|
MembershipRecord {
|
|
version: 1,
|
|
term: 2,
|
|
members: vec![
|
|
member(0, "us-east", MemberRole::Voter),
|
|
member(1, "eu-west", MemberRole::Voter),
|
|
member(2, "ap-south", MemberRole::Voter),
|
|
],
|
|
}
|
|
}
|
|
|
|
/// A v2 record that adds a learner (the join → learner conf-change shape).
|
|
fn roster_v2() -> MembershipRecord {
|
|
MembershipRecord {
|
|
version: 2,
|
|
term: 2,
|
|
members: vec![
|
|
member(0, "us-east", MemberRole::Voter),
|
|
member(1, "eu-west", MemberRole::Voter),
|
|
member(2, "ap-south", MemberRole::Voter),
|
|
member(3, "us-west", MemberRole::Learner),
|
|
],
|
|
}
|
|
}
|
|
|
|
/// A leader appends a membership record; the cell folds it and the seqno is
|
|
/// non-zero (a real durable WAL slot, never the dedup sentinel).
|
|
#[test]
|
|
fn append_folds_the_cell_and_is_durable() {
|
|
let home = TempTidalHome::new().unwrap();
|
|
let db = open_cluster_node(&home, make_schema());
|
|
|
|
// Topology era: no kind-4 record applied yet.
|
|
assert_eq!(db.cluster_membership(), None);
|
|
|
|
let seq = db.append_membership_record(roster_v1()).unwrap();
|
|
assert!(
|
|
seq >= 1,
|
|
"a membership record consumes a real seqno, got {seq}"
|
|
);
|
|
|
|
let (version, term, members) = db.cluster_membership().expect("cell folded the record");
|
|
assert_eq!(version, 1);
|
|
assert_eq!(term, 2);
|
|
assert_eq!(members, roster_v1().members);
|
|
}
|
|
|
|
/// Recovery fold: two records durable in the WAL, then a hard CRASH (no
|
|
/// graceful shutdown — so no compaction runs over the control records), then
|
|
/// reopen. Recovery re-derives the LATEST version (full snapshots, no merge
|
|
/// logic; the highest version below the durable frontier wins).
|
|
///
|
|
/// A crash — not a clean `close()` — is the faithful test: it exercises exactly
|
|
/// the path that re-derives the roster from the surviving log on restart, and
|
|
/// it sidesteps the shutdown-time WAL compaction that (absent the §2.1
|
|
/// retention pin, a later m11p5 stage) would reclaim a segment holding only
|
|
/// control records.
|
|
#[test]
|
|
fn recovery_re_derives_the_latest_roster_after_crash() {
|
|
let home = TempTidalHome::new().unwrap();
|
|
let schema = make_schema();
|
|
|
|
// The FIRST CheckpointPreFlush passes (our explicit force_replication_checkpoint
|
|
// lands the WAL marker durably at the records' seq); the SECOND (the shutdown
|
|
// ledger checkpoint) fires the crash, so no compaction runs.
|
|
let injector = CrashInjector::new(CrashPoint::CheckpointPreFlush, 1);
|
|
let outcome = run_with_crash(&injector, || {
|
|
let db = open_cluster_node(&home, schema.clone());
|
|
db.append_membership_record(roster_v1()).unwrap();
|
|
db.append_membership_record(roster_v2()).unwrap();
|
|
// The live cell already holds v2 before the crash.
|
|
assert_eq!(db.cluster_membership().unwrap().0, 2);
|
|
|
|
// Land a crash-consistent on-disk checkpoint (WAL marker at the records'
|
|
// seq); the membership records' seqnos are durable in the WAL.
|
|
db.force_replication_checkpoint().unwrap();
|
|
|
|
// CRASH during the shutdown ledger checkpoint — nothing past here, and
|
|
// no compaction, reaches disk.
|
|
let _ = db.close();
|
|
});
|
|
assert!(
|
|
matches!(outcome, Err(CrashPoint::CheckpointPreFlush)),
|
|
"the crash injector must have fired during shutdown, got {outcome:?}"
|
|
);
|
|
|
|
// Reopen the crashed node: recovery scans the surviving log and re-derives
|
|
// the highest version. v1 sits below v2 in the log; the monotonic-by-version
|
|
// advance makes "latest wins" without any delta merge.
|
|
let db = open_cluster_node(&home, schema);
|
|
let (version, term, members) = db
|
|
.cluster_membership()
|
|
.expect("recovery re-derived the roster from the surviving log");
|
|
assert_eq!(version, 2, "the highest-version record wins after recovery");
|
|
assert_eq!(term, 2);
|
|
assert_eq!(members, roster_v2().members);
|
|
assert_eq!(
|
|
members.len(),
|
|
4,
|
|
"the learner is present in the recovered roster"
|
|
);
|
|
}
|
|
|
|
/// A version that does not strictly advance the applied cell is refused — the
|
|
/// cell advances only forward, so a regressing or re-appended version could
|
|
/// never fold and must fail loudly rather than journal a silent no-op.
|
|
#[test]
|
|
fn version_regression_is_refused() {
|
|
let home = TempTidalHome::new().unwrap();
|
|
let db = open_cluster_node(&home, make_schema());
|
|
|
|
db.append_membership_record(roster_v2()).unwrap(); // version 2
|
|
|
|
// Re-appending the same version is refused.
|
|
let same = db.append_membership_record(roster_v2()).unwrap_err();
|
|
assert!(same.to_string().contains("does not advance"), "got: {same}");
|
|
|
|
// A lower version is refused.
|
|
let lower = db.append_membership_record(roster_v1()).unwrap_err();
|
|
assert!(
|
|
lower.to_string().contains("does not advance"),
|
|
"got: {lower}"
|
|
);
|
|
|
|
// The cell still holds v2 (no failed append mutated it).
|
|
assert_eq!(db.cluster_membership().unwrap().0, 2);
|
|
}
|
|
|
|
/// A structurally-invalid roster is refused before it enters the log (it would
|
|
/// fail apply identically on every follower).
|
|
#[test]
|
|
fn invalid_roster_is_refused() {
|
|
let home = TempTidalHome::new().unwrap();
|
|
let db = open_cluster_node(&home, make_schema());
|
|
|
|
let dup = MembershipRecord {
|
|
version: 1,
|
|
term: 1,
|
|
members: vec![
|
|
member(5, "a", MemberRole::Voter),
|
|
member(5, "b", MemberRole::Voter),
|
|
],
|
|
};
|
|
let err = db.append_membership_record(dup).unwrap_err();
|
|
assert!(
|
|
err.to_string().contains("duplicate member id"),
|
|
"got: {err}"
|
|
);
|
|
assert_eq!(db.cluster_membership(), None, "no record entered the log");
|
|
}
|
|
|
|
/// A non-cluster (standalone) node refuses the append: the WAL is not a
|
|
/// replicated log there.
|
|
#[test]
|
|
fn standalone_node_refuses_membership_append() {
|
|
let home = TempTidalHome::new().unwrap();
|
|
let db = TidalDb::builder()
|
|
.with_data_dir(home.path())
|
|
.with_schema(make_schema())
|
|
.open()
|
|
.expect("standalone node opens");
|
|
|
|
let err = db.append_membership_record(roster_v1()).unwrap_err();
|
|
assert!(err.to_string().contains("cluster mode"), "got: {err}");
|
|
}
|