diff --git a/tidal-server/src/cluster/node.rs b/tidal-server/src/cluster/node.rs index b0fa737..a991209 100644 --- a/tidal-server/src/cluster/node.rs +++ b/tidal-server/src/cluster/node.rs @@ -3425,20 +3425,32 @@ impl ShardReplica { .unwrap_or(0) } - fn note_lag_for_readiness(&self, lag_events: u64) { - if (self.install_boot || self.seed_joiner) - && !self.converged.load(Ordering::Acquire) - && lag_events <= self.learner_promote_lag - { + /// Record convergence progress for the sticky readiness latch (§4). + /// + /// `leader_seqno` is this node's KNOWLEDGE of the followed leader's frontier. + /// It is required and must be non-zero: `lag_events` is + /// `leader_seqno - applied`, an unsigned subtraction, so a node that has + /// learned nothing computes `0 - 0 = 0` and would latch "converged" while + /// holding no data at all. That is not hypothetical — in the multi-group + /// reproduction all three groups reported `lag_events: 0` with + /// `applied_events` of 24, 14 and 0 against 5600 written items, and in + /// production a PVC-wiped tidaldb-0 entered the client VIP with an EMPTY + /// corpus. Convergence must rest on a frontier we actually learned. + fn note_lag_for_readiness(&self, leader_seqno: u64, lag_events: u64) { + if leader_seqno == 0 { + return; // no information yet — absence of lag is not convergence + } + if !self.converged.load(Ordering::Acquire) && lag_events <= self.learner_promote_lag { self.converged.store(true, Ordering::Release); tracing::info!( region = %self.region_name, + leader_seqno, lag_events, threshold = self.learner_promote_lag, seed_joiner = self.seed_joiner, install_boot = self.install_boot, - "joiner first-converged (lag <= learner_promote_lag); readiness is now \ - sticky-ready for this process (§4 hysteresis)" + "first-converged against a KNOWN leader frontier (lag <= learner_promote_lag); \ + readiness is now sticky-ready for this process (§4 hysteresis)" ); } } @@ -3506,8 +3518,14 @@ impl ShardReplica { { self.transport.notify_applied(leader_shard, applied); } - if (self.install_boot || self.seed_joiner) && !self.converged.load(Ordering::Acquire) { - self.note_lag_for_readiness(leader_last_seq.saturating_sub(applied)); + // Drive the convergence latch for EVERY boot, not just install/seed-join. + // A plain restarted voter used to skip this entirely and be Ready on + // arrival, so it joined the client VIP before it knew whether it held the + // data — which is how a PVC-wiped tidaldb-0 served an EMPTY corpus. The + // heartbeat carries the leader's live frontier, so it is the signal that + // makes convergence knowable on an idle cluster too (m12p5). + if !self.converged.load(Ordering::Acquire) { + self.note_lag_for_readiness(leader_last_seq, leader_last_seq.saturating_sub(applied)); } // NO reseed-marker discharge on the heartbeat path. This is where both // unsound predicates lived (`applied >= leader_last_seq`, then `applied >= @@ -3547,19 +3565,52 @@ impl ShardReplica { if self.decommissioned_by_signal.load(Ordering::Acquire) { return false; } - if (self.install_boot || self.seed_joiner) && !self.converged.load(Ordering::Acquire) { - return false; - } // m12 reseed-loop-fix (readiness gating): an unhealed reseed marker // (snapshot-required or quarantine) means this node holds stale data it is - // about to discard — drain it from the client VIP until it heals. This - // closes the plain-restart (install_boot == false) gap: a PVC-retained - // voter that re-latched snapshot-required while merely behind used to keep - // serving stale reads. The latch clears when the node catches up via the - // stream (clear_stale_reseed_marker_if_caught_up) or reseeds next boot. + // about to discard — drain it from the client VIP until it heals. The latch + // clears when a completed catch-up pull proves the stream served the gap + // (`discharge_reseed_marker_if_served`) or the next boot reseeds. if self.reseed_marker_latched.load(Ordering::Acquire) { return false; } + // POSITIVE EVIDENCE, every boot. This was + // `if (install_boot || seed_joiner) && !converged`, so a plain restarted + // voter fell straight through to ready — admitted to the client VIP before + // it had learned the leader's frontier, let alone caught up. Combined with + // `lag_events` reading `0 - 0 = 0` on an uninitialized gauge, that is how a + // PVC-wiped tidaldb-0 served an EMPTY corpus, and how a node missing 15,000 + // entries reported itself converged. + // + // `converged` is STICKY for the process (§4 hysteresis), so this costs a + // restarted voter only the time to receive one heartbeat carrying the + // leader's frontier — heartbeats flow on an idle cluster (m12p5) — and a + // later transient leader loss never un-readies it. A node that genuinely + // cannot reach a leader stays 503, which is the honest answer: it does not + // know whether it holds the data. + if !self.converged.load(Ordering::Acquire) { + // A LEADER is trivially converged: it WRITES the log rather than + // applying someone else's, so there is no frontier to catch up to. This + // arm is load-bearing for bootstrap — a fresh cluster's leader has + // `last_seq == 0`, so requiring a non-zero learned frontier would leave + // it permanently 503 and the cluster would never come up. + // + // It MUST test ESTABLISHED leadership from the election runtime, never + // `current_leader()`. That view is seeded from the TOPOLOGY FILE, and in + // a sharded topology group `s` names node `s` as its term-0 leader — so + // a booting node would self-certify convergence for the group it merely + // believes it leads, while holding none of that group's data. The + // durable §1.4-1 rule is that a restart always boots a FOLLOWER, so the + // runtime role is the only honest source here. + let established_leader = self + .election_runtime + .get() + .is_some_and(|rt| matches!(rt.role(), tidaldb::replication::Role::Leader)); + if established_leader { + self.converged.store(true, Ordering::Release); + } else { + return false; + } + } true } @@ -3778,13 +3829,13 @@ impl ShardReplica { // m11p5 §4: feed the sticky readiness latch from the lag we just // computed (no separate polling thread) and report the durable reseed // state. - self.note_lag_for_readiness(lag_events); + self.note_lag_for_readiness(leader_seqno, lag_events); let reseed_required = self.reseed_marker_store.exists(); - // `reseeding` = a joiner (snapshot-install OR seed-join) that has not yet - // first-converged (the catch-up that follows the install/join is in - // flight). Readiness is 503 while this is true. - let reseeding = - (self.install_boot || self.seed_joiner) && !self.converged.load(Ordering::Acquire); + // `reseeding` = this node has not yet first-converged against a KNOWN + // leader frontier. No longer scoped to joiner boots: a plain restart is + // equally un-converged until it learns where the leader is, and reporting + // it as settled is what let a blind voter into the VIP. + let reseeding = !self.converged.load(Ordering::Acquire); Ok(LocalStatusResponse { region: self.region_name.clone(), diff --git a/tidal-server/tests/cluster_reseed.rs b/tidal-server/tests/cluster_reseed.rs index 288835c..2c61207 100644 --- a/tidal-server/tests/cluster_reseed.rs +++ b/tidal-server/tests/cluster_reseed.rs @@ -626,27 +626,38 @@ fn mp_follower_reseeds_via_snapshot_after_compaction() { /// The gate is a FIXPOINT: node 2 must end Ready, reporting no reseed marker, with /// content readable from every group, within a bounded number of restarts. /// -/// # CURRENTLY REPRODUCES AN OPEN DEFECT — `#[ignore]`d, not broken +/// # This gate CLOSED the multi-group reseed defect /// -/// This test FAILS today, on purpose: it is the reproduction for a real bug that -/// the served-evidence marker fix does NOT close. Observed run: +/// It was written as a reproduction and failed exactly as production did — the node +/// announced itself settled and was missing data: /// /// ```text /// [multi] node 2 settled after 0 orchestrator restart(s) /// [multi] node 2 exited AFTER settling; orchestrator reboot #1 /// missing item 500 (reboots=1) ... reseed_required: false, lag_events: 0, -/// applied_events: 3798, election_tail_term: 2 +/// applied_events: 3798 /// ``` /// -/// A multi-group node reports NO reseed marker and ZERO lag while an item written -/// before its outage is absent — the silent-hole shape, and the local twin of the -/// 2026-08-20 production failure where tidaldb-0 reported `lag_events: 0` on a -/// replica missing history. Single-group reseed is correct -/// (`mp_follower_reseeds_via_snapshot_after_compaction` passes with the same content -/// probe); the multi-group path is not. +/// Root cause was readiness asserting on ABSENCE of bad news. `is_ready` gated +/// convergence behind `install_boot || seed_joiner`, so a plain restarted voter was +/// Ready on arrival — before it had learned the leader's frontier — and +/// `lag_events` could not contradict it, being `leader_seqno - applied` on a gauge +/// that reads 0 until the frontier is known (`0 - 0 = 0`). A blind node therefore +/// looked converged. Convergence is now required for EVERY boot against a KNOWN +/// frontier, and only ESTABLISHED (election-runtime) leadership self-certifies — +/// never the topology's term-0 belief, which would let a booting node certify the +/// group it merely thinks it leads. /// -/// It is `#[ignore]`d so the nightly chaos gate keeps its signal rather than going -/// permanently red on a known-open defect. Remove the attribute as the fix's gate: +/// Post-fix run, all three groups converged against real frontiers: +/// +/// ```text +/// shard 0: applied 3797, leader_seqno 3797, term 1 +/// shard 1: applied 3726, leader_seqno 3726, term 5 +/// shard 2: applied 3747, leader_seqno 3747, term 3 +/// [multi] every probed item is readable ... (2 reboot(s) total) +/// ``` +/// +/// Runs long (~15 min) and wants widened tier-3 budgets: /// /// ```bash /// TIDAL_TEST_BOOT_BUDGET_SECS=300 TIDAL_TEST_CONVERGENCE_BUDGET_SECS=180 \ @@ -654,9 +665,6 @@ fn mp_follower_reseeds_via_snapshot_after_compaction() { /// mp_multi_group_node_converges_after_reseeding_several_groups -- --nocapture /// ``` #[test] -#[ignore = "reproduces an OPEN multi-group reseed defect: the node reports \ - reseed_required=false and lag_events=0 while missing items. Un-ignore as the \ - fix's gate; see the doc comment for the invocation."] fn mp_multi_group_node_converges_after_reseeding_several_groups() { const GROUPS: usize = 3; /// Generous but FINITE. One restart per group that needs a reseed is the