diff --git a/tidal-server/src/cluster/node.rs b/tidal-server/src/cluster/node.rs index 0db2216..e10f68e 100644 --- a/tidal-server/src/cluster/node.rs +++ b/tidal-server/src/cluster/node.rs @@ -3165,6 +3165,26 @@ impl ShardReplica { return; } self.self_restart_refused.store(false, Ordering::Release); + // m12 reseed-loop-fix: re-check the marker before committing to the exit. + // A snapshot_required latch from a TRANSIENT classification — the leader's + // baseline advanced past this node's persisted frontier while it was down, + // so the first heartbeat's decide_join saw it briefly behind — self-heals + // via the stream: `clear_stale_reseed_marker_if_caught_up` clears the marker + // once caught up, journaling the durable term marker on the clean join. The + // quorum poll above took time (a blocking peer fan-out); if the marker + // healed meanwhile, a reboot would reseed NOTHING (the leader answers + // needed=false for a caught-up shard), so self-restarting is futile and + // loops. Abort. Only a marker that CANNOT self-heal (a genuine compacted + // gap, still latched here) proceeds to the restart. + if !self.reseed_marker_latched.load(Ordering::Acquire) { + tracing::info!( + region = %self.region_name, + "reseed_self_restart: the reseed marker healed via stream catch-up during the \ + quorum check — the shard is caught up, a reboot would reseed nothing. Aborting \ + the self-restart (m12 reseed-loop-fix)." + ); + return; + } // m12 reseed-loop-fix (Fix 3): the process-wide exit is owned by the // node-level coordinator, not this single shard. It fires once, only // after every hosted shard has also requested a restart or a bounded @@ -3199,6 +3219,21 @@ impl ShardReplica { if deadline > now { std::thread::sleep(deadline - now); } + // m12 reseed-loop-fix: the grace window is exactly when a + // transient snapshot_required latch self-heals via stream + // catch-up. Re-check before firing — if THIS shard's marker + // healed, a reboot would reseed nothing (caught up), so abort + // the futile self-restart. Only a still-latched marker (a + // genuine compacted gap) fires the exit. + if !node.reseed_marker_latched.load(Ordering::Acquire) { + tracing::info!( + region = %node.region_name, + "reseed_self_restart: the reseed marker healed via stream \ + catch-up during the Fix 3 grace — aborting the self-restart \ + (the shard is caught up; a reboot would reseed nothing)." + ); + return; + } if super::reseed_restart::fire_due() { node.fire_graceful_self_restart(); }