From 973f07355857257c47be040c6a7216fe2d8d5381 Mon Sep 17 00:00:00 2001 From: jx12n Date: Thu, 18 Jun 2026 21:34:47 -0600 Subject: [PATCH] fix(m12): abort the futile self-restart when a snapshot_required marker self-heals via catch-up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rc4 live deploy converged tidaldb-2 but through ~4 needless self-restarts: a caught-up shard whose persisted frontier is briefly behind the leader's ADVANCED baseline (the leader kept writing while the node was down) latches a snapshot_required marker on the first heartbeat's decide_join, which ARMS a self-restart. The shard then catches up via the stream — note_term_joined journals the durable term marker and clear_stale_reseed_marker_if_caught_up clears the marker — but the already-armed self-restart still fires (Fix 3 defers it 5s, then exits). The reboot reseeds NOTHING (the leader answers needed=false for a caught-up shard), so it is futile and flaps readiness. Fix: gate the self-restart on the marker still being LATCHED at the fire point — re-check after the (slow) quorum poll and again in the Fix 3 deferred timer (where the catch-up actually completes within the grace). A marker that healed via stream catch-up aborts the restart; only a marker that CANNOT self-heal (a genuine compacted gap, still latched) proceeds to reseed at the next boot. This converges a caught-up shard IN PLACE (no reboot), while preserving the real reseed for a genuinely-behind shard. Verified: tidal-server clippy -D warnings clean; the cluster_reseed e2e (quarantine reseed still fires, rolling restart still 0-reseed) and the live rollout confirm the genuine-reseed path is unaffected. --- tidal-server/src/cluster/node.rs | 35 ++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tidal-server/src/cluster/node.rs b/tidal-server/src/cluster/node.rs index 0db2216..e10f68e 100644 --- a/tidal-server/src/cluster/node.rs +++ b/tidal-server/src/cluster/node.rs @@ -3165,6 +3165,26 @@ impl ShardReplica { return; } self.self_restart_refused.store(false, Ordering::Release); + // m12 reseed-loop-fix: re-check the marker before committing to the exit. + // A snapshot_required latch from a TRANSIENT classification — the leader's + // baseline advanced past this node's persisted frontier while it was down, + // so the first heartbeat's decide_join saw it briefly behind — self-heals + // via the stream: `clear_stale_reseed_marker_if_caught_up` clears the marker + // once caught up, journaling the durable term marker on the clean join. The + // quorum poll above took time (a blocking peer fan-out); if the marker + // healed meanwhile, a reboot would reseed NOTHING (the leader answers + // needed=false for a caught-up shard), so self-restarting is futile and + // loops. Abort. Only a marker that CANNOT self-heal (a genuine compacted + // gap, still latched here) proceeds to the restart. + if !self.reseed_marker_latched.load(Ordering::Acquire) { + tracing::info!( + region = %self.region_name, + "reseed_self_restart: the reseed marker healed via stream catch-up during the \ + quorum check — the shard is caught up, a reboot would reseed nothing. Aborting \ + the self-restart (m12 reseed-loop-fix)." + ); + return; + } // m12 reseed-loop-fix (Fix 3): the process-wide exit is owned by the // node-level coordinator, not this single shard. It fires once, only // after every hosted shard has also requested a restart or a bounded @@ -3199,6 +3219,21 @@ impl ShardReplica { if deadline > now { std::thread::sleep(deadline - now); } + // m12 reseed-loop-fix: the grace window is exactly when a + // transient snapshot_required latch self-heals via stream + // catch-up. Re-check before firing — if THIS shard's marker + // healed, a reboot would reseed nothing (caught up), so abort + // the futile self-restart. Only a still-latched marker (a + // genuine compacted gap) fires the exit. + if !node.reseed_marker_latched.load(Ordering::Acquire) { + tracing::info!( + region = %node.region_name, + "reseed_self_restart: the reseed marker healed via stream \ + catch-up during the Fix 3 grace — aborting the self-restart \ + (the shard is caught up; a reboot would reseed nothing)." + ); + return; + } if super::reseed_restart::fire_due() { node.fire_graceful_self_restart(); }