From fc1cc901a17fa8a9ec7d0133f50bae997d45a1a2 Mon Sep 17 00:00:00 2001 From: jordan Date: Tue, 18 Aug 2026 09:57:32 -0600 Subject: [PATCH] fix(cluster): refresh the frontier gauges on the driver tick Publishing the pair only from a satisfied `ack=quorum` wait left both halves at 0 for the entire life of an `ack=leader` workload: the live three-voter cluster served writes at 201/204 and reported `commit_index: 13324714` through `/cluster/status` while `/metrics` showed relay_last_seq 0, relay_durable_seq 0. A gauge nobody can populate is the same blind spot as a gauge that lies. The election driver already ticks every replica ~20x/s and m12p5 seeds the readiness lag gauge from the heartbeat for exactly this reason - the tick flows even when writes do not. Two relaxed stores per tick behind a commit-index lock the quorum waiters already share. --- tidal-server/src/cluster/election_driver.rs | 10 ++++++++++ tidal-server/src/cluster/node.rs | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/tidal-server/src/cluster/election_driver.rs b/tidal-server/src/cluster/election_driver.rs index 44e9961..e49ea4b 100644 --- a/tidal-server/src/cluster/election_driver.rs +++ b/tidal-server/src/cluster/election_driver.rs @@ -904,6 +904,16 @@ pub fn start( // bounded grace elapses. A cheap no-op when nothing is pending // (only the leader arms a grace, on a remove verb). node.poll_deferred_retires(); + // Refresh the frontier pair (flushed high-water mark + + // quorum commit index) on the tick, for the same reason + // m12p5 seeds the lag gauge from the heartbeat: the tick + // flows even when no writes do. Publishing only on a + // satisfied `ack=quorum` wait left both gauges at 0 for + // the whole life of an `ack=leader` workload while + // `/cluster/status` reported a live commit index. Two + // relaxed stores, throttled by nothing — cheaper than the + // breaker refresh above. + node.set_frontier_gauges(); } } tracing::info!("election driver stopped"); diff --git a/tidal-server/src/cluster/node.rs b/tidal-server/src/cluster/node.rs index 086e69e..cff76df 100644 --- a/tidal-server/src/cluster/node.rs +++ b/tidal-server/src/cluster/node.rs @@ -1811,7 +1811,7 @@ impl ShardReplica { /// flushed frontier claimed the entire relay log as lag: on the fleet's /// three-voter cluster that read as 13.5M events behind and held /// `TidalDBClusterQuorumLag` critical while every region sat at lag 0. - fn set_frontier_gauges(&self) { + pub(crate) fn set_frontier_gauges(&self) { let flushed = self.ship_feed.flushed_seq(); let commit = if self.commit.needed_peers() == 0 { flushed