diff --git a/tidal-server/src/cluster/node.rs b/tidal-server/src/cluster/node.rs index a991209..16363e2 100644 --- a/tidal-server/src/cluster/node.rs +++ b/tidal-server/src/cluster/node.rs @@ -3857,6 +3857,16 @@ impl ShardReplica { election_frontier: election_pos.frontier, leader_acked: self.leader_acked_frontier(), leader_seqno, + // Every tracked stream key, not just the current leader's. See + // `ReplicationState::applied_by_key` — a key retained from a PREVIOUS + // leadership is what the receiver's gap check can keep chasing, and it + // was invisible in every other field. + applied_by_key: db + .replication_state() + .applied_by_key() + .into_iter() + .map(|(k, v)| (u32::from(k.0), v)) + .collect(), reseed_required, reseeding, self_restart_refused: self.self_restart_refused.load(Ordering::Acquire), @@ -5223,6 +5233,19 @@ pub struct LocalStatusResponse { /// `leader_seqno: 0` means NO INFORMATION, not converged. #[serde(default)] leader_seqno: u64, + /// Every tracked replication stream key with its applied frontier, as + /// `(key, applied)` pairs ascending by key. + /// + /// A key is a per-LEADER-REGION stream id (`shard_of_region`), not a shard + /// group, so a group accumulates one key per leadership it has followed. Every + /// other field reports only the CURRENT leader's key, which made a position + /// retained from a previous leadership unobservable — while the receiver's gap + /// check (`receiver.rs`: `request_catchup(key, applied + 1)`) will chase ANY + /// key that received data. In the 2026-08-20 incident a node pulled + /// `from_seqno=13540653` (a key at 13540652) while the group it reported on sat + /// at 13540661, and nothing in the status could say which key that was. + #[serde(default)] + applied_by_key: Vec<(u32, u64)>, /// Whether this node is quarantined with a divergent suffix (m11p4): /// fenced from the data plane until reseeded. quarantined: bool, diff --git a/tidal/src/replication/state.rs b/tidal/src/replication/state.rs index a4b80a9..ad2b932 100644 --- a/tidal/src/replication/state.rs +++ b/tidal/src/replication/state.rs @@ -136,6 +136,29 @@ impl ReplicationState { .map(|a| a.load(Ordering::Acquire)) } + /// Every tracked stream key with its contiguous applied frontier, ascending by + /// key. + /// + /// A key is a per-LEADER-REGION stream id (`shard_of_region`), NOT a shard + /// group, so one group accumulates a key per leadership it has followed. Only + /// the CURRENT leader's key appears in the per-group status row, which left a + /// position tracked under a PREVIOUS leadership completely unobservable — and + /// the receiver's gap check issues `request_catchup(key, applied + 1)` for any + /// key that received data, current or not. During the 2026-08-20 incident a + /// node kept pulling `from_seqno=13540653` (some key at 13540652) while the + /// group it reported on sat at 13540661, and there was no way to see which key + /// was responsible. This is that instrument. + #[must_use] + pub fn applied_by_key(&self) -> Vec<(ShardId, u64)> { + let mut out: Vec<(ShardId, u64)> = self + .applied + .iter() + .map(|(k, v)| (*k, v.load(Ordering::Acquire))) + .collect(); + out.sort_unstable_by_key(|(k, _)| k.0); + out + } + /// Whether the batch range `[first, last]` has already been applied for /// `shard_id` — its whole span is at or below the contiguous frontier, or it /// is recorded verbatim in the `ahead` buffer as an out-of-order apply. The