diff --git a/k8s/cluster/statefulset.yaml b/k8s/cluster/statefulset.yaml index 5d56a30..25a851d 100644 --- a/k8s/cluster/statefulset.yaml +++ b/k8s/cluster/statefulset.yaml @@ -50,6 +50,15 @@ spec: prometheus.io/scrape: "true" prometheus.io/port: "9091" prometheus.io/path: "/metrics" + # `tmp` is an emptyDir. velero-fleet-daily runs with + # defaultVolumesToFsBackup, so it snapshotted this scratch mount on + # every pod: three 0-byte PodVolumeBackups a night, and a live failure + # candidate — a file that vanishes mid-snapshot fails the PVB, which + # marks the WHOLE fleet backup PartiallyFailed and freezes + # velero_backup_last_successful_timestamp. The `data` volume (the + # corpus) is what must be captured; scratch never is. Fleet convention: + # deployments/k8s/base/databases/backups.yaml. + backup.velero.io/backup-volumes-excludes: tmp spec: # SIGTERM flips readiness to 503 (pod leaves the client Service), drains # in-flight requests, then checkpoints + fsyncs the WAL AND saves every @@ -334,8 +343,15 @@ spec: name: data labels: app.kubernetes.io/name: tidaldb - backup.orchard9.ai/class: expendable - backup.orchard9.ai/method: tidal-stress-reseed + # PRODUCTION data as of 2026-08-18: these three volumes hold the + # serving corpus, so they are backed up, not reseeded. They carried + # `expendable / tidal-stress-reseed` while this cluster existed only + # to run the nightly soak - a label that authorised throwing the + # store away. `volumeClaimTemplates` is immutable on a live + # StatefulSet, so the running PVCs were relabelled in place with + # `kubectl label`; this block is what a fresh install gets. + backup.orchard9.ai/class: protected + backup.orchard9.ai/method: velero-kopia spec: accessModes: ["ReadWriteOnce"] storageClassName: local-path diff --git a/tidal-server/src/cluster/node.rs b/tidal-server/src/cluster/node.rs index eefb33e..086e69e 100644 --- a/tidal-server/src/cluster/node.rs +++ b/tidal-server/src/cluster/node.rs @@ -1803,12 +1803,23 @@ impl ShardReplica { /// frontier, `relay_durable_seq` = the quorum commit index (m11p3 — the /// gap between them is the cluster's quorum lag). With zero peers the /// leader alone is the majority, so the gauges coincide. + /// + /// Publishes BOTH halves or neither. `CommitIndex::committed` returns 0 + /// as a sentinel for "no quorum information yet", not as a seqno, so a + /// replicated node that has not yet satisfied a quorum wait in this term + /// has no commit index to report. Publishing the sentinel against a live + /// flushed frontier claimed the entire relay log as lag: on the fleet's + /// three-voter cluster that read as 13.5M events behind and held + /// `TidalDBClusterQuorumLag` critical while every region sat at lag 0. fn set_frontier_gauges(&self) { let flushed = self.ship_feed.flushed_seq(); let commit = if self.commit.needed_peers() == 0 { flushed } else { - self.commit.committed() + match self.commit.committed() { + 0 => return, + commit => commit, + } }; self.cluster_metrics.set_relay_frontiers(flushed, commit); } @@ -6793,6 +6804,7 @@ async fn await_quorum( if !active { return Err(ClusterAppError(state.not_leader())); } + state.set_frontier_gauges(); return Ok(seq); } let mut watch = state.commit_watch.clone(); @@ -6811,6 +6823,12 @@ async fn await_quorum( return Err(ClusterAppError(state.not_leader())); } if commit >= seq { + // A satisfied wait is the one moment both halves of the + // frontier pair are known: publish them together. Signal + // writes are not the only writers, so leaving this to + // `complete_signal_write` left the pair stale on an + // item-or-embedding workload. + state.set_frontier_gauges(); return Ok(commit); } } diff --git a/tidal/src/db/metrics/cluster.rs b/tidal/src/db/metrics/cluster.rs index 1fb1223..f754d2c 100644 --- a/tidal/src/db/metrics/cluster.rs +++ b/tidal/src/db/metrics/cluster.rs @@ -79,12 +79,26 @@ pub struct ClusterMetrics { write_pool_depth: AtomicU64, /// Total write-pool submissions rejected with backpressure (HTTP 429). write_pool_rejections_total: AtomicU64, - /// The relay's last committed seqno (leader stream high-water mark). + /// The relay's flushed high-water mark (leader stream frontier). + /// + /// # Invariant + /// + /// This gauge and [`relay_durable_seq`] are a SUBTRACTABLE PAIR and have + /// exactly ONE writer: [`set_relay_frontiers`]. Nothing else may move + /// either half. A second writer that bumps only the frontier (the ship + /// loop used to) leaves the other half at its initial 0, and the + /// documented quorum lag then reads as the entire relay log — a + /// permanent false critical on a perfectly healthy cluster. + /// + /// [`relay_durable_seq`]: Self::relay_durable_seq + /// [`set_relay_frontiers`]: Self::set_relay_frontiers relay_last_seq: AtomicU64, /// The quorum commit index (m11p3) — the highest seqno a majority of the /// replica set durably holds. Pre-m11p3 this gauge carried the leader's /// own durable frontier; the name is kept for dashboard continuity, and /// `relay_last_seq - relay_durable_seq` is now the cluster's quorum lag. + /// Paired with [`relay_last_seq`](Self::relay_last_seq) — see its + /// invariant before adding a writer. relay_durable_seq: AtomicU64, /// Total `ack=quorum` writes that timed out awaiting the commit index /// (each returned a retryable 503 naming the laggards). @@ -375,7 +389,10 @@ impl ClusterMetrics { let rtt_us = rtt.as_micros() as u64; self.ship_rtt.observe(rtt_us); self.ship_batch_events.observe(events); - self.relay_last_seq.store(last_seq, Ordering::Relaxed); + // `last_seq` feeds this peer's queue depth ONLY. It must not touch + // `relay_last_seq`: that gauge is half of a subtractable pair whose + // other half moves only on a satisfied quorum wait, so bumping it + // here reported the whole relay log as quorum lag. let cell = self.peer(peer); cell.rtt.observe(rtt_us); cell.batches_total.fetch_add(1, Ordering::Relaxed); @@ -839,4 +856,42 @@ mod tests { // The owner's unlabeled form must NOT appear in a sibling render. assert!(!out.contains("tidaldb_cluster_relay_last_seq 40")); } + + /// Regression, measured on the live three-voter fleet cluster: shipping + /// bumped `relay_last_seq` while `relay_durable_seq` sat at its initial + /// 0, so `last - durable` read as the whole relay log and pinned + /// `TidalDBClusterQuorumLag` critical with every region at lag 0. The + /// pair has ONE writer; a busy ship loop must leave it alone. + #[test] + fn shipping_never_moves_the_frontier_pair() { + let m = ClusterMetrics::new(); + m.mark_active(); + for _ in 0..3 { + m.observe_ship( + ShardId(1), + std::time::Duration::from_millis(1), + 8, + 6, + 13_324_712, + ); + } + + let mut out = String::new(); + m.render_into(&mut out, 1); + // The ship's frontier reached the PEER gauge… + assert!(out.contains( + "tidaldb_cluster_peer_ship_queue_depth{peer_shard=\"1\",partition_id=\"1\"} 13324706" + )); + // …and nothing else. Both halves of the pair stay at 0, so the + // derived lag is 0 — not 13.3M. + assert!(out.contains("tidaldb_cluster_relay_last_seq 0")); + assert!(out.contains("tidaldb_cluster_relay_durable_seq 0")); + + // The single writer moves them together. + m.set_relay_frontiers(13_324_712, 13_324_712); + let mut after = String::new(); + m.render_into(&mut after, 1); + assert!(after.contains("tidaldb_cluster_relay_last_seq 13324712")); + assert!(after.contains("tidaldb_cluster_relay_durable_seq 13324712")); + } }