fix(cluster): the frontier pair has one writer, and 0 is not a commit index

`TidalDBClusterQuorumLag` sat CRITICAL all session against the live three-voter
cluster while every region reported lag 0 and every per-peer ship queue was
empty. Two independent defects fed it:

- `observe_ship` bumped `relay_last_seq` on every batch ship while
  `relay_durable_seq` only moved when a signal write completed. The two are
  documented as a subtractable pair, so a shipping-but-not-committing node
  reported the whole relay log (13.3M events) as quorum lag. The ship path now
  feeds only its own per-peer queue-depth gauge; the pair has one writer.
- `set_frontier_gauges` published `CommitIndex::committed()` verbatim, but that
  returns 0 as a SENTINEL for "no quorum information in this term yet". It now
  publishes both halves or neither, and every satisfied `await_quorum` -- not
  just signal writes -- refreshes them, so item and embedding workloads keep
  the pair live.

Regression test asserts a busy ship loop leaves both halves at 0 (lag 0, not
13.3M) and that the single writer still moves them together.

Also excludes the `tmp` emptyDir from velero fs-backup: three 0-byte
PodVolumeBackups a night whose only other outcome is failing the whole fleet
backup when a scratch file vanishes mid-snapshot.

k8s/cluster/statefulset.yaml

@@ -50,6 +50,15 @@ spec:
         prometheus.io/scrape: "true"
         prometheus.io/port: "9091"
         prometheus.io/path: "/metrics"
+        # `tmp` is an emptyDir. velero-fleet-daily runs with
+        # defaultVolumesToFsBackup, so it snapshotted this scratch mount on
+        # every pod: three 0-byte PodVolumeBackups a night, and a live failure
+        # candidate — a file that vanishes mid-snapshot fails the PVB, which
+        # marks the WHOLE fleet backup PartiallyFailed and freezes
+        # velero_backup_last_successful_timestamp. The `data` volume (the
+        # corpus) is what must be captured; scratch never is. Fleet convention:
+        # deployments/k8s/base/databases/backups.yaml.
+        backup.velero.io/backup-volumes-excludes: tmp
     spec:
       # SIGTERM flips readiness to 503 (pod leaves the client Service), drains
       # in-flight requests, then checkpoints + fsyncs the WAL AND saves every
@@ -334,8 +343,15 @@ spec:
         name: data
         labels:
           app.kubernetes.io/name: tidaldb
-          backup.orchard9.ai/class: expendable
-          backup.orchard9.ai/method: tidal-stress-reseed
+          # PRODUCTION data as of 2026-08-18: these three volumes hold the
+          # serving corpus, so they are backed up, not reseeded. They carried
+          # `expendable / tidal-stress-reseed` while this cluster existed only
+          # to run the nightly soak - a label that authorised throwing the
+          # store away. `volumeClaimTemplates` is immutable on a live
+          # StatefulSet, so the running PVCs were relabelled in place with
+          # `kubectl label`; this block is what a fresh install gets.
+          backup.orchard9.ai/class: protected
+          backup.orchard9.ai/method: velero-kopia
       spec:
         accessModes: ["ReadWriteOnce"]
         storageClassName: local-path

tidal-server/src/cluster/node.rs

@@ -1803,12 +1803,23 @@ impl ShardReplica {
     /// frontier, `relay_durable_seq` = the quorum commit index (m11p3 — the
     /// gap between them is the cluster's quorum lag). With zero peers the
     /// leader alone is the majority, so the gauges coincide.
+    ///
+    /// Publishes BOTH halves or neither. `CommitIndex::committed` returns 0
+    /// as a sentinel for "no quorum information yet", not as a seqno, so a
+    /// replicated node that has not yet satisfied a quorum wait in this term
+    /// has no commit index to report. Publishing the sentinel against a live
+    /// flushed frontier claimed the entire relay log as lag: on the fleet's
+    /// three-voter cluster that read as 13.5M events behind and held
+    /// `TidalDBClusterQuorumLag` critical while every region sat at lag 0.
     fn set_frontier_gauges(&self) {
         let flushed = self.ship_feed.flushed_seq();
         let commit = if self.commit.needed_peers() == 0 {
             flushed
         } else {
-            self.commit.committed()
+            match self.commit.committed() {
+                0 => return,
+                commit => commit,
+            }
         };
         self.cluster_metrics.set_relay_frontiers(flushed, commit);
     }
@@ -6793,6 +6804,7 @@ async fn await_quorum(
         if !active {
             return Err(ClusterAppError(state.not_leader()));
         }
+        state.set_frontier_gauges();
         return Ok(seq);
     }
     let mut watch = state.commit_watch.clone();
@@ -6811,6 +6823,12 @@ async fn await_quorum(
                 return Err(ClusterAppError(state.not_leader()));
             }
             if commit >= seq {
+                // A satisfied wait is the one moment both halves of the
+                // frontier pair are known: publish them together. Signal
+                // writes are not the only writers, so leaving this to
+                // `complete_signal_write` left the pair stale on an
+                // item-or-embedding workload.
+                state.set_frontier_gauges();
                 return Ok(commit);
             }
         }

tidal/src/db/metrics/cluster.rs

@@ -79,12 +79,26 @@ pub struct ClusterMetrics {
     write_pool_depth: AtomicU64,
     /// Total write-pool submissions rejected with backpressure (HTTP 429).
     write_pool_rejections_total: AtomicU64,
-    /// The relay's last committed seqno (leader stream high-water mark).
+    /// The relay's flushed high-water mark (leader stream frontier).
+    ///
+    /// # Invariant
+    ///
+    /// This gauge and [`relay_durable_seq`] are a SUBTRACTABLE PAIR and have
+    /// exactly ONE writer: [`set_relay_frontiers`]. Nothing else may move
+    /// either half. A second writer that bumps only the frontier (the ship
+    /// loop used to) leaves the other half at its initial 0, and the
+    /// documented quorum lag then reads as the entire relay log — a
+    /// permanent false critical on a perfectly healthy cluster.
+    ///
+    /// [`relay_durable_seq`]: Self::relay_durable_seq
+    /// [`set_relay_frontiers`]: Self::set_relay_frontiers
     relay_last_seq: AtomicU64,
     /// The quorum commit index (m11p3) — the highest seqno a majority of the
     /// replica set durably holds. Pre-m11p3 this gauge carried the leader's
     /// own durable frontier; the name is kept for dashboard continuity, and
     /// `relay_last_seq - relay_durable_seq` is now the cluster's quorum lag.
+    /// Paired with [`relay_last_seq`](Self::relay_last_seq) — see its
+    /// invariant before adding a writer.
     relay_durable_seq: AtomicU64,
     /// Total `ack=quorum` writes that timed out awaiting the commit index
     /// (each returned a retryable 503 naming the laggards).
@@ -375,7 +389,10 @@ impl ClusterMetrics {
         let rtt_us = rtt.as_micros() as u64;
         self.ship_rtt.observe(rtt_us);
         self.ship_batch_events.observe(events);
-        self.relay_last_seq.store(last_seq, Ordering::Relaxed);
+        // `last_seq` feeds this peer's queue depth ONLY. It must not touch
+        // `relay_last_seq`: that gauge is half of a subtractable pair whose
+        // other half moves only on a satisfied quorum wait, so bumping it
+        // here reported the whole relay log as quorum lag.
         let cell = self.peer(peer);
         cell.rtt.observe(rtt_us);
         cell.batches_total.fetch_add(1, Ordering::Relaxed);
@@ -839,4 +856,42 @@ mod tests {
         // The owner's unlabeled form must NOT appear in a sibling render.
         assert!(!out.contains("tidaldb_cluster_relay_last_seq 40"));
     }
+
+    /// Regression, measured on the live three-voter fleet cluster: shipping
+    /// bumped `relay_last_seq` while `relay_durable_seq` sat at its initial
+    /// 0, so `last - durable` read as the whole relay log and pinned
+    /// `TidalDBClusterQuorumLag` critical with every region at lag 0. The
+    /// pair has ONE writer; a busy ship loop must leave it alone.
+    #[test]
+    fn shipping_never_moves_the_frontier_pair() {
+        let m = ClusterMetrics::new();
+        m.mark_active();
+        for _ in 0..3 {
+            m.observe_ship(
+                ShardId(1),
+                std::time::Duration::from_millis(1),
+                8,
+                6,
+                13_324_712,
+            );
+        }
+
+        let mut out = String::new();
+        m.render_into(&mut out, 1);
+        // The ship's frontier reached the PEER gauge…
+        assert!(out.contains(
+            "tidaldb_cluster_peer_ship_queue_depth{peer_shard=\"1\",partition_id=\"1\"} 13324706"
+        ));
+        // …and nothing else. Both halves of the pair stay at 0, so the
+        // derived lag is 0 — not 13.3M.
+        assert!(out.contains("tidaldb_cluster_relay_last_seq 0"));
+        assert!(out.contains("tidaldb_cluster_relay_durable_seq 0"));
+
+        // The single writer moves them together.
+        m.set_relay_frontiers(13_324_712, 13_324_712);
+        let mut after = String::new();
+        m.render_into(&mut after, 1);
+        assert!(after.contains("tidaldb_cluster_relay_last_seq 13324712"));
+        assert!(after.contains("tidaldb_cluster_relay_durable_seq 13324712"));
+    }
 }