diff --git a/k8s/cluster/statefulset.yaml b/k8s/cluster/statefulset.yaml index 582b817..1ca78b8 100644 --- a/k8s/cluster/statefulset.yaml +++ b/k8s/cluster/statefulset.yaml @@ -236,18 +236,40 @@ spec: failureThreshold: 3 resources: requests: - # Measured Ref-A soak envelope: the busiest leader sustained - # ~1.7 cores and reached ~2.5, so 2 cores is the honest scheduler - # reservation. A fleet that cannot place this request cannot run - # the 200 rps gate without request-level contention. - cpu: "2" - # Baseline working set was ~3.6 GiB before load. Reserving 4 GiB - # prevents the scheduler from hiding that resident footprint. + # CONSUMER-SIZED, not gate-sized (2026-08-18). + # + # The 2-core reservation this replaced came from the Ref-A soak + # envelope: the busiest leader sustained ~1.7 cores and peaked at + # ~2.5 under the 200 rps write-heavy mix. That is the ENDURANCE + # GATE's price, and three voters at 2 cores need 6,000m plus 2,000m + # free on each of the three nodes their local-path volumes are + # pinned to. The live fleet is 82-91% committed on requests, so that + # contract cannot be placed and the cluster stayed parked for it. + # + # 300m is what the tightest pinned node (k3s-agent-1, 355m free) + # can actually reserve for a voter, and it is honest for a FIRST + # CONSUMER's load - not for the gate. The limit below keeps the + # measured burst reachable without reserving it. + # + # This is provisioned optimism with named detectors: if real write + # volume approaches the knee, TidalDBClusterQuorumLag, + # TidalDBClusterCommitIndexStall, TidalDBClusterWritePoolShedding + # and TidalDBClusterQuorumTimeouts fire before users see it. Raise + # the request (or add shards - writes hash-route across shard + # groups) rather than waiting for a stall. + cpu: "300m" + # Baseline working set was ~3.6 GiB before load, and full placement + # means every pod holds the WHOLE 1536-D corpus. This is a resident + # footprint, not a gate artifact: it stays at 4 GiB. memory: 4Gi limits: - # Three cores preserves one core for kubelet/system on the Ref-A - # four-core node while allowing the measured query burst. - cpu: "3" + # Two cores keeps the measured query/apply burst reachable on the + # tightest node without reserving it. Note the ratio: a burstable + # pod whose neighbours are also bursting gets CFS-throttled, which + # is precisely how CockroachDB was pushed into multi-second Raft + # stalls on this fleet with nodes 70% idle. The quorum alerts above + # are the detector for that; more request is the fix. + cpu: "2" # Four independent OOMKills occurred at 3.97-4.00 GiB. Six GiB is # measured peak plus 50% recovery/profiling headroom; the exact # internal growth source still requires heap/allocation profiling. diff --git a/tidal-server/src/scatter_gather.rs b/tidal-server/src/scatter_gather.rs index 466c8bf..744b57b 100644 --- a/tidal-server/src/scatter_gather.rs +++ b/tidal-server/src/scatter_gather.rs @@ -105,9 +105,27 @@ impl ShardCoordinator for SimCoordinator<'_> { } } -/// Default query deadline (50ms as per spec Section 7.4). +/// Default query deadline for an in-process / plaintext fan-out (50ms as per +/// spec Section 7.4). const DEFAULT_DEADLINE_MS: u64 = 50; +/// Default query deadline when the fan-out crosses an inter-node TLS hop. +/// +/// The 50ms spec figure budgets for a shard read, not for establishing a +/// connection to another pod. Measured on the live three-node cluster with +/// m11p7 inter-node HTTPS: a COLD peer fetch (TCP + TLS handshake + remote +/// 1536-D search) takes ~50ms, a warm one ~1ms. With `DEFAULT_DEADLINE_MS` +/// the per-shard budget is 45ms, so the very first cross-shard query after a +/// rollout, an idle period, or a pod restart timed out on EVERY peer and the +/// query returned `degraded: true` with HTTP 200 and PARTIAL results - a +/// ranking store quietly answering from one third of its corpus. +/// +/// 250ms covers the cold handshake with margin while staying an order of +/// magnitude below [`MAX_DEADLINE_MS`]. A latency-sensitive caller still pins +/// its own budget with `?deadline_ms=`; this is only the default for callers +/// that state none. +const TLS_DEFAULT_DEADLINE_MS: u64 = 250; + /// Estimated network overhead per shard hop (subtracted from deadline). const NETWORK_OVERHEAD_MS: u64 = 5; @@ -123,14 +141,27 @@ const NETWORK_OVERHEAD_MS: u64 = 5; /// capping the worst case. See [`clamp_deadline_ms`]. const MAX_DEADLINE_MS: u64 = 10_000; +/// The default per-query budget for the transport this process fans out over. +/// +/// Split out from [`clamp_deadline_ms`] so the choice is unit-testable without +/// mutating the process-global inter-node scheme flag other tests read. +const fn default_deadline_ms(inter_node_tls: bool) -> u64 { + if inter_node_tls { + TLS_DEFAULT_DEADLINE_MS + } else { + DEFAULT_DEADLINE_MS + } +} + /// Clamp a client-supplied scatter-gather deadline to [`MAX_DEADLINE_MS`]. /// -/// `None` keeps the [`DEFAULT_DEADLINE_MS`] default. Any explicit value above -/// the ceiling is logged once and reduced, so a malicious or buggy client -/// cannot hold a worker indefinitely. +/// `None` takes the transport-aware default: [`DEFAULT_DEADLINE_MS`] in-process, +/// [`TLS_DEFAULT_DEADLINE_MS`] once inter-node TLS is configured. Any explicit +/// value above the ceiling is logged once and reduced, so a malicious or buggy +/// client cannot hold a worker indefinitely. fn clamp_deadline_ms(requested: Option) -> u64 { match requested { - None => DEFAULT_DEADLINE_MS, + None => default_deadline_ms(crate::cluster::forward::inter_node_https()), Some(ms) if ms > MAX_DEADLINE_MS => { tracing::warn!( requested_ms = ms, @@ -1353,6 +1384,22 @@ fn urlencode(s: &str) -> String { out } +/// Count a degraded read fan-out on the engine's cluster metrics. +/// +/// A degraded fan-out answers `200 OK` with results assembled from a SUBSET of +/// the shards, so nothing in the HTTP status, the error path, or the latency +/// histograms records it. Before this counter existed the only trace was a +/// per-request WARN in one pod's log - a feed ranking over a third of its +/// corpus looked identical to a healthy one from the outside. No-op on the +/// healthy path. +fn record_degraded_fanout(db: &Arc, meta: &ScatterGatherMeta) { + if !meta.degraded { + return; + } + db.cluster_metrics() + .incr_scatter_degraded(meta.unavailable_shards.len() as u64); +} + /// Multi-process scatter-gather RETRIEVE across every region from the gateway. /// /// Local region served locally, remote regions over HTTP. Preserves the @@ -1434,6 +1481,7 @@ pub fn scatter_gather_retrieve_http( elapsed_ms: elapsed.as_millis() as u64, shard_deadline_ms, }; + record_degraded_fanout(&ctx.db, &meta); let results = RetrieveResults { items: all_items, next_cursor: None, @@ -1539,6 +1587,7 @@ pub fn scatter_gather_search_http( elapsed_ms: elapsed.as_millis() as u64, shard_deadline_ms, }; + record_degraded_fanout(&ctx.db, &meta); let results = SearchResults { items: all_items, next_cursor: None, @@ -1611,6 +1660,40 @@ mod tests { builder.build().unwrap() } + /// The default read budget must cover the transport it actually crosses. + /// + /// Regression, measured on the live three-node cluster: with inter-node + /// HTTPS the 50ms in-process default left a 45ms per-shard budget, a COLD + /// peer fetch (TCP + TLS handshake + remote 1536-D search) took ~50ms, and + /// every cross-shard read came back `degraded: true` with HTTP 200 and + /// results from ONE shard. A ranking store cannot answer from a third of its + /// corpus and call it success. + #[test] + fn default_read_budget_covers_a_cold_inter_node_tls_hop() { + assert_eq!( + default_deadline_ms(false), + DEFAULT_DEADLINE_MS, + "in-process fan-out keeps the spec §7.4 budget" + ); + assert_eq!( + default_deadline_ms(true), + TLS_DEFAULT_DEADLINE_MS, + "an inter-node TLS fan-out gets the handshake-aware budget" + ); + let measured_cold_hop_ms = 50; + assert!( + TLS_DEFAULT_DEADLINE_MS.saturating_sub(NETWORK_OVERHEAD_MS) > measured_cold_hop_ms, + "per-shard budget {} must exceed the measured cold hop {measured_cold_hop_ms}ms", + TLS_DEFAULT_DEADLINE_MS - NETWORK_OVERHEAD_MS + ); + // An explicit client budget still wins in both directions. + assert_eq!(clamp_deadline_ms(Some(10)), 10); + assert_eq!( + clamp_deadline_ms(Some(MAX_DEADLINE_MS + 1)), + MAX_DEADLINE_MS + ); + } + fn four_region_cluster() -> ( Arc, Vec, diff --git a/tidal/src/db/metrics/cluster.rs b/tidal/src/db/metrics/cluster.rs index 89ceaba..1fb1223 100644 --- a/tidal/src/db/metrics/cluster.rs +++ b/tidal/src/db/metrics/cluster.rs @@ -153,6 +153,21 @@ pub struct ClusterMetrics { /// Self-driving heal (m11p8): peers this node is currently driving back to /// convergence (partitioned-or-lagging set size). 0 = fully converged. healing_peers: AtomicU64, + /// Total scatter-gather READ fan-outs that returned PARTIAL results because + /// one or more shards errored or missed the per-shard deadline. + /// + /// This is the one cluster failure that answers `200 OK`: the caller gets a + /// ranked list assembled from a subset of the corpus, with `degraded: true` + /// buried in the response metadata. Without this counter the only trace was + /// a per-request WARN, so a feed silently ranking over one third of its + /// candidates produced no signal an operator could alert on. Any sustained + /// rate is a correctness problem, not a latency one. + scatter_degraded_total: AtomicU64, + /// Total individual shard fetches inside a fan-out that came back + /// unavailable (error, queue-full, or deadline). Against + /// `scatter_degraded_total` it separates "one flaky peer" from "this node + /// can reach nobody". + scatter_shard_unavailable_total: AtomicU64, } impl ClusterMetrics { @@ -186,6 +201,8 @@ impl ClusterMetrics { heal_successes_total: AtomicU64::new(0), heal_noops_total: AtomicU64::new(0), healing_peers: AtomicU64::new(0), + scatter_degraded_total: AtomicU64::new(0), + scatter_shard_unavailable_total: AtomicU64::new(0), } } @@ -263,6 +280,17 @@ impl ClusterMetrics { self.healing_peers.store(count, Ordering::Relaxed); } + /// Count one scatter-gather READ fan-out that returned partial results, and + /// the number of shards that were unavailable inside it. + /// + /// Called once per degraded fan-out, never on the healthy path, so a zero + /// rate means every read saw every shard. + pub fn incr_scatter_degraded(&self, unavailable_shards: u64) { + self.scatter_degraded_total.fetch_add(1, Ordering::Relaxed); + self.scatter_shard_unavailable_total + .fetch_add(unavailable_shards, Ordering::Relaxed); + } + /// Record this node's election term + role (m11p4): role is 0 follower, /// 1 pre-candidate, 2 candidate, 3 leader. pub fn set_election_view(&self, term: u64, role: u64) { @@ -589,6 +617,22 @@ impl ClusterMetrics { self.healing_peers.load(Ordering::Relaxed) as f64, &extra, ); + emit_scalar( + out, + "tidaldb_cluster_scatter_degraded_total", + "Scatter-gather READ fan-outs that answered 200 with PARTIAL results because a shard errored or missed its deadline — a correctness signal, not a latency one", + "counter", + self.scatter_degraded_total.load(Ordering::Relaxed) as f64, + &extra, + ); + emit_scalar( + out, + "tidaldb_cluster_scatter_shard_unavailable_total", + "Individual shard fetches inside a read fan-out that came back unavailable (error, queue-full, or deadline)", + "counter", + self.scatter_shard_unavailable_total.load(Ordering::Relaxed) as f64, + &extra, + ); emit_scalar( out, @@ -728,6 +772,7 @@ mod tests { m.incr_heal_successes(); m.incr_heal_noops(); m.set_healing_peers(1); + m.incr_scatter_degraded(2); let mut out = String::new(); m.render_into(&mut out, 7); @@ -746,6 +791,8 @@ mod tests { assert!(out.contains("tidaldb_cluster_heal_successes_total 1")); assert!(out.contains("tidaldb_cluster_heal_noops_total 1")); assert!(out.contains("tidaldb_cluster_healing_peers 1")); + assert!(out.contains("tidaldb_cluster_scatter_degraded_total 1")); + assert!(out.contains("tidaldb_cluster_scatter_shard_unavailable_total 2")); assert!( out.contains( "tidaldb_cluster_peer_acked_seqno{peer_shard=\"2\",partition_id=\"7\"} 64"