diff --git a/docs/runbooks/cluster.md b/docs/runbooks/cluster.md index dbaa6e9..a6e6f59 100644 --- a/docs/runbooks/cluster.md +++ b/docs/runbooks/cluster.md @@ -221,6 +221,7 @@ Useful environment variables: | `TIDAL_API_KEY` | both | Bearer token for protected routes. Unset ⇒ unauthenticated + WARN. Set the SAME key on every region in multi-process mode. | | `TIDAL_HLC_SKEW_MS` | multi-process | Signed ms offset applied to THIS process's HLC. Affects reconcile-time LWW stamping ONLY (not signal-decay timestamps). A test/ops escape hatch for verifying causal convergence under clock skew — do not set it in normal operation. | | `TIDAL_CONFIG` | both | Config dir holding `default-schema.yaml` / `default-cluster.yaml` (used when `--schema` / `--topology` omitted). | +| `TIDAL_SEED_STATUS_TIMEOUT_MS` | seed-join | Per-status-poll HTTP timeout during seed-join leader discovery (default `5000`). Raise on a TLS cluster under heavy CPU contention where a cold rustls handshake alone can blow a tighter budget — the joiner would then time out every poll and burn the whole 120s discovery window despite the peer being reachable. Lower only for fast loopback/test rigs. | | `PORT` | both | Listen address. A bare port (`9500`) normalises to `0.0.0.0:9500`. | | `TIDAL_SERVER_LOG` | both | `tracing` filter (default `info`). | diff --git a/k8s/cluster-t4-kind/kustomization.yaml b/k8s/cluster-t4-kind/kustomization.yaml new file mode 100644 index 0000000..a7764cd --- /dev/null +++ b/k8s/cluster-t4-kind/kustomization.yaml @@ -0,0 +1,85 @@ +# m12p5 T4 (idle-readiness + scale 3→5→3 elasticity) overlay for local `kind`. +# +# Reuses the canonical k8s/cluster/ base but (1) collapses to a SINGLE +# replication group (topology-singlegroup-patch.yaml — T4 is replica elasticity, +# not sharding), (2) swaps the registry image for the locally-built m12p5 tag, +# (3) repoints the PVC at kind's `standard` StorageClass, and (4) gives each pod +# room for a 1536-dim corpus + a catch-up joiner. +# +# Run (the build tags MUST match the `images:` newTag below and the Job specs in +# tidal-stress/k8s/t4-*.yaml — kind serves only locally-loaded images, so a tag +# mismatch is an ImagePullBackOff): +# # server image — tag MUST equal the `images: newTag` below (m12p5-fix2) +# docker build -f docker/deploy/Dockerfile -t tidaldb-server:m12p5-fix2 . +# kind load docker-image tidaldb-server:m12p5-fix2 --name canopy +# # stress generator — the m12p4 binary is reused unchanged for m12p5; tag MUST +# # equal the `image:` in tidal-stress/k8s/t4-{seed,load}-job.yaml (m12p4-local) +# docker build -f docker/stress/Dockerfile -t tidaldb-stress:m12p4-local . +# kind load docker-image tidaldb-stress:m12p4-local --name canopy +# kubectl create namespace tidaldb-cluster +# kubectl -n tidaldb-cluster create secret generic tidaldb-credentials \ +# --from-literal=TIDAL_API_KEY="$(openssl rand -hex 32)" \ +# --from-literal=TIDAL_CLUSTER_KEY="$(openssl rand -hex 32)" +# kubectl apply -k k8s/cluster-t4-kind/ +# kubectl apply -f tidal-stress/k8s/t4-seed-job.yaml # seed the 1536-dim corpus +# kubectl apply -f tidal-stress/k8s/t4-load-job.yaml # steady load across the scale +# # scale 3→5 (idle-readiness fix + wildcard-SAN mTLS), then 5→3. +# kubectl -n tidaldb-cluster scale statefulset tidaldb --replicas=5 +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: tidaldb-cluster + +resources: + - ../cluster + +images: + - name: registry.threesix.ai/tidal/server + newName: tidaldb-server + newTag: m12p5-fix2 + +patches: + # Single replication group (drop the base's 3-group `shards:` block). + - path: topology-singlegroup-patch.yaml + target: + kind: ConfigMap + name: tidaldb-cluster-topology + + # kind ships `standard` (rancher.io/local-path), not the `local-path` the base + # names; repoint the PVC so claims bind. Bump the per-pod ceiling so a 1536-dim + # corpus + a joining replica's catch-up index fit (the 2Gi base is sized for a + # smoke corpus, not a real read-recall shape). + - target: + kind: StatefulSet + name: tidaldb + patch: |- + - op: replace + path: /spec/volumeClaimTemplates/0/spec/storageClassName + value: standard + - op: replace + path: /spec/template/spec/containers/0/resources/limits/memory + value: 6Gi + - op: replace + path: /spec/template/spec/containers/0/resources/limits/cpu + value: "3" + - op: replace + path: /spec/template/spec/containers/0/resources/requests/memory + value: 512Mi + # The 1536-dim HNSW index is rebuilt from durable storage on every open; + # under CPU contention on a single shared kind node that can exceed the + # base 5-min startup budget (a restarted/joining pod 503s its startup probe + # → SIGKILL → re-rebuild → cascade). Widen the budget to 10 min so a real + # catch-up + index rebuild finishes before the probe gives up. + - op: replace + path: /spec/template/spec/containers/0/startupProbe/failureThreshold + value: 120 + # Local-run convenience: only roll pods with ordinal >= 3 (the joiners) on a + # template change, so iterating on the scale-up command never restarts the + # seeded 0/1/2 (and never triggers a simultaneous index-rebuild cascade on + # the shared kind node). NOT for the canonical base — production rolls all. + - op: add + path: /spec/updateStrategy + value: + type: RollingUpdate + rollingUpdate: + partition: 3 diff --git a/k8s/cluster-t4-kind/topology-singlegroup-patch.yaml b/k8s/cluster-t4-kind/topology-singlegroup-patch.yaml new file mode 100644 index 0000000..64d193a --- /dev/null +++ b/k8s/cluster-t4-kind/topology-singlegroup-patch.yaml @@ -0,0 +1,53 @@ +# m12p5 T4 overlay — SINGLE replication group (no `shards:`). +# +# T4 is REPLICA elasticity + idle-readiness (scale the voter set 3→5→3), NOT +# sharding (that is m12p4/T5). A single group RF=all is the faithful T4 shape: +# it mirrors the in-process exit gate `mp_scale_3_5_3_under_load_zero_loss` and +# uses the well-proven m11p5 seed-join path (the sharded-cluster *replica* +# scale-up — 5 replicas per group — is an untested path outside m12p5's scope). +# +# Strategic-merge replaces the base ConfigMap's `cluster-topology.yaml` value: +# identical to k8s/cluster/topology-configmap.yaml MINUS the `shards:` block, so +# every pod runs ONE group binding gRPC 9601 (the extra 9602/9603 containerPorts +# in the StatefulSet are simply unbound — harmless). +apiVersion: v1 +kind: ConfigMap +metadata: + name: tidaldb-cluster-topology + namespace: tidaldb-cluster +data: + cluster-topology.yaml: | + regions: + - name: tidaldb-0 + grpc_addr: tidaldb-0.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9601 + grpc_bind: 0.0.0.0:9601 + http_addr: tidaldb-0.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 + metrics_addr: 0.0.0.0:9091 + grpc_tls: &grpc_tls + ca_cert: /etc/tidaldb/tls/ca.crt + server_cert: /etc/tidaldb/tls/tls.crt + server_key: /etc/tidaldb/tls/tls.key + client_cert: /etc/tidaldb/tls/tls.crt + client_key: /etc/tidaldb/tls/tls.key + - name: tidaldb-1 + grpc_addr: tidaldb-1.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9601 + grpc_bind: 0.0.0.0:9601 + http_addr: tidaldb-1.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 + metrics_addr: 0.0.0.0:9091 + grpc_tls: *grpc_tls + - name: tidaldb-2 + grpc_addr: tidaldb-2.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9601 + grpc_bind: 0.0.0.0:9601 + http_addr: tidaldb-2.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 + metrics_addr: 0.0.0.0:9091 + grpc_tls: *grpc_tls + leader: tidaldb-0 + replication: + ack: quorum + reseed_self_restart: true + election: + heartbeat_interval_ms: 300 + election_timeout_min_ms: 1500 + election_timeout_max_ms: 3000 + leader_lease_ms: 900 + auto_election: true diff --git a/k8s/cluster/statefulset.yaml b/k8s/cluster/statefulset.yaml index d016b66..3bdcd86 100644 --- a/k8s/cluster/statefulset.yaml +++ b/k8s/cluster/statefulset.yaml @@ -103,7 +103,18 @@ spec: - | set -eu ORD="${POD_NAME##*-}" + # Per-pod STABLE DNS (headless Service) — what this pod ADVERTISES + # for peers to dial it directly. The headless Service publishes + # not-ready addresses (so a pod has DNS before it is Ready), so it + # resolves to EVERY pod incl. still-joining ones. DOMAIN="tidaldb-peers.tidaldb-cluster.svc.cluster.local" + # READY-ONLY client Service (ClusterIP VIP) — the seed-join discovery + # target. It excludes not-ready pods, so a joiner always reaches a + # LIVE serving peer instead of round-robining onto a not-ready pod + # (often ITSELF, since the headless Service includes the joiner) and + # failing discovery for the whole 120s window — the real T4 scale-up + # blocker. Carries its own cert SAN (certs.yaml). + SEED_SVC="tidaldb.tidaldb-cluster.svc.cluster.local" # Common args for every pod. set -- cluster \ --listen 0.0.0.0:9500 \ @@ -112,12 +123,21 @@ spec: --topology /etc/tidal-server/cluster-topology.yaml \ --experimental-cluster if [ "$ORD" -ge 3 ]; then - # SCALE-UP pod: seed-join as a learner. Advertise THIS pod's - # stable per-pod DNS for gRPC (9601) and HTTP (9500); point --seed - # at the headless Service (it resolves to a live peer). --metrics - # gives the joiner a metrics listener (it has no topology entry). + # SCALE-UP pod: seed-join as a learner. Discover a live leader via + # the READY-ONLY client Service ($SEED_SVC); advertise THIS pod's + # stable per-pod DNS ($DOMAIN) for gRPC (9601) and HTTP (9500) so + # peers dial it directly. --metrics gives the joiner a metrics + # listener (it has no topology entry). + # m11p7/m12p5: the :9500 plane serves TLS, and `peer_url` honors an + # explicit URL scheme VERBATIM (forward.rs) — so the seed MUST be + # `https://`, not `http://` (with `http://` the joiner dials + # plaintext to the TLS port and seed-join fails). The discovery + # target is the ready-only client Service, NOT the headless peers + # Service, so a joiner never round-robins onto a not-ready pod + # (incl. itself) and burns the 120s discovery window — both were + # real T4 scale-up blockers. set -- "$@" \ - --seed "http://${DOMAIN}:9500" \ + --seed "https://${SEED_SVC}:9500" \ --advertise-grpc "${POD_NAME}.${DOMAIN}:9601" \ --advertise-http "${POD_NAME}.${DOMAIN}:9500" \ --metrics 0.0.0.0:9091 diff --git a/tidal-net/src/lib.rs b/tidal-net/src/lib.rs index 2353a2c..14a4435 100644 --- a/tidal-net/src/lib.rs +++ b/tidal-net/src/lib.rs @@ -41,10 +41,13 @@ pub mod proto { pub use config::{GrpcTransportConfig, TlsConfig}; pub use error::GrpcTransportError; pub use sources::{ - ClaimRejection, ElectionHooks, HeartbeatExchange, JoinAsk, JoinHooks, JoinOutcome, MemberInfo, - SnapshotRequiredSink, SnapshotSource, SnapshotStageError, SnapshotStaging, + ClaimRejection, ElectionHooks, HeartbeatContext, HeartbeatExchange, JoinAsk, JoinHooks, + JoinOutcome, MemberInfo, SnapshotRequiredSink, SnapshotSource, SnapshotStageError, + SnapshotStaging, +}; +pub use transport::{ + ElectionNet, ElectionNetEvent, GrpcTransport, GrpcTransportFactory, ensure_crypto_provider, }; -pub use transport::{ElectionNet, ElectionNetEvent, GrpcTransport, GrpcTransportFactory}; // m11p7: the inter-node HTTP listener (in tidal-server) reuses these to serve TLS // over the SAME cert FILES the gRPC plane uses — via its OWN `DynamicCertResolver` // + reload poller, not a shared resolver instance. A single secret rotation is diff --git a/tidal-net/src/server.rs b/tidal-net/src/server.rs index 62da728..5c03228 100644 --- a/tidal-net/src/server.rs +++ b/tidal-net/src/server.rs @@ -717,16 +717,16 @@ impl WalShipping for WalShippingService { // member, so it is trivially "not removed". let removed = u16::try_from(req.region_id).is_ok_and(|sender| hooks.is_removed_member(sender)); - let verdict = hooks.on_heartbeat( - req.term, + let verdict = hooks.on_heartbeat(crate::HeartbeatContext { + term: req.term, leader_region, - req.stream_baseline, - tidaldb::replication::LogPosition { + stream_baseline: req.stream_baseline, + prev_log: tidaldb::replication::LogPosition { tail_term: req.prev_log_term, frontier: req.prev_log_seq, }, - req.leader_last_seq, - ); + leader_last_seq: req.leader_last_seq, + }); return Ok(Response::new(HeartbeatResponse { acknowledged: true, term: verdict.term, diff --git a/tidal-net/src/sources.rs b/tidal-net/src/sources.rs index 7e16bae..4d6810c 100644 --- a/tidal-net/src/sources.rs +++ b/tidal-net/src/sources.rs @@ -238,6 +238,29 @@ pub struct HeartbeatExchange { pub accepted: bool, } +/// The leader-heartbeat payload a follower folds (m11p4 + m12p5). +/// +/// Grouped into a struct so new heartbeat-carried state is an ADDITIVE field +/// rather than a wider positional signature on [`ElectionHooks::on_heartbeat`] +/// — and so the several `u64`s (`term`, `stream_baseline`, `leader_last_seq`) +/// can't be silently transposed at a call site. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HeartbeatContext { + /// The leader's election term. + pub term: u64, + /// The leader's region id. + pub leader_region: u16, + /// The term's immutable activation baseline (the stream's term-start point). + pub stream_baseline: u64, + /// The leader's election-time log position `(prev_log_term, prev_log_seq)`. + pub prev_log: tidaldb::replication::LogPosition, + /// The leader's LIVE flushed frontier at heartbeat time (m12p5): the + /// high-water-mark every follower converges to, in the same stream numbering + /// as a follower's per-shard `applied_seqno`. `0` = a pre-m12p5 leader + /// conveyed no frontier (the heartbeat readiness drive is skipped). + pub leader_last_seq: u64, +} + /// Why an inbound leadership-stamped payload was refused (m11p4). #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ClaimRejection { @@ -292,22 +315,15 @@ pub trait ElectionHooks: Send + Sync + 'static { first_seq: u64, ) -> Result<(), ClaimRejection>; - /// A leader heartbeat: term + leadership + the term's activation - /// baseline + the leader's election-time log position + /// A leader heartbeat (payload: [`HeartbeatContext`]): term + leadership + + /// the term's activation baseline + the leader's election-time log position /// `(prev_log_term, prev_log_seq)` + the leader's LIVE flushed frontier /// (`leader_last_seq`). Drives the failure detector, the term-join /// divergence check, and (m12p5) the sticky readiness latch — a caught-up /// joiner converges from the heartbeat (which flows on an idle cluster), /// not only from observed ship traffic. `leader_last_seq == 0` means a /// pre-m12p5 leader conveyed no frontier (the readiness drive is skipped). - fn on_heartbeat( - &self, - term: u64, - leader_region: u16, - stream_baseline: u64, - prev_log: tidaldb::replication::LogPosition, - leader_last_seq: u64, - ) -> HeartbeatExchange; + fn on_heartbeat(&self, hb: HeartbeatContext) -> HeartbeatExchange; /// A pre-vote or vote request. The grant (and any term adoption) is /// durable before this returns. diff --git a/tidal-net/src/transport.rs b/tidal-net/src/transport.rs index a17052d..6e642b7 100644 --- a/tidal-net/src/transport.rs +++ b/tidal-net/src/transport.rs @@ -503,7 +503,9 @@ impl ShutdownSignal { } /// Install a process-wide rustls [`CryptoProvider`] before tonic's TLS -/// builders run. tidal-net pulls rustls only transitively (via tonic), with no +/// builders run. +/// +/// tidal-net pulls rustls only transitively (via tonic), with no /// provider feature in its own dependency closure, so under a narrow build the /// process-level default is absent and rustls panics (could not automatically /// determine the process-level [`CryptoProvider`]); under a wider build two @@ -512,7 +514,14 @@ impl ShutdownSignal { /// (e.g. a reqwest-based client elsewhere in the process) already set a default /// — removes that fragility for the mTLS tests, a standalone tidal-server, and /// the replication path alike. See `tidal-net/BUILD.bazel`. -pub(crate) fn ensure_crypto_provider() { +/// +/// Public so a binary can install the provider at the TOP of `main()`, BEFORE +/// any code builds a rustls client. The seed-join / reseed boot path +/// (`tidal-server`) constructs a blocking `reqwest` (rustls) HTTPS client on a +/// dedicated boot thread *before* `GrpcTransport::new` (which calls this) runs — +/// so without an up-front install that client panics. Idempotent and cheap; +/// safe to call unconditionally. +pub fn ensure_crypto_provider() { static ONCE: std::sync::Once = std::sync::Once::new(); ONCE.call_once(|| { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); diff --git a/tidal-net/tests/election_rpc.rs b/tidal-net/tests/election_rpc.rs index 5f00492..e870851 100644 --- a/tidal-net/tests/election_rpc.rs +++ b/tidal-net/tests/election_rpc.rs @@ -111,16 +111,16 @@ impl ElectionHooks for ScriptedHooks { Ok(()) } - fn on_heartbeat( - &self, - term: u64, - leader_region: u16, - baseline: u64, - _prev_log: tidaldb::replication::LogPosition, - leader_last_seq: u64, - ) -> HeartbeatExchange { + fn on_heartbeat(&self, hb: tidal_net::HeartbeatContext) -> HeartbeatExchange { + let tidal_net::HeartbeatContext { + term, + leader_region, + stream_baseline, + leader_last_seq, + .. + } = hb; self.seen.lock().unwrap().push(format!( - "hb:{term}:{leader_region}:{baseline}:{leader_last_seq}" + "hb:{term}:{leader_region}:{stream_baseline}:{leader_last_seq}" )); HeartbeatExchange { term: self.term, diff --git a/tidal-net/tests/snapshot_fetch.rs b/tidal-net/tests/snapshot_fetch.rs index bc3e940..6f449cf 100644 --- a/tidal-net/tests/snapshot_fetch.rs +++ b/tidal-net/tests/snapshot_fetch.rs @@ -351,14 +351,7 @@ fn fetch_snapshot_term_fence_refuses_stale_puller() { ) -> Result<(), tidal_net::ClaimRejection> { Ok(()) } - fn on_heartbeat( - &self, - _term: u64, - _leader_region: u16, - _stream_baseline: u64, - _prev_log: tidaldb::replication::LogPosition, - _leader_last_seq: u64, - ) -> tidal_net::HeartbeatExchange { + fn on_heartbeat(&self, _hb: tidal_net::HeartbeatContext) -> tidal_net::HeartbeatExchange { tidal_net::HeartbeatExchange { term: self.0, accepted: true, diff --git a/tidal-server/src/cluster/election_driver.rs b/tidal-server/src/cluster/election_driver.rs index 5b05ab7..58800dc 100644 --- a/tidal-server/src/cluster/election_driver.rs +++ b/tidal-server/src/cluster/election_driver.rs @@ -545,14 +545,14 @@ impl tidal_net::ElectionHooks for NodeElectionHooks { Ok(()) } - fn on_heartbeat( - &self, - term: u64, - leader_region: u16, - stream_baseline: u64, - prev_log: LogPosition, - leader_last_seq: u64, - ) -> HeartbeatExchange { + fn on_heartbeat(&self, hb: tidal_net::HeartbeatContext) -> HeartbeatExchange { + let tidal_net::HeartbeatContext { + term, + leader_region, + stream_baseline, + prev_log, + leader_last_seq, + } = hb; let now = Instant::now(); let current = self.runtime.current_term(); // Same topology-era allowance as the payload gate: a term-0 leader's diff --git a/tidal-server/src/cluster/join_boot.rs b/tidal-server/src/cluster/join_boot.rs index 28d9a36..292153d 100644 --- a/tidal-server/src/cluster/join_boot.rs +++ b/tidal-server/src/cluster/join_boot.rs @@ -53,8 +53,20 @@ use super::topology::{ }; use crate::error::{Result, ServerError}; -/// Per-status-poll HTTP timeout (the discovery loop iterates). -const STATUS_POLL_TIMEOUT: Duration = Duration::from_millis(500); +/// Per-status-poll HTTP timeout (the discovery loop iterates). Generous because +/// a COLD poll pays a full TLS handshake (rustls/aws-lc-rs) on top of DNS + TCP, +/// and on a real (TLS) cluster under CPU contention that handshake alone can blow +/// a sub-second budget — the joiner then times out every poll and burns the whole +/// 120 s discovery window despite the peer being reachable (TCP connects fine). +/// The in-process tests never saw this: plaintext loopback has no handshake cost. +/// Env-overridable via `TIDAL_SEED_STATUS_TIMEOUT_MS`. +fn status_poll_timeout() -> Duration { + std::env::var("TIDAL_SEED_STATUS_TIMEOUT_MS") + .ok() + .and_then(|v| v.trim().parse::().ok()) + .filter(|&ms| ms > 0) + .map_or(Duration::from_secs(5), Duration::from_millis) +} /// Per-join RPC timeout (the leader's bounded same-term commit wait is itself /// bounded by `quorum_timeout`, so allow generous headroom). const JOIN_RPC_TIMEOUT: Duration = Duration::from_secs(10); @@ -175,7 +187,7 @@ pub fn seed_join_boot(input: &SeedJoinInput<'_>) -> Result { let deadline = Instant::now() + join_window; let mut backoff = BACKOFF_MIN; - let status_client = build_join_client(STATUS_POLL_TIMEOUT, tls.as_ref())?; + let status_client = build_join_client(status_poll_timeout(), tls.as_ref())?; while Instant::now() < deadline { // (1) Discover a leader over (seeds ∪ cache). The cache (when present) @@ -300,7 +312,14 @@ fn build_join_client( timeout: std::time::Duration, tls: Option<&tidal_net::config::TlsConfig>, ) -> Result { - let mut builder = reqwest::blocking::Client::builder().timeout(timeout); + // Separate connect timeout from the (generous) total timeout: a genuinely + // dead seed (SYN blackhole) fails fast at the connect stage instead of + // blocking the full `timeout` before the discovery loop tries the next seed, + // while a slow-but-live TLS handshake (the reason `timeout` is generous — + // see `status_poll_timeout`) still gets the full budget to complete. + let mut builder = reqwest::blocking::Client::builder() + .connect_timeout(Duration::from_secs(2)) + .timeout(timeout); if let Some(t) = tls { let pem = std::fs::read(&t.ca_cert) .map_err(|e| ServerError::Cluster(format!("seed-join: read CA cert: {e}")))?; @@ -382,11 +401,21 @@ fn discover_leader( if let Some(key) = api_key { req = req.bearer_auth(key); } - let Ok(resp) = req.send() else { continue }; + // Log every failure mode at debug — a silent discovery loop turned a + // too-tight TLS poll timeout into an opaque "could not join within 120s". + let resp = match req.send() { + Ok(resp) => resp, + Err(e) => { + tracing::debug!(%url, error = %e, "seed-join discover: status poll failed (transport/TLS/timeout)"); + continue; + } + }; if !resp.status().is_success() { + tracing::debug!(%url, status = %resp.status(), "seed-join discover: status poll non-2xx"); continue; } let Ok(json) = resp.json::() else { + tracing::debug!(%url, "seed-join discover: status body was not JSON"); continue; }; // Any reachable member is a fallback join target (it forwards to the diff --git a/tidal-server/src/cluster/node.rs b/tidal-server/src/cluster/node.rs index 0ba379a..54cf1e8 100644 --- a/tidal-server/src/cluster/node.rs +++ b/tidal-server/src/cluster/node.rs @@ -2800,6 +2800,20 @@ impl ShardReplica { /// install-boot node becomes ready once its catch-up lag first falls at or /// below `learner_promote_lag` (never `lag == 0`). Called from the status /// path (which already computes lag) so no new polling thread is needed. + /// This follower's applied frontier for the CURRENT leader's source shard + /// (`shard_of_region(leader)`), or 0 if the shard is untracked or the db is + /// gone. The single reader for "how far have I applied the leader's stream", + /// shared by the heartbeat readiness drive ([`note_leader_frontier_for_readiness`]) + /// and `local_status` so the per-source-shard keying (BUG 1) lives in one place. + /// + /// [`note_leader_frontier_for_readiness`]: Self::note_leader_frontier_for_readiness + fn applied_for_leader_shard(&self, leader_shard: ShardId) -> u64 { + self.db() + .ok() + .and_then(|db| db.replication_state().applied_seqno(leader_shard)) + .unwrap_or(0) + } + fn note_lag_for_readiness(&self, lag_events: u64) { if (self.install_boot || self.seed_joiner) && !self.converged.load(Ordering::Acquire) @@ -2853,10 +2867,7 @@ impl ShardReplica { .lag_gauge() .update_leader_seqno_for(leader_shard, leader_last_seq); if (self.install_boot || self.seed_joiner) && !self.converged.load(Ordering::Acquire) { - let applied = db - .replication_state() - .applied_seqno(leader_shard) - .unwrap_or(0); + let applied = self.applied_for_leader_shard(leader_shard); self.note_lag_for_readiness(leader_last_seq.saturating_sub(applied)); } } @@ -3033,9 +3044,7 @@ impl ShardReplica { let applied_events = if is_leader { last_seq } else { - db.replication_state() - .applied_seqno(leader_shard) - .unwrap_or(0) + self.applied_for_leader_shard(leader_shard) }; // lag = the CURRENT leader's per-source-shard high-water-mark − applied // for that same shard (BUG 1). The gauge tracks the leader HWM PER SOURCE @@ -3047,8 +3056,7 @@ impl ShardReplica { let lag_events = if is_leader { 0 } else { - self.db()? - .control_plane() + db.control_plane() .lag_gauge() .leader_seqno_for(leader_shard) .saturating_sub(applied_events) diff --git a/tidal-server/src/main.rs b/tidal-server/src/main.rs index ca37cc1..d531eab 100644 --- a/tidal-server/src/main.rs +++ b/tidal-server/src/main.rs @@ -121,6 +121,15 @@ async fn run() -> Result<()> { let cli = Cli::parse(); init_tracing(); + // Install the process-wide rustls CryptoProvider BEFORE any code can build a + // rustls client. The seed-join / reseed boot path builds a blocking reqwest + // (rustls) HTTPS client on a dedicated boot thread *before* the gRPC + // transport installs the provider — without this up-front call that client + // panics ("could not automatically determine the process-level + // CryptoProvider"), crash-looping every TLS scale-up / snapshot-install + // joiner. Idempotent; harmless for the plaintext standalone path. + tidal_net::ensure_crypto_provider(); + match cli.mode { Command::Standalone(args) => run_standalone(args).await, Command::Cluster(args) => run_cluster(args).await, diff --git a/tidal-server/tests/cluster_membership.rs b/tidal-server/tests/cluster_membership.rs index fe24dcd..c43be91 100644 --- a/tidal-server/tests/cluster_membership.rs +++ b/tidal-server/tests/cluster_membership.rs @@ -569,9 +569,20 @@ fn mp_idle_cluster_snapshot_joiner_flips_ready_without_traffic() { ); thread::sleep(Duration::from_millis(100)); } - println!( - "[idle-ready] snapshot joiner flipped /health READY in {:?} on an idle cluster", - ready_start.elapsed() + let flip_elapsed = ready_start.elapsed(); + println!("[idle-ready] snapshot joiner flipped /health READY in {flip_elapsed:?} on an idle cluster"); + // Tight bound: heartbeat convergence is sub-second after catch-up (measured + // 257µs / 101ms). A flip that only just beats the full budget would mean + // convergence regressed onto a SLOW path (e.g. a periodic self-heal tick or a + // reintroduced status-poll dependency) — which the binary budget check above + // would wave through. Half the budget is a >100× margin over the observed flip + // yet still well below the negative control's full-budget stall. + assert!( + flip_elapsed < convergence_budget() / 2, + "IDLE-READINESS SLOW-PATH REGRESSION (m12p5): joiner converged in {flip_elapsed:?}, not \ + within {:?} (½ budget). Heartbeat convergence is sub-second after catch-up; a multi-second \ + flip means convergence regressed off the heartbeat onto a slow periodic path.", + convergence_budget() / 2 ); // ── HONESTY: readiness must mean actually-caught-up, not a premature latch. diff --git a/tidal-stress/k8s/t4-load-job.yaml b/tidal-stress/k8s/t4-load-job.yaml new file mode 100644 index 0000000..4473cec --- /dev/null +++ b/tidal-stress/k8s/t4-load-job.yaml @@ -0,0 +1,111 @@ +# T4 (m12p5) — sustained load during the 3→5→3 scale (Phase B). +# +# A single generator holds a steady ack=quorum read+write mix for ~5 min, long +# enough to span a scale-down (5→3) and scale-up (3→5) under load. `--skip-seed` +# (the corpus is already registered by t4-seed-job) keeps the id space; the mix +# carries item/embed WRITES so the transition is exercised by live quorum writes, +# not just reads. `--max-error-pct 0` is the zero-acked-loss gate: a quorum write +# that cannot reach its majority during a membership change fails to ack → error +# → the Job exits non-zero. `--json-summary` captures the per-stage p99 envelope +# (before / during / after the scale) for the evidence table. +# +# NO --poll-status: this phase is UNDER LOAD (the idle-readiness isolation is the +# separate idle Phase A driven from the host), so status polling is unneeded and +# omitted to keep the offered load pure. +# +# Apply: kubectl apply -f tidal-stress/k8s/t4-load-job.yaml +# Watch: kubectl logs -f job/tidal-stress-t4-load -n tidaldb-cluster +apiVersion: batch/v1 +kind: Job +metadata: + name: tidal-stress-t4-load + namespace: tidaldb-cluster + labels: + app.kubernetes.io/name: tidal-stress + app.kubernetes.io/part-of: tidaldb +spec: + backoffLimit: 0 + ttlSecondsAfterFinished: 7200 + template: + metadata: + labels: + app.kubernetes.io/name: tidal-stress + app.kubernetes.io/part-of: tidaldb + spec: + restartPolicy: Never + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: stress + image: tidaldb-stress:m12p4-local + imagePullPolicy: IfNotPresent + args: + - --target + - https://tidaldb-0.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 + - --target + - https://tidaldb-1.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 + - --target + - https://tidaldb-2.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 + - --ca-cert + - /etc/tidaldb/tls/ca.crt + - --ack + - quorum + - --skip-seed + - --corpus + - "10000" + - --embedding-dim + - "1536" + - --users + - "100000" + # ~5 min steady hold: three 100s stages so the per-stage JSON gives a + # before / during / after p99 read across the scale window. + - --ramp + - "400:100,400:100,400:100" + - --mix + - "feed=4,search=1,view=2,like=1,item=1,embed=1" + - --max-inflight + - "2000" + - --json-summary + - /tmp/t4-load-summary.json + - --max-error-pct + - "0" + env: + - name: TIDAL_API_KEY + valueFrom: + secretKeyRef: + name: tidaldb-credentials + key: TIDAL_API_KEY + - name: TIDAL_STRESS_LOG + value: warn + resources: + requests: + cpu: 500m + memory: 256Mi + limits: + cpu: "3" + memory: 1Gi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: cluster-tls + mountPath: /etc/tidaldb/tls + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: cluster-tls + secret: + secretName: tidaldb-cluster-tls + items: + - key: ca.crt + path: ca.crt + - name: tmp + emptyDir: {} diff --git a/tidal-stress/k8s/t4-seed-job.yaml b/tidal-stress/k8s/t4-seed-job.yaml new file mode 100644 index 0000000..56fdd31 --- /dev/null +++ b/tidal-stress/k8s/t4-seed-job.yaml @@ -0,0 +1,99 @@ +# T4 (m12p5) — corpus seed. Seeds items id 1..=N with 1536-dim embeddings at +# ack=quorum, so the registered corpus is a KNOWN, durably-committed (acked) set +# — the ground truth for the zero-acked-loss check across the 3→5→3 scale and +# the content-parity check on each joiner (honest convergence). +# +# Single pod, runs to completion. The trailing trivial ramp (1 rps / 1 s) exists +# only because the generator always seeds before a ramp; the seed is the point. +# +# Apply: kubectl apply -f tidal-stress/k8s/t4-seed-job.yaml +# Watch: kubectl logs -f job/tidal-stress-t4-seed -n tidaldb-cluster +apiVersion: batch/v1 +kind: Job +metadata: + name: tidal-stress-t4-seed + namespace: tidaldb-cluster + labels: + app.kubernetes.io/name: tidal-stress + app.kubernetes.io/part-of: tidaldb +spec: + backoffLimit: 0 + ttlSecondsAfterFinished: 7200 + template: + metadata: + labels: + app.kubernetes.io/name: tidal-stress + app.kubernetes.io/part-of: tidaldb + spec: + restartPolicy: Never + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: stress + image: tidaldb-stress:m12p4-local + imagePullPolicy: IfNotPresent + args: + - --target + - https://tidaldb-0.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 + - --target + - https://tidaldb-1.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 + - --target + - https://tidaldb-2.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 + - --ca-cert + - /etc/tidaldb/tls/ca.crt + - --ack + - quorum + - --corpus + - "10000" + - --embedding-dim + - "1536" + - --users + - "100000" + - --seed-concurrency + - "64" + - --ramp + - "1:1" + - --mix + - writes + - --max-inflight + - "512" + env: + - name: TIDAL_API_KEY + valueFrom: + secretKeyRef: + name: tidaldb-credentials + key: TIDAL_API_KEY + - name: TIDAL_STRESS_LOG + value: warn + resources: + requests: + cpu: 500m + memory: 256Mi + limits: + cpu: "3" + memory: 1Gi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: cluster-tls + mountPath: /etc/tidaldb/tls + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: cluster-tls + secret: + secretName: tidaldb-cluster-tls + items: + - key: ca.crt + path: ca.crt + - name: tmp + emptyDir: {}