m11p1 — decoupled ack/ship path: staged writes (seqno+WAL+relay-push, microseconds) separate from group-commit fsync; ShipQueue batches+windows outbound segments; receiver coalesces inbound chunks before applying. Adds first tidaldb_cluster_* metrics. m11p2 — leader WAL is now THE replicated log: fsynced batches feed a bounded WalShipFeed and ship byte-identical to followers; WAL seqnos survive restarts (relay-reset hazard gone). Item metadata and embeddings journal kind-1/2 blob records on the same stream as signals; the m8p10 HTTP broadcast is deleted. StreamSegments catch-up is follower-pulled via server-streaming RPC, triggered on gap detection, follower boot, and leader heal nudge. Promote carries a stream baseline so peers skip pre-stream history.
264 lines
12 KiB
Rust
264 lines
12 KiB
Rust
//! Transport abstraction for WAL segment shipping between shards.
|
|
//!
|
|
//! The [`Transport`] trait defines how segments are sent and received between
|
|
//! nodes. Implementations include:
|
|
//! - [`super::in_process::InProcessTransport`]: crossbeam channels for tests
|
|
//! and single-process multi-shard setups.
|
|
//! - (future) gRPC or TCP transport for multi-node deployments.
|
|
|
|
use crate::replication::{segment_id::WalSegmentId, shard::ShardId};
|
|
|
|
/// A WAL segment ready for shipping to a peer shard.
|
|
///
|
|
/// Contains the raw batch bytes from a sealed WAL segment file, plus
|
|
/// metadata for tracking and idempotent replay on the receiver.
|
|
pub struct WalSegmentPayload {
|
|
/// Globally unique identifier for this segment.
|
|
pub id: WalSegmentId,
|
|
/// Raw WAL bytes (one or more encoded batches).
|
|
pub bytes: Vec<u8>,
|
|
/// Total number of events across all batches in this segment.
|
|
pub event_count: u64,
|
|
/// The segment's authoritative last WAL sequence number, as known by the
|
|
/// leader **before** any community-overlay filtering.
|
|
///
|
|
/// The shipper computes this from the *original* (unfiltered) segment bytes
|
|
/// (`first_seq + total_event_count - 1`), so it is correct even when the
|
|
/// shipped `bytes` are an all-local empty filtration that decodes to zero
|
|
/// batches. The receiver feeds it to the replication lag gauge's leader
|
|
/// high-water-mark independently of how many batches survive filtering, so
|
|
/// an all-local (empty) segment does not make the gauge under-report the
|
|
/// leader's progress (obs-REPL-1). `0` means "unknown / no events" and the
|
|
/// receiver falls back to the per-batch boundaries it decodes.
|
|
pub leader_last_seq: u64,
|
|
/// The shipping stream's baseline: the WAL seqno at which this leader's
|
|
/// stream STARTED (0 = from the beginning, the default). Non-zero only on
|
|
/// catch-up stream chunks from a promoted leader: seqnos at or below the
|
|
/// baseline are pre-stream history (data the follower already holds via
|
|
/// the previous leader's stream), so the receiver jumps its frontier to
|
|
/// the baseline instead of waiting for a gap that will never close.
|
|
pub stream_baseline: u64,
|
|
}
|
|
|
|
/// Errors that can occur during WAL segment transport.
|
|
#[derive(Debug, thiserror::Error)]
|
|
pub enum TransportError {
|
|
/// The target shard is not known to this transport.
|
|
#[error("unknown peer shard: {0}")]
|
|
UnknownPeer(ShardId),
|
|
/// The transport channel has been closed (peer disconnected or shutdown).
|
|
///
|
|
/// This is a **transient** failure: the peer is known but temporarily
|
|
/// unable to accept the segment (process down, circuit breaker open,
|
|
/// follower backpressure). The shipper leaves the peer's high-water-mark
|
|
/// untouched and re-ships the same seqno on the next poll, preserving a
|
|
/// gap-free in-order stream.
|
|
#[error("transport channel closed")]
|
|
Closed,
|
|
/// The send failed in a way that re-shipping the **same** segment will
|
|
/// never resolve: a mutually-distrusted TLS chain, a misconfigured CA, an
|
|
/// authentication/authorization rejection, a payload the wire codec will
|
|
/// never accept, or an RPC the peer does not implement.
|
|
///
|
|
/// Unlike [`Closed`](Self::Closed), retrying a `Permanent` failure against
|
|
/// the same seqno is an **infinite silent stall** — nothing about the
|
|
/// segment becomes shippable by trying again until an operator changes the
|
|
/// peer/config. The shipper must therefore quarantine the peer and surface
|
|
/// an actionable error instead of re-shipping forever. `reason` carries the
|
|
/// human-readable cause (from the underlying transport) for logs/alerts.
|
|
#[error("permanent transport failure: {reason}")]
|
|
Permanent {
|
|
/// Human-readable cause of the permanent failure (TLS/auth/codec/etc.).
|
|
reason: String,
|
|
},
|
|
/// The payload exceeds the transport's maximum size limit.
|
|
#[error("payload too large: {size} bytes (max {max})")]
|
|
PayloadTooLarge {
|
|
/// Actual payload size in bytes.
|
|
size: usize,
|
|
/// Maximum allowed payload size in bytes.
|
|
max: usize,
|
|
},
|
|
}
|
|
|
|
/// Trait for shipping WAL segments between shards.
|
|
///
|
|
/// Implementations must be `Send + Sync + 'static` to allow sharing across
|
|
/// the shipper and receiver threads.
|
|
///
|
|
/// # Contract
|
|
///
|
|
/// - `send_segment` delivers best-effort and **MAY block** the calling thread up
|
|
/// to an implementation-defined timeout. It is **not** a non-blocking call: the
|
|
/// gRPC implementation (`tidal_net::GrpcTransport`) bridges its async client to
|
|
/// this sync trait with `runtime.block_on(pool.send_to(..))`, which parks the
|
|
/// caller for up to its `request_timeout` (default ~10s) per call — connect plus
|
|
/// send plus response. It MUST therefore be called only from a dedicated WAL
|
|
/// shipper thread (a `std::thread`, never inside a tokio runtime or any
|
|
/// async/latency-sensitive context). The in-process implementation returns
|
|
/// promptly, but callers must not depend on that — code to the blocking contract.
|
|
/// - `recv_segment` blocks until a segment is available or the transport shuts down.
|
|
/// - `local_shard` returns this node's shard identity.
|
|
pub trait Transport: Send + Sync + 'static {
|
|
/// Ship a WAL segment payload to a peer shard.
|
|
///
|
|
/// **Blocking:** this call MAY block the caller up to an implementation-defined
|
|
/// timeout (the gRPC implementation blocks up to `request_timeout`, ~10s). Call
|
|
/// it only from a dedicated shipper thread, never from an async or
|
|
/// latency-sensitive context. See the trait-level `# Contract` for the rationale
|
|
/// and the `GrpcTransport` `block_on` bridge.
|
|
///
|
|
/// Blanket implementation for `Arc<dyn Transport>` so callers can use
|
|
/// type-erased transports with the generic `spawn_shipper` / `spawn_receiver` APIs.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// - `TransportError::UnknownPeer` if `to` is not a recognized shard.
|
|
/// - `TransportError::Closed` if the peer's receive channel is disconnected
|
|
/// (transient — the shipper retries the same seqno next poll).
|
|
/// - `TransportError::Permanent` if the send can never succeed for this
|
|
/// segment without a config/peer change (bad TLS/CA, auth rejection,
|
|
/// malformed payload, unimplemented RPC) — the shipper quarantines the
|
|
/// peer instead of retrying forever.
|
|
/// - `TransportError::PayloadTooLarge` if the payload exceeds the size limit.
|
|
fn send_segment(&self, to: ShardId, payload: WalSegmentPayload) -> Result<(), TransportError>;
|
|
|
|
/// Block until a segment payload is received, or return `None` on shutdown.
|
|
///
|
|
/// Returns `None` when the transport is closed and no more segments will
|
|
/// arrive. The receiver thread should exit when this returns `None`.
|
|
fn recv_segment(&self) -> Option<WalSegmentPayload>;
|
|
|
|
/// Return an immediately-available segment payload WITHOUT blocking, or
|
|
/// `None` when the inbound queue is empty (or the transport does not
|
|
/// support non-blocking receives — the conservative default).
|
|
///
|
|
/// The segment receiver uses this to drain a backlog after each blocking
|
|
/// `recv_segment` and apply the drained segments through ONE shared
|
|
/// follower group-commit (m11p1): without coalescing, every segment pays
|
|
/// a solo fsync (~ms-scale on real volumes), capping follower apply
|
|
/// throughput at `events-per-segment / fsync-cost` and letting lag grow
|
|
/// without bound under sustained leader load.
|
|
fn try_recv_segment(&self) -> Option<WalSegmentPayload> {
|
|
None
|
|
}
|
|
|
|
/// Ask the source shard for a catch-up stream from `from_seqno` (m11p2:
|
|
/// follower-pulled `StreamSegments` over the leader's durable WAL).
|
|
///
|
|
/// Called by the segment receiver when an apply round leaves a gap below
|
|
/// a parked range — data older than the leader's in-memory ship tail that
|
|
/// push-based shipping can no longer deliver. Implementations MUST be
|
|
/// non-blocking (fire the pull in the background), rate-limited, and
|
|
/// de-duplicated per shard (at most one in-flight pull per source).
|
|
/// Pulled chunks arrive through the SAME inbound path as live ships
|
|
/// ([`recv_segment`](Self::recv_segment)), so the apply pipeline is
|
|
/// identical for both.
|
|
///
|
|
/// The default is a no-op: in-process transports deliver everything
|
|
/// in-order and never rotate a tail, so they have nothing to pull.
|
|
fn request_catchup(&self, _from_shard: ShardId, _from_seqno: u64) {}
|
|
|
|
/// The destination peer's last self-reported applied seqno (for this
|
|
/// node's stream), or `0` when unknown — a monotonic HINT, not a
|
|
/// guarantee.
|
|
///
|
|
/// The gRPC transport learns it from every `ShipSegmentResponse` (m11p2:
|
|
/// the ack carries the follower's applied seqno); the ship queue folds it
|
|
/// into its acked frontier so retries of data the follower already holds
|
|
/// are pruned and heal needs no separate status fetch.
|
|
fn peer_applied_hint(&self, _peer: ShardId) -> u64 {
|
|
0
|
|
}
|
|
|
|
/// The shard identity of this transport endpoint.
|
|
fn local_shard(&self) -> ShardId;
|
|
}
|
|
|
|
/// Blanket implementation of `Transport` for `Arc<dyn Transport>`.
|
|
///
|
|
/// Allows type-erased `Arc<dyn Transport>` values to be passed to generic
|
|
/// APIs like `spawn_shipper<T: Transport>` and `spawn_receiver<T: Transport>`.
|
|
impl Transport for std::sync::Arc<dyn Transport> {
|
|
fn send_segment(&self, to: ShardId, payload: WalSegmentPayload) -> Result<(), TransportError> {
|
|
(**self).send_segment(to, payload)
|
|
}
|
|
|
|
fn recv_segment(&self) -> Option<WalSegmentPayload> {
|
|
(**self).recv_segment()
|
|
}
|
|
|
|
fn try_recv_segment(&self) -> Option<WalSegmentPayload> {
|
|
// Forward explicitly: falling through to the trait default would
|
|
// silently disable receiver-side coalescing for type-erased callers.
|
|
(**self).try_recv_segment()
|
|
}
|
|
|
|
fn request_catchup(&self, from_shard: ShardId, from_seqno: u64) {
|
|
// Forward explicitly: the trait default no-op would silently disable
|
|
// pull-based catch-up for type-erased callers.
|
|
(**self).request_catchup(from_shard, from_seqno);
|
|
}
|
|
|
|
fn peer_applied_hint(&self, peer: ShardId) -> u64 {
|
|
// Forward explicitly: the trait default `0` would silently disable
|
|
// the follower-reported acked fold for type-erased callers.
|
|
(**self).peer_applied_hint(peer)
|
|
}
|
|
|
|
fn local_shard(&self) -> ShardId {
|
|
(**self).local_shard()
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::replication::RegionId;
|
|
|
|
#[test]
|
|
fn transport_error_display_unknown_peer() {
|
|
let err = TransportError::UnknownPeer(ShardId(5));
|
|
assert!(err.to_string().contains("unknown peer shard"), "got: {err}");
|
|
}
|
|
|
|
#[test]
|
|
fn transport_error_display_closed() {
|
|
let err = TransportError::Closed;
|
|
assert!(err.to_string().contains("closed"), "got: {err}");
|
|
}
|
|
|
|
#[test]
|
|
fn transport_error_display_permanent() {
|
|
let err = TransportError::Permanent {
|
|
reason: "mutually-distrusted TLS chain".into(),
|
|
};
|
|
let s = err.to_string();
|
|
assert!(s.contains("permanent"), "got: {s}");
|
|
assert!(s.contains("mutually-distrusted TLS chain"), "got: {s}");
|
|
}
|
|
|
|
#[test]
|
|
fn transport_error_display_payload_too_large() {
|
|
let err = TransportError::PayloadTooLarge { size: 100, max: 50 };
|
|
let s = err.to_string();
|
|
assert!(s.contains("100"), "got: {s}");
|
|
assert!(s.contains("50"), "got: {s}");
|
|
}
|
|
|
|
#[test]
|
|
fn wal_segment_payload_construction() {
|
|
let payload = WalSegmentPayload {
|
|
id: WalSegmentId::new(RegionId::SINGLE, ShardId::SINGLE, 42),
|
|
bytes: vec![1, 2, 3],
|
|
event_count: 1,
|
|
leader_last_seq: 42,
|
|
stream_baseline: 0,
|
|
};
|
|
assert_eq!(payload.id.seqno, 42);
|
|
assert_eq!(payload.bytes.len(), 3);
|
|
assert_eq!(payload.event_count, 1);
|
|
assert_eq!(payload.leader_last_seq, 42);
|
|
}
|
|
}
|