tidaldb/tidal/src/replication/transport.rs
jx12n 225751d34d feat(m11): WAL-as-stream replication + perf floor (m11p1+m11p2)
m11p1 — decoupled ack/ship path: staged writes (seqno+WAL+relay-push,
microseconds) separate from group-commit fsync; ShipQueue batches+windows
outbound segments; receiver coalesces inbound chunks before applying.
Adds first tidaldb_cluster_* metrics.

m11p2 — leader WAL is now THE replicated log: fsynced batches feed a
bounded WalShipFeed and ship byte-identical to followers; WAL seqnos
survive restarts (relay-reset hazard gone). Item metadata and embeddings
journal kind-1/2 blob records on the same stream as signals; the m8p10
HTTP broadcast is deleted. StreamSegments catch-up is follower-pulled via
server-streaming RPC, triggered on gap detection, follower boot, and
leader heal nudge. Promote carries a stream baseline so peers skip
pre-stream history.
2026-06-11 09:10:06 -06:00

264 lines
12 KiB
Rust

//! Transport abstraction for WAL segment shipping between shards.
//!
//! The [`Transport`] trait defines how segments are sent and received between
//! nodes. Implementations include:
//! - [`super::in_process::InProcessTransport`]: crossbeam channels for tests
//! and single-process multi-shard setups.
//! - (future) gRPC or TCP transport for multi-node deployments.
use crate::replication::{segment_id::WalSegmentId, shard::ShardId};
/// A WAL segment ready for shipping to a peer shard.
///
/// Contains the raw batch bytes from a sealed WAL segment file, plus
/// metadata for tracking and idempotent replay on the receiver.
pub struct WalSegmentPayload {
/// Globally unique identifier for this segment.
pub id: WalSegmentId,
/// Raw WAL bytes (one or more encoded batches).
pub bytes: Vec<u8>,
/// Total number of events across all batches in this segment.
pub event_count: u64,
/// The segment's authoritative last WAL sequence number, as known by the
/// leader **before** any community-overlay filtering.
///
/// The shipper computes this from the *original* (unfiltered) segment bytes
/// (`first_seq + total_event_count - 1`), so it is correct even when the
/// shipped `bytes` are an all-local empty filtration that decodes to zero
/// batches. The receiver feeds it to the replication lag gauge's leader
/// high-water-mark independently of how many batches survive filtering, so
/// an all-local (empty) segment does not make the gauge under-report the
/// leader's progress (obs-REPL-1). `0` means "unknown / no events" and the
/// receiver falls back to the per-batch boundaries it decodes.
pub leader_last_seq: u64,
/// The shipping stream's baseline: the WAL seqno at which this leader's
/// stream STARTED (0 = from the beginning, the default). Non-zero only on
/// catch-up stream chunks from a promoted leader: seqnos at or below the
/// baseline are pre-stream history (data the follower already holds via
/// the previous leader's stream), so the receiver jumps its frontier to
/// the baseline instead of waiting for a gap that will never close.
pub stream_baseline: u64,
}
/// Errors that can occur during WAL segment transport.
#[derive(Debug, thiserror::Error)]
pub enum TransportError {
/// The target shard is not known to this transport.
#[error("unknown peer shard: {0}")]
UnknownPeer(ShardId),
/// The transport channel has been closed (peer disconnected or shutdown).
///
/// This is a **transient** failure: the peer is known but temporarily
/// unable to accept the segment (process down, circuit breaker open,
/// follower backpressure). The shipper leaves the peer's high-water-mark
/// untouched and re-ships the same seqno on the next poll, preserving a
/// gap-free in-order stream.
#[error("transport channel closed")]
Closed,
/// The send failed in a way that re-shipping the **same** segment will
/// never resolve: a mutually-distrusted TLS chain, a misconfigured CA, an
/// authentication/authorization rejection, a payload the wire codec will
/// never accept, or an RPC the peer does not implement.
///
/// Unlike [`Closed`](Self::Closed), retrying a `Permanent` failure against
/// the same seqno is an **infinite silent stall** — nothing about the
/// segment becomes shippable by trying again until an operator changes the
/// peer/config. The shipper must therefore quarantine the peer and surface
/// an actionable error instead of re-shipping forever. `reason` carries the
/// human-readable cause (from the underlying transport) for logs/alerts.
#[error("permanent transport failure: {reason}")]
Permanent {
/// Human-readable cause of the permanent failure (TLS/auth/codec/etc.).
reason: String,
},
/// The payload exceeds the transport's maximum size limit.
#[error("payload too large: {size} bytes (max {max})")]
PayloadTooLarge {
/// Actual payload size in bytes.
size: usize,
/// Maximum allowed payload size in bytes.
max: usize,
},
}
/// Trait for shipping WAL segments between shards.
///
/// Implementations must be `Send + Sync + 'static` to allow sharing across
/// the shipper and receiver threads.
///
/// # Contract
///
/// - `send_segment` delivers best-effort and **MAY block** the calling thread up
/// to an implementation-defined timeout. It is **not** a non-blocking call: the
/// gRPC implementation (`tidal_net::GrpcTransport`) bridges its async client to
/// this sync trait with `runtime.block_on(pool.send_to(..))`, which parks the
/// caller for up to its `request_timeout` (default ~10s) per call — connect plus
/// send plus response. It MUST therefore be called only from a dedicated WAL
/// shipper thread (a `std::thread`, never inside a tokio runtime or any
/// async/latency-sensitive context). The in-process implementation returns
/// promptly, but callers must not depend on that — code to the blocking contract.
/// - `recv_segment` blocks until a segment is available or the transport shuts down.
/// - `local_shard` returns this node's shard identity.
pub trait Transport: Send + Sync + 'static {
/// Ship a WAL segment payload to a peer shard.
///
/// **Blocking:** this call MAY block the caller up to an implementation-defined
/// timeout (the gRPC implementation blocks up to `request_timeout`, ~10s). Call
/// it only from a dedicated shipper thread, never from an async or
/// latency-sensitive context. See the trait-level `# Contract` for the rationale
/// and the `GrpcTransport` `block_on` bridge.
///
/// Blanket implementation for `Arc<dyn Transport>` so callers can use
/// type-erased transports with the generic `spawn_shipper` / `spawn_receiver` APIs.
///
/// # Errors
///
/// - `TransportError::UnknownPeer` if `to` is not a recognized shard.
/// - `TransportError::Closed` if the peer's receive channel is disconnected
/// (transient — the shipper retries the same seqno next poll).
/// - `TransportError::Permanent` if the send can never succeed for this
/// segment without a config/peer change (bad TLS/CA, auth rejection,
/// malformed payload, unimplemented RPC) — the shipper quarantines the
/// peer instead of retrying forever.
/// - `TransportError::PayloadTooLarge` if the payload exceeds the size limit.
fn send_segment(&self, to: ShardId, payload: WalSegmentPayload) -> Result<(), TransportError>;
/// Block until a segment payload is received, or return `None` on shutdown.
///
/// Returns `None` when the transport is closed and no more segments will
/// arrive. The receiver thread should exit when this returns `None`.
fn recv_segment(&self) -> Option<WalSegmentPayload>;
/// Return an immediately-available segment payload WITHOUT blocking, or
/// `None` when the inbound queue is empty (or the transport does not
/// support non-blocking receives — the conservative default).
///
/// The segment receiver uses this to drain a backlog after each blocking
/// `recv_segment` and apply the drained segments through ONE shared
/// follower group-commit (m11p1): without coalescing, every segment pays
/// a solo fsync (~ms-scale on real volumes), capping follower apply
/// throughput at `events-per-segment / fsync-cost` and letting lag grow
/// without bound under sustained leader load.
fn try_recv_segment(&self) -> Option<WalSegmentPayload> {
None
}
/// Ask the source shard for a catch-up stream from `from_seqno` (m11p2:
/// follower-pulled `StreamSegments` over the leader's durable WAL).
///
/// Called by the segment receiver when an apply round leaves a gap below
/// a parked range — data older than the leader's in-memory ship tail that
/// push-based shipping can no longer deliver. Implementations MUST be
/// non-blocking (fire the pull in the background), rate-limited, and
/// de-duplicated per shard (at most one in-flight pull per source).
/// Pulled chunks arrive through the SAME inbound path as live ships
/// ([`recv_segment`](Self::recv_segment)), so the apply pipeline is
/// identical for both.
///
/// The default is a no-op: in-process transports deliver everything
/// in-order and never rotate a tail, so they have nothing to pull.
fn request_catchup(&self, _from_shard: ShardId, _from_seqno: u64) {}
/// The destination peer's last self-reported applied seqno (for this
/// node's stream), or `0` when unknown — a monotonic HINT, not a
/// guarantee.
///
/// The gRPC transport learns it from every `ShipSegmentResponse` (m11p2:
/// the ack carries the follower's applied seqno); the ship queue folds it
/// into its acked frontier so retries of data the follower already holds
/// are pruned and heal needs no separate status fetch.
fn peer_applied_hint(&self, _peer: ShardId) -> u64 {
0
}
/// The shard identity of this transport endpoint.
fn local_shard(&self) -> ShardId;
}
/// Blanket implementation of `Transport` for `Arc<dyn Transport>`.
///
/// Allows type-erased `Arc<dyn Transport>` values to be passed to generic
/// APIs like `spawn_shipper<T: Transport>` and `spawn_receiver<T: Transport>`.
impl Transport for std::sync::Arc<dyn Transport> {
fn send_segment(&self, to: ShardId, payload: WalSegmentPayload) -> Result<(), TransportError> {
(**self).send_segment(to, payload)
}
fn recv_segment(&self) -> Option<WalSegmentPayload> {
(**self).recv_segment()
}
fn try_recv_segment(&self) -> Option<WalSegmentPayload> {
// Forward explicitly: falling through to the trait default would
// silently disable receiver-side coalescing for type-erased callers.
(**self).try_recv_segment()
}
fn request_catchup(&self, from_shard: ShardId, from_seqno: u64) {
// Forward explicitly: the trait default no-op would silently disable
// pull-based catch-up for type-erased callers.
(**self).request_catchup(from_shard, from_seqno);
}
fn peer_applied_hint(&self, peer: ShardId) -> u64 {
// Forward explicitly: the trait default `0` would silently disable
// the follower-reported acked fold for type-erased callers.
(**self).peer_applied_hint(peer)
}
fn local_shard(&self) -> ShardId {
(**self).local_shard()
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::replication::RegionId;
#[test]
fn transport_error_display_unknown_peer() {
let err = TransportError::UnknownPeer(ShardId(5));
assert!(err.to_string().contains("unknown peer shard"), "got: {err}");
}
#[test]
fn transport_error_display_closed() {
let err = TransportError::Closed;
assert!(err.to_string().contains("closed"), "got: {err}");
}
#[test]
fn transport_error_display_permanent() {
let err = TransportError::Permanent {
reason: "mutually-distrusted TLS chain".into(),
};
let s = err.to_string();
assert!(s.contains("permanent"), "got: {s}");
assert!(s.contains("mutually-distrusted TLS chain"), "got: {s}");
}
#[test]
fn transport_error_display_payload_too_large() {
let err = TransportError::PayloadTooLarge { size: 100, max: 50 };
let s = err.to_string();
assert!(s.contains("100"), "got: {s}");
assert!(s.contains("50"), "got: {s}");
}
#[test]
fn wal_segment_payload_construction() {
let payload = WalSegmentPayload {
id: WalSegmentId::new(RegionId::SINGLE, ShardId::SINGLE, 42),
bytes: vec![1, 2, 3],
event_count: 1,
leader_last_seq: 42,
stream_baseline: 0,
};
assert_eq!(payload.id.seqno, 42);
assert_eq!(payload.bytes.len(), 3);
assert_eq!(payload.event_count, 1);
assert_eq!(payload.leader_last_seq, 42);
}
}