//! Transport abstraction for WAL segment shipping between shards. //! //! The [`Transport`] trait defines how segments are sent and received between //! nodes. Implementations include: //! - [`super::in_process::InProcessTransport`]: crossbeam channels for tests //! and single-process multi-shard setups. //! - (future) gRPC or TCP transport for multi-node deployments. use crate::replication::{segment_id::WalSegmentId, shard::ShardId}; /// A WAL segment ready for shipping to a peer shard. /// /// Contains the raw batch bytes from a sealed WAL segment file, plus /// metadata for tracking and idempotent replay on the receiver. pub struct WalSegmentPayload { /// Globally unique identifier for this segment. pub id: WalSegmentId, /// Raw WAL bytes (one or more encoded batches). pub bytes: Vec, /// Total number of events across all batches in this segment. pub event_count: u64, /// The segment's authoritative last WAL sequence number, as known by the /// leader **before** any community-overlay filtering. /// /// The shipper computes this from the *original* (unfiltered) segment bytes /// (`first_seq + total_event_count - 1`), so it is correct even when the /// shipped `bytes` are an all-local empty filtration that decodes to zero /// batches. The receiver feeds it to the replication lag gauge's leader /// high-water-mark independently of how many batches survive filtering, so /// an all-local (empty) segment does not make the gauge under-report the /// leader's progress (obs-REPL-1). `0` means "unknown / no events" and the /// receiver falls back to the per-batch boundaries it decodes. pub leader_last_seq: u64, /// The shipping stream's baseline: the WAL seqno at which this leader's /// stream STARTED (0 = from the beginning, the default). Non-zero only on /// catch-up stream chunks from a promoted leader: seqnos at or below the /// baseline are pre-stream history (data the follower already holds via /// the previous leader's stream), so the receiver jumps its frontier to /// the baseline instead of waiting for a gap that will never close. pub stream_baseline: u64, } /// Errors that can occur during WAL segment transport. #[derive(Debug, thiserror::Error)] pub enum TransportError { /// The target shard is not known to this transport. #[error("unknown peer shard: {0}")] UnknownPeer(ShardId), /// The transport channel has been closed (peer disconnected or shutdown). /// /// This is a **transient** failure: the peer is known but temporarily /// unable to accept the segment (process down, circuit breaker open, /// follower backpressure). The shipper leaves the peer's high-water-mark /// untouched and re-ships the same seqno on the next poll, preserving a /// gap-free in-order stream. #[error("transport channel closed")] Closed, /// The send failed in a way that re-shipping the **same** segment will /// never resolve: a mutually-distrusted TLS chain, a misconfigured CA, an /// authentication/authorization rejection, a payload the wire codec will /// never accept, or an RPC the peer does not implement. /// /// Unlike [`Closed`](Self::Closed), retrying a `Permanent` failure against /// the same seqno is an **infinite silent stall** — nothing about the /// segment becomes shippable by trying again until an operator changes the /// peer/config. The shipper must therefore quarantine the peer and surface /// an actionable error instead of re-shipping forever. `reason` carries the /// human-readable cause (from the underlying transport) for logs/alerts. #[error("permanent transport failure: {reason}")] Permanent { /// Human-readable cause of the permanent failure (TLS/auth/codec/etc.). reason: String, }, /// The payload exceeds the transport's maximum size limit. #[error("payload too large: {size} bytes (max {max})")] PayloadTooLarge { /// Actual payload size in bytes. size: usize, /// Maximum allowed payload size in bytes. max: usize, }, } /// Trait for shipping WAL segments between shards. /// /// Implementations must be `Send + Sync + 'static` to allow sharing across /// the shipper and receiver threads. /// /// # Contract /// /// - `send_segment` delivers best-effort and **MAY block** the calling thread up /// to an implementation-defined timeout. It is **not** a non-blocking call: the /// gRPC implementation (`tidal_net::GrpcTransport`) bridges its async client to /// this sync trait with `runtime.block_on(pool.send_to(..))`, which parks the /// caller for up to its `request_timeout` (default ~10s) per call — connect plus /// send plus response. It MUST therefore be called only from a dedicated WAL /// shipper thread (a `std::thread`, never inside a tokio runtime or any /// async/latency-sensitive context). The in-process implementation returns /// promptly, but callers must not depend on that — code to the blocking contract. /// - `recv_segment` blocks until a segment is available or the transport shuts down. /// - `local_shard` returns this node's shard identity. pub trait Transport: Send + Sync + 'static { /// Ship a WAL segment payload to a peer shard. /// /// **Blocking:** this call MAY block the caller up to an implementation-defined /// timeout (the gRPC implementation blocks up to `request_timeout`, ~10s). Call /// it only from a dedicated shipper thread, never from an async or /// latency-sensitive context. See the trait-level `# Contract` for the rationale /// and the `GrpcTransport` `block_on` bridge. /// /// Blanket implementation for `Arc` so callers can use /// type-erased transports with the generic `spawn_shipper` / `spawn_receiver` APIs. /// /// # Errors /// /// - `TransportError::UnknownPeer` if `to` is not a recognized shard. /// - `TransportError::Closed` if the peer's receive channel is disconnected /// (transient — the shipper retries the same seqno next poll). /// - `TransportError::Permanent` if the send can never succeed for this /// segment without a config/peer change (bad TLS/CA, auth rejection, /// malformed payload, unimplemented RPC) — the shipper quarantines the /// peer instead of retrying forever. /// - `TransportError::PayloadTooLarge` if the payload exceeds the size limit. fn send_segment(&self, to: ShardId, payload: WalSegmentPayload) -> Result<(), TransportError>; /// Block until a segment payload is received, or return `None` on shutdown. /// /// Returns `None` when the transport is closed and no more segments will /// arrive. The receiver thread should exit when this returns `None`. fn recv_segment(&self) -> Option; /// Return an immediately-available segment payload WITHOUT blocking, or /// `None` when the inbound queue is empty (or the transport does not /// support non-blocking receives — the conservative default). /// /// The segment receiver uses this to drain a backlog after each blocking /// `recv_segment` and apply the drained segments through ONE shared /// follower group-commit (m11p1): without coalescing, every segment pays /// a solo fsync (~ms-scale on real volumes), capping follower apply /// throughput at `events-per-segment / fsync-cost` and letting lag grow /// without bound under sustained leader load. fn try_recv_segment(&self) -> Option { None } /// Ask the source shard for a catch-up stream from `from_seqno` (m11p2: /// follower-pulled `StreamSegments` over the leader's durable WAL). /// /// Called by the segment receiver when an apply round leaves a gap below /// a parked range — data older than the leader's in-memory ship tail that /// push-based shipping can no longer deliver. Implementations MUST be /// non-blocking (fire the pull in the background), rate-limited, and /// de-duplicated per shard (at most one in-flight pull per source). /// Pulled chunks arrive through the SAME inbound path as live ships /// ([`recv_segment`](Self::recv_segment)), so the apply pipeline is /// identical for both. /// /// The default is a no-op: in-process transports deliver everything /// in-order and never rotate a tail, so they have nothing to pull. fn request_catchup(&self, _from_shard: ShardId, _from_seqno: u64) {} /// The destination peer's last self-reported applied seqno (for this /// node's stream), or `0` when unknown — a monotonic HINT, not a /// guarantee. /// /// The gRPC transport learns it from every `ShipSegmentResponse` (m11p2: /// the ack carries the follower's applied seqno); the ship queue folds it /// into its acked frontier so retries of data the follower already holds /// are pruned and heal needs no separate status fetch. fn peer_applied_hint(&self, _peer: ShardId) -> u64 { 0 } /// The shard identity of this transport endpoint. fn local_shard(&self) -> ShardId; } /// Blanket implementation of `Transport` for `Arc`. /// /// Allows type-erased `Arc` values to be passed to generic /// APIs like `spawn_shipper` and `spawn_receiver`. impl Transport for std::sync::Arc { fn send_segment(&self, to: ShardId, payload: WalSegmentPayload) -> Result<(), TransportError> { (**self).send_segment(to, payload) } fn recv_segment(&self) -> Option { (**self).recv_segment() } fn try_recv_segment(&self) -> Option { // Forward explicitly: falling through to the trait default would // silently disable receiver-side coalescing for type-erased callers. (**self).try_recv_segment() } fn request_catchup(&self, from_shard: ShardId, from_seqno: u64) { // Forward explicitly: the trait default no-op would silently disable // pull-based catch-up for type-erased callers. (**self).request_catchup(from_shard, from_seqno); } fn peer_applied_hint(&self, peer: ShardId) -> u64 { // Forward explicitly: the trait default `0` would silently disable // the follower-reported acked fold for type-erased callers. (**self).peer_applied_hint(peer) } fn local_shard(&self) -> ShardId { (**self).local_shard() } } #[cfg(test)] mod tests { use super::*; use crate::replication::RegionId; #[test] fn transport_error_display_unknown_peer() { let err = TransportError::UnknownPeer(ShardId(5)); assert!(err.to_string().contains("unknown peer shard"), "got: {err}"); } #[test] fn transport_error_display_closed() { let err = TransportError::Closed; assert!(err.to_string().contains("closed"), "got: {err}"); } #[test] fn transport_error_display_permanent() { let err = TransportError::Permanent { reason: "mutually-distrusted TLS chain".into(), }; let s = err.to_string(); assert!(s.contains("permanent"), "got: {s}"); assert!(s.contains("mutually-distrusted TLS chain"), "got: {s}"); } #[test] fn transport_error_display_payload_too_large() { let err = TransportError::PayloadTooLarge { size: 100, max: 50 }; let s = err.to_string(); assert!(s.contains("100"), "got: {s}"); assert!(s.contains("50"), "got: {s}"); } #[test] fn wal_segment_payload_construction() { let payload = WalSegmentPayload { id: WalSegmentId::new(RegionId::SINGLE, ShardId::SINGLE, 42), bytes: vec![1, 2, 3], event_count: 1, leader_last_seq: 42, stream_baseline: 0, }; assert_eq!(payload.id.seqno, 42); assert_eq!(payload.bytes.len(), 3); assert_eq!(payload.event_count, 1); assert_eq!(payload.leader_last_seq, 42); } }