m11p1 — decoupled ack/ship path: staged writes (seqno+WAL+relay-push, microseconds) separate from group-commit fsync; ShipQueue batches+windows outbound segments; receiver coalesces inbound chunks before applying. Adds first tidaldb_cluster_* metrics. m11p2 — leader WAL is now THE replicated log: fsynced batches feed a bounded WalShipFeed and ship byte-identical to followers; WAL seqnos survive restarts (relay-reset hazard gone). Item metadata and embeddings journal kind-1/2 blob records on the same stream as signals; the m8p10 HTTP broadcast is deleted. StreamSegments catch-up is follower-pulled via server-streaming RPC, triggered on gap detection, follower boot, and leader heal nudge. Promote carries a stream baseline so peers skip pre-stream history.
265 lines
11 KiB
Rust
265 lines
11 KiB
Rust
//! gRPC client connection pool for shipping WAL segments to peers.
|
|
|
|
use std::collections::HashMap;
|
|
|
|
use tidaldb::replication::{shard::ShardId, transport::WalSegmentPayload};
|
|
use tonic::transport::Channel;
|
|
|
|
use crate::{
|
|
circuit_breaker::CircuitBreaker,
|
|
config::GrpcTransportConfig,
|
|
error::GrpcTransportError,
|
|
proto::{ShipSegmentRequest, wal_shipping_client::WalShippingClient},
|
|
tls,
|
|
};
|
|
|
|
/// A connection to a single peer shard.
|
|
struct PeerConnection {
|
|
client: WalShippingClient<Channel>,
|
|
circuit_breaker: CircuitBreaker,
|
|
}
|
|
|
|
/// Connection pool managing one gRPC channel per peer shard.
|
|
pub struct PeerPool {
|
|
peers: HashMap<ShardId, PeerConnection>,
|
|
}
|
|
|
|
impl PeerPool {
|
|
/// Build a pool with lazy connections to all configured peers.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns [`GrpcTransportError`] if the config fails its numeric invariant
|
|
/// check ([`GrpcTransportConfig::validate`]), if TLS is unset but plaintext
|
|
/// was not explicitly opted into (`insecure == false`), if a peer URI is
|
|
/// invalid, or if the client TLS configuration cannot be built.
|
|
pub fn new(config: &GrpcTransportConfig) -> Result<Self, GrpcTransportError> {
|
|
config.validate()?;
|
|
|
|
// Refuse to build a PLAINTEXT channel unless plaintext was explicitly
|
|
// opted into. With no TLS config the scheme below falls back to `http`,
|
|
// so silently honoring `insecure == false` would ship WAL bytes in the
|
|
// clear while the operator believes the peer is authenticated. The server
|
|
// (server::start_server) rejects this same asymmetry; the client must
|
|
// match it so a half-configured deployment fails loudly on BOTH ends
|
|
// instead of one end quietly downgrading to plaintext. `TlsConfig` is the
|
|
// right variant: it is classified `is_permanent` (see error.rs), so the
|
|
// shipper quarantines rather than retrying a config that cannot self-heal.
|
|
if config.tls.is_none() && !config.insecure {
|
|
return Err(GrpcTransportError::TlsConfig(
|
|
"TLS not configured and insecure not set; refusing to ship WAL segments over \
|
|
plaintext"
|
|
.into(),
|
|
));
|
|
}
|
|
|
|
let mut peers = HashMap::new();
|
|
|
|
for (&shard_id, addr) in &config.peers {
|
|
let scheme = if config.tls.is_some() {
|
|
"https"
|
|
} else {
|
|
"http"
|
|
};
|
|
let uri = format!("{scheme}://{addr}");
|
|
|
|
// Build the endpoint with hard connect/request timeouts and HTTP/2
|
|
// keep-alive so a stalled peer fails fast (transient error → circuit
|
|
// breaker) instead of hanging the single-threaded WAL shipper. These
|
|
// apply on both the plaintext and TLS paths.
|
|
let mut endpoint = Channel::from_shared(uri)
|
|
.map_err(|e| GrpcTransportError::Internal(format!("invalid URI: {e}")))?
|
|
.connect_timeout(config.connect_timeout)
|
|
.timeout(config.request_timeout)
|
|
.tcp_keepalive(Some(config.keep_alive_interval))
|
|
.http2_keep_alive_interval(config.keep_alive_interval)
|
|
.keep_alive_timeout(config.keep_alive_timeout)
|
|
.keep_alive_while_idle(true);
|
|
|
|
// Configure client TLS if available.
|
|
if let Some(ref tls_config) = config.tls {
|
|
let client_tls = tls::client_tls_config(tls_config)?;
|
|
endpoint = endpoint
|
|
.tls_config(client_tls)
|
|
.map_err(GrpcTransportError::TonicTransport)?;
|
|
}
|
|
|
|
let channel = endpoint.connect_lazy();
|
|
// Raise tonic's default 4 MiB codec limits to the configured max
|
|
// payload on BOTH directions. The client is the ENCODER for the
|
|
// outbound ShipSegmentRequest (and the DECODER for any future
|
|
// server-streaming response). Without this the client encoder trips
|
|
// FIRST on a full-size WAL segment (16 MiB default, up to 64 MiB)
|
|
// before bytes ever leave the process, surfacing as a codec error
|
|
// the shipper would retry forever. Both ends must agree on the same
|
|
// ceiling the application already validates against.
|
|
let client = WalShippingClient::new(channel)
|
|
.max_encoding_message_size(config.max_payload_bytes)
|
|
.max_decoding_message_size(config.max_payload_bytes);
|
|
let circuit_breaker = CircuitBreaker::new(
|
|
config.circuit_breaker_threshold,
|
|
config.circuit_breaker_reset,
|
|
);
|
|
|
|
peers.insert(
|
|
shard_id,
|
|
PeerConnection {
|
|
client,
|
|
circuit_breaker,
|
|
},
|
|
);
|
|
}
|
|
|
|
Ok(Self { peers })
|
|
}
|
|
|
|
/// Send a WAL segment to a peer shard.
|
|
///
|
|
/// On success returns the peer's self-reported applied seqno from the ack
|
|
/// (`0` = unknown / older peer) — the m11p2 piggyback the ship queue folds
|
|
/// into its acked frontier.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns [`GrpcTransportError`] if the peer is unknown, the circuit breaker
|
|
/// is open, the follower signals backpressure, or the gRPC call fails.
|
|
// `peer` is a plain `&PeerConnection` borrow into `self.peers` (no Drop of
|
|
// its own) used both before the await (clone the client) and after (record
|
|
// success/failure on its circuit breaker), so it legitimately spans the
|
|
// whole fn — significant_drop_tightening is a false positive for a borrow.
|
|
#[allow(clippy::significant_drop_tightening)]
|
|
pub async fn send_to(
|
|
&self,
|
|
shard: ShardId,
|
|
payload: WalSegmentPayload,
|
|
) -> Result<u64, GrpcTransportError> {
|
|
let peer = self
|
|
.peers
|
|
.get(&shard)
|
|
.ok_or(GrpcTransportError::PeerUnreachable(shard))?;
|
|
|
|
// Check circuit breaker.
|
|
peer.circuit_breaker
|
|
.check()
|
|
.map_err(|_| GrpcTransportError::CircuitOpen(shard))?;
|
|
|
|
let request: ShipSegmentRequest = payload.into();
|
|
|
|
// Clone the client (tonic clients are cheap clones over a shared channel).
|
|
let mut client = peer.client.clone();
|
|
|
|
match client.ship_segment(request).await {
|
|
Ok(response) => {
|
|
// The follower signals backpressure with `accepted=false` when
|
|
// its inbound channel is full: the segment was NOT enqueued.
|
|
// This is normal flow control from a healthy-but-busy peer, not
|
|
// a connection failure — so it must NOT count toward opening the
|
|
// circuit breaker (that would convert transient congestion into
|
|
// a self-inflicted `circuit_breaker_reset` replication stall,
|
|
// amplifying follower lag exactly when the leader must keep
|
|
// shipping). We route it through `record_backpressure`, which
|
|
// leaves Closed/Open breaker state untouched (it only resolves an
|
|
// outstanding half-open probe), and return a transient
|
|
// `SegmentRejected` so the shipper does not advance its cursor
|
|
// and retries on the next poll. Only the genuine transport error
|
|
// arm below calls `record_failure`.
|
|
let inner = response.into_inner();
|
|
if inner.accepted {
|
|
peer.circuit_breaker.record_success();
|
|
Ok(inner.applied_seqno)
|
|
} else {
|
|
peer.circuit_breaker.record_backpressure();
|
|
Err(GrpcTransportError::SegmentRejected(shard))
|
|
}
|
|
}
|
|
Err(status) => {
|
|
peer.circuit_breaker.record_failure();
|
|
Err(GrpcTransportError::Grpc(Box::new(status)))
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Open a `StreamSegments` catch-up stream from `shard` starting at
|
|
/// `from_seqno` (m11p2: follower-pulled catch-up over the leader's
|
|
/// durable WAL).
|
|
///
|
|
/// Deliberately bypasses the circuit breaker: pulls are already
|
|
/// rate-limited and single-flight per shard at the transport layer, and a
|
|
/// breaker opened by live-ship backpressure must not also starve the very
|
|
/// mechanism that drains the backlog.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns [`GrpcTransportError`] if the peer is unknown or the stream
|
|
/// cannot be opened.
|
|
pub async fn stream_from(
|
|
&self,
|
|
shard: ShardId,
|
|
from_seqno: u64,
|
|
) -> Result<tonic::Streaming<ShipSegmentRequest>, GrpcTransportError> {
|
|
let peer = self
|
|
.peers
|
|
.get(&shard)
|
|
.ok_or(GrpcTransportError::PeerUnreachable(shard))?;
|
|
let mut client = peer.client.clone();
|
|
let request = crate::proto::StreamRequest {
|
|
shard_id: u32::from(shard.0),
|
|
from_seqno,
|
|
};
|
|
match client.stream_segments(request).await {
|
|
Ok(response) => Ok(response.into_inner()),
|
|
Err(status) => Err(GrpcTransportError::Grpc(Box::new(status))),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::config::GrpcTransportConfig;
|
|
|
|
#[test]
|
|
fn rejects_plaintext_when_insecure_unset() {
|
|
// insecure=false with no TLS config must error rather than silently
|
|
// building a plaintext channel — symmetric with the server's rejection.
|
|
// No peers means the early-return guard fires before any endpoint is
|
|
// built, so this needs no tokio reactor.
|
|
let config = GrpcTransportConfig {
|
|
tls: None,
|
|
insecure: false,
|
|
..GrpcTransportConfig::default()
|
|
};
|
|
// `let-else` instead of `expect_err` so the test does not require
|
|
// `PeerPool: Debug` (it holds tonic channel handles that are not Debug).
|
|
let Err(err) = PeerPool::new(&config) else {
|
|
panic!("plaintext without opt-in must be rejected");
|
|
};
|
|
assert!(
|
|
matches!(err, GrpcTransportError::TlsConfig(_)),
|
|
"expected a permanent TlsConfig error, got {err:?}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn allows_plaintext_when_insecure_set() {
|
|
// The explicit opt-in path must still succeed (no peers => no endpoints).
|
|
let config = GrpcTransportConfig {
|
|
tls: None,
|
|
insecure: true,
|
|
..GrpcTransportConfig::default()
|
|
};
|
|
assert!(PeerPool::new(&config).is_ok());
|
|
}
|
|
|
|
#[test]
|
|
fn rejects_invalid_config() {
|
|
// PeerPool::new must run the config invariant check; a zero capacity is
|
|
// caught here instead of panicking later in the mpsc channel build.
|
|
let config = GrpcTransportConfig {
|
|
channel_capacity: 0,
|
|
..GrpcTransportConfig::default()
|
|
};
|
|
assert!(PeerPool::new(&config).is_err());
|
|
}
|
|
}
|