95461d3cf8
Kind-3 term markers in the WAL stream, STREAM-relative vote frontiers, heartbeat-only divergence detection + quarantine, and fenced promote. Elections converge in 0.6–1.0s; zero acked-write loss across all kill points. Closes G5 (leaderless recovery) from the v0.9 wave.
239 lines
9.8 KiB
Rust
239 lines
9.8 KiB
Rust
//! Error types for the gRPC transport layer.
|
|
|
|
use tidaldb::replication::{shard::ShardId, transport::TransportError};
|
|
|
|
/// Errors specific to the gRPC transport.
|
|
#[derive(Debug, thiserror::Error)]
|
|
pub enum GrpcTransportError {
|
|
/// Circuit breaker is open for this peer.
|
|
#[error("circuit breaker open for peer {0}; will retry after reset period")]
|
|
CircuitOpen(ShardId),
|
|
|
|
/// Peer is not reachable (not in the configured peer map).
|
|
#[error("peer shard {0} not in configuration")]
|
|
PeerUnreachable(ShardId),
|
|
|
|
/// TLS configuration error.
|
|
#[error("TLS configuration error: {0}")]
|
|
TlsConfig(String),
|
|
|
|
/// The follower replied `accepted=false` (its inbound channel was full).
|
|
/// This is follower backpressure: the segment was NOT durably enqueued, so
|
|
/// the shipper must treat it as un-shipped and retry on the next poll.
|
|
#[error("peer {0} rejected segment (accepted=false); follower backpressure")]
|
|
SegmentRejected(ShardId),
|
|
|
|
/// Internal transport error.
|
|
#[error("internal transport error: {0}")]
|
|
Internal(String),
|
|
|
|
/// gRPC status error from tonic.
|
|
#[error("gRPC error: {0}")]
|
|
Grpc(Box<tonic::Status>),
|
|
|
|
/// Tonic transport-level error.
|
|
#[error("transport error: {0}")]
|
|
TonicTransport(#[from] tonic::transport::Error),
|
|
}
|
|
|
|
impl From<tonic::Status> for GrpcTransportError {
|
|
fn from(status: tonic::Status) -> Self {
|
|
Self::Grpc(Box::new(status))
|
|
}
|
|
}
|
|
|
|
impl GrpcTransportError {
|
|
/// Whether this failure is *permanent* — re-shipping the same segment will
|
|
/// keep failing identically until an operator changes configuration.
|
|
///
|
|
/// The shipper's retry loop is correct ONLY for transient failures (peer
|
|
/// down, circuit open, follower backpressure): it leaves the per-peer HWM
|
|
/// untouched and re-sends the same seqno next poll. For a permanent failure
|
|
/// that retry is an infinite silent stall — a mutually-distrusted TLS chain,
|
|
/// a misconfigured CA, an authentication rejection, or a payload the wire
|
|
/// codec will never accept does not become shippable by trying again.
|
|
///
|
|
/// Classification:
|
|
/// - [`TlsConfig`](Self::TlsConfig): local TLS/config material is bad →
|
|
/// permanent.
|
|
/// - [`Grpc`](Self::Grpc): permanent iff the gRPC status code is one the
|
|
/// server will return identically on retry — `Unauthenticated` /
|
|
/// `PermissionDenied` (mTLS / authz rejection), `InvalidArgument` /
|
|
/// `OutOfRange` (the payload itself is malformed), `Unimplemented` (the
|
|
/// RPC does not exist on the peer), `FailedPrecondition` (the m11p4
|
|
/// term fence: a stale-term ship fails identically until leadership
|
|
/// changes — quarantining the peer is right, and the deposed sender's
|
|
/// step-down deactivates the whole queue moments later anyway).
|
|
/// Everything else (`Unavailable`, `ResourceExhausted`,
|
|
/// `DeadlineExceeded`, codec-size on a transient over-large segment, …)
|
|
/// is transient.
|
|
/// - [`TonicTransport`](Self::TonicTransport): a connect/handshake error.
|
|
/// These are predominantly transient (peer not up yet) and are treated as
|
|
/// such; a genuinely permanent handshake fault surfaces as a `Grpc` TLS
|
|
/// status on the next attempt.
|
|
/// - [`PeerUnreachable`](Self::PeerUnreachable): routing error (unknown
|
|
/// shard) — handled separately via `UnknownPeer`, never as a retry.
|
|
/// - All other variants (circuit open, segment rejected, internal): transient.
|
|
#[must_use]
|
|
pub fn is_permanent(&self) -> bool {
|
|
match self {
|
|
Self::TlsConfig(_) => true,
|
|
Self::Grpc(status) => matches!(
|
|
status.code(),
|
|
tonic::Code::Unauthenticated
|
|
| tonic::Code::PermissionDenied
|
|
| tonic::Code::InvalidArgument
|
|
| tonic::Code::OutOfRange
|
|
| tonic::Code::Unimplemented
|
|
| tonic::Code::FailedPrecondition
|
|
),
|
|
Self::CircuitOpen(_)
|
|
| Self::PeerUnreachable(_)
|
|
| Self::SegmentRejected(_)
|
|
| Self::Internal(_)
|
|
| Self::TonicTransport(_) => false,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl From<GrpcTransportError> for TransportError {
|
|
fn from(e: GrpcTransportError) -> Self {
|
|
// Routing failure: the shard is not in our peer map. Surface as the
|
|
// dedicated non-retry variant.
|
|
if let GrpcTransportError::PeerUnreachable(shard) = e {
|
|
return Self::UnknownPeer(shard);
|
|
}
|
|
|
|
// Permanent failures (bad TLS/CA, auth rejection, malformed payload, RPC
|
|
// not implemented on the peer) will fail identically on every retry.
|
|
// Mapping them to the transient `Closed` variant would make the shipper
|
|
// re-ship the same seqno forever — a silent replication stall with no
|
|
// actionable signal. Surface them as the dedicated
|
|
// `TransportError::Permanent` variant (carrying the human-readable
|
|
// cause) so the shipper quarantines the peer and stops advancing it,
|
|
// and log LOUDLY at error! so the fault is observable in logs/alerts.
|
|
if e.is_permanent() {
|
|
tracing::error!(
|
|
error = %e,
|
|
"gRPC transport PERMANENT failure (bad TLS/CA, auth rejection, malformed payload, \
|
|
or unimplemented RPC); retrying this segment will not help — fix peer/config. \
|
|
Quarantining the peer via TransportError::Permanent",
|
|
);
|
|
return Self::Permanent {
|
|
reason: e.to_string(),
|
|
};
|
|
}
|
|
|
|
// Genuinely transient failures map to Closed: circuit-open and follower
|
|
// backpressure are temporary (peer known but unable to accept right
|
|
// now); the remaining internal/transport variants are non-routing and
|
|
// recoverable. Mapping to Closed keeps the shipper from advancing its
|
|
// per-peer cursor so it retries the same seqno, in order, next poll.
|
|
Self::Closed
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn tls_config_failure_is_permanent() {
|
|
let e = GrpcTransportError::TlsConfig("bad CA".into());
|
|
assert!(e.is_permanent(), "bad TLS material never becomes shippable");
|
|
}
|
|
|
|
#[test]
|
|
fn auth_and_malformed_grpc_codes_are_permanent() {
|
|
// mTLS / authz rejection and malformed-payload codes are permanent:
|
|
// the server returns them identically on every retry.
|
|
for code in [
|
|
tonic::Code::Unauthenticated,
|
|
tonic::Code::PermissionDenied,
|
|
tonic::Code::InvalidArgument,
|
|
tonic::Code::OutOfRange,
|
|
tonic::Code::Unimplemented,
|
|
tonic::Code::FailedPrecondition,
|
|
] {
|
|
let e = GrpcTransportError::Grpc(Box::new(tonic::Status::new(code, "x")));
|
|
assert!(e.is_permanent(), "{code:?} must be permanent");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn unavailable_and_exhausted_grpc_codes_are_transient() {
|
|
// A peer that is down or temporarily overloaded WILL accept the same
|
|
// segment once it recovers — these must stay transient so the shipper
|
|
// keeps retrying.
|
|
for code in [
|
|
tonic::Code::Unavailable,
|
|
tonic::Code::ResourceExhausted,
|
|
tonic::Code::DeadlineExceeded,
|
|
tonic::Code::Aborted,
|
|
] {
|
|
let e = GrpcTransportError::Grpc(Box::new(tonic::Status::new(code, "x")));
|
|
assert!(!e.is_permanent(), "{code:?} must be transient");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn circuit_and_backpressure_are_transient() {
|
|
assert!(!GrpcTransportError::CircuitOpen(ShardId(1)).is_permanent());
|
|
assert!(!GrpcTransportError::SegmentRejected(ShardId(1)).is_permanent());
|
|
assert!(!GrpcTransportError::Internal("transient".into()).is_permanent());
|
|
}
|
|
|
|
#[test]
|
|
fn peer_unreachable_maps_to_unknown_peer_not_closed() {
|
|
let mapped: TransportError = GrpcTransportError::PeerUnreachable(ShardId(9)).into();
|
|
assert!(
|
|
matches!(mapped, TransportError::UnknownPeer(ShardId(9))),
|
|
"routing failure must surface as UnknownPeer, not a retry"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn permanent_tls_failure_maps_to_permanent() {
|
|
// A permanent failure (bad TLS material) must surface as the dedicated
|
|
// `Permanent` variant so the shipper quarantines the peer instead of
|
|
// re-shipping the same seqno forever. The reason carries the cause for
|
|
// logs/alerts.
|
|
let mapped: TransportError = GrpcTransportError::TlsConfig("bad CA".into()).into();
|
|
match mapped {
|
|
TransportError::Permanent { reason } => {
|
|
assert!(
|
|
reason.contains("bad CA"),
|
|
"reason must carry cause: {reason}"
|
|
);
|
|
}
|
|
other => panic!("expected Permanent, got {other:?}"),
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn permanent_grpc_code_maps_to_permanent() {
|
|
// Auth/authz and malformed-payload gRPC codes are permanent and must
|
|
// map to `Permanent`, never the transient `Closed`.
|
|
for code in [
|
|
tonic::Code::Unauthenticated,
|
|
tonic::Code::PermissionDenied,
|
|
tonic::Code::InvalidArgument,
|
|
tonic::Code::OutOfRange,
|
|
tonic::Code::Unimplemented,
|
|
] {
|
|
let e = GrpcTransportError::Grpc(Box::new(tonic::Status::new(code, "x")));
|
|
let mapped: TransportError = e.into();
|
|
assert!(
|
|
matches!(mapped, TransportError::Permanent { .. }),
|
|
"{code:?} must map to Permanent, got {mapped:?}"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn transient_failure_maps_to_closed() {
|
|
let mapped: TransportError = GrpcTransportError::SegmentRejected(ShardId(2)).into();
|
|
assert!(matches!(mapped, TransportError::Closed));
|
|
}
|
|
}
|