//! Error types for the gRPC transport layer. use tidaldb::replication::{shard::ShardId, transport::TransportError}; /// Errors specific to the gRPC transport. #[derive(Debug, thiserror::Error)] pub enum GrpcTransportError { /// Circuit breaker is open for this peer. #[error("circuit breaker open for peer {0}; will retry after reset period")] CircuitOpen(ShardId), /// Peer is not reachable (not in the configured peer map). #[error("peer shard {0} not in configuration")] PeerUnreachable(ShardId), /// TLS configuration error. #[error("TLS configuration error: {0}")] TlsConfig(String), /// The follower replied `accepted=false` (its inbound channel was full). /// This is follower backpressure: the segment was NOT durably enqueued, so /// the shipper must treat it as un-shipped and retry on the next poll. #[error("peer {0} rejected segment (accepted=false); follower backpressure")] SegmentRejected(ShardId), /// Internal transport error. #[error("internal transport error: {0}")] Internal(String), /// gRPC status error from tonic. #[error("gRPC error: {0}")] Grpc(Box), /// Tonic transport-level error. #[error("transport error: {0}")] TonicTransport(#[from] tonic::transport::Error), } impl From for GrpcTransportError { fn from(status: tonic::Status) -> Self { Self::Grpc(Box::new(status)) } } impl GrpcTransportError { /// Whether this failure is *permanent* — re-shipping the same segment will /// keep failing identically until an operator changes configuration. /// /// The shipper's retry loop is correct ONLY for transient failures (peer /// down, circuit open, follower backpressure): it leaves the per-peer HWM /// untouched and re-sends the same seqno next poll. For a permanent failure /// that retry is an infinite silent stall — a mutually-distrusted TLS chain, /// a misconfigured CA, an authentication rejection, or a payload the wire /// codec will never accept does not become shippable by trying again. /// /// Classification: /// - [`TlsConfig`](Self::TlsConfig): local TLS/config material is bad → /// permanent. /// - [`Grpc`](Self::Grpc): permanent iff the gRPC status code is one the /// server will return identically on retry — `Unauthenticated` / /// `PermissionDenied` (mTLS / authz rejection), `InvalidArgument` / /// `OutOfRange` (the payload itself is malformed), `Unimplemented` (the /// RPC does not exist on the peer), `FailedPrecondition` (the m11p4 /// term fence: a stale-term ship fails identically until leadership /// changes — quarantining the peer is right, and the deposed sender's /// step-down deactivates the whole queue moments later anyway). /// Everything else (`Unavailable`, `ResourceExhausted`, /// `DeadlineExceeded`, codec-size on a transient over-large segment, …) /// is transient. /// - [`TonicTransport`](Self::TonicTransport): a connect/handshake error. /// These are predominantly transient (peer not up yet) and are treated as /// such; a genuinely permanent handshake fault surfaces as a `Grpc` TLS /// status on the next attempt. /// - [`PeerUnreachable`](Self::PeerUnreachable): routing error (unknown /// shard) — handled separately via `UnknownPeer`, never as a retry. /// - All other variants (circuit open, segment rejected, internal): transient. #[must_use] pub fn is_permanent(&self) -> bool { match self { Self::TlsConfig(_) => true, Self::Grpc(status) => matches!( status.code(), tonic::Code::Unauthenticated | tonic::Code::PermissionDenied | tonic::Code::InvalidArgument | tonic::Code::OutOfRange | tonic::Code::Unimplemented | tonic::Code::FailedPrecondition ), Self::CircuitOpen(_) | Self::PeerUnreachable(_) | Self::SegmentRejected(_) | Self::Internal(_) | Self::TonicTransport(_) => false, } } } impl From for TransportError { fn from(e: GrpcTransportError) -> Self { // Routing failure: the shard is not in our peer map. Surface as the // dedicated non-retry variant. if let GrpcTransportError::PeerUnreachable(shard) = e { return Self::UnknownPeer(shard); } // Permanent failures (bad TLS/CA, auth rejection, malformed payload, RPC // not implemented on the peer) will fail identically on every retry. // Mapping them to the transient `Closed` variant would make the shipper // re-ship the same seqno forever — a silent replication stall with no // actionable signal. Surface them as the dedicated // `TransportError::Permanent` variant (carrying the human-readable // cause) so the shipper quarantines the peer and stops advancing it, // and log LOUDLY at error! so the fault is observable in logs/alerts. if e.is_permanent() { tracing::error!( error = %e, "gRPC transport PERMANENT failure (bad TLS/CA, auth rejection, malformed payload, \ or unimplemented RPC); retrying this segment will not help — fix peer/config. \ Quarantining the peer via TransportError::Permanent", ); return Self::Permanent { reason: e.to_string(), }; } // Genuinely transient failures map to Closed: circuit-open and follower // backpressure are temporary (peer known but unable to accept right // now); the remaining internal/transport variants are non-routing and // recoverable. Mapping to Closed keeps the shipper from advancing its // per-peer cursor so it retries the same seqno, in order, next poll. Self::Closed } } #[cfg(test)] mod tests { use super::*; #[test] fn tls_config_failure_is_permanent() { let e = GrpcTransportError::TlsConfig("bad CA".into()); assert!(e.is_permanent(), "bad TLS material never becomes shippable"); } #[test] fn auth_and_malformed_grpc_codes_are_permanent() { // mTLS / authz rejection and malformed-payload codes are permanent: // the server returns them identically on every retry. for code in [ tonic::Code::Unauthenticated, tonic::Code::PermissionDenied, tonic::Code::InvalidArgument, tonic::Code::OutOfRange, tonic::Code::Unimplemented, tonic::Code::FailedPrecondition, ] { let e = GrpcTransportError::Grpc(Box::new(tonic::Status::new(code, "x"))); assert!(e.is_permanent(), "{code:?} must be permanent"); } } #[test] fn unavailable_and_exhausted_grpc_codes_are_transient() { // A peer that is down or temporarily overloaded WILL accept the same // segment once it recovers — these must stay transient so the shipper // keeps retrying. for code in [ tonic::Code::Unavailable, tonic::Code::ResourceExhausted, tonic::Code::DeadlineExceeded, tonic::Code::Aborted, ] { let e = GrpcTransportError::Grpc(Box::new(tonic::Status::new(code, "x"))); assert!(!e.is_permanent(), "{code:?} must be transient"); } } #[test] fn circuit_and_backpressure_are_transient() { assert!(!GrpcTransportError::CircuitOpen(ShardId(1)).is_permanent()); assert!(!GrpcTransportError::SegmentRejected(ShardId(1)).is_permanent()); assert!(!GrpcTransportError::Internal("transient".into()).is_permanent()); } #[test] fn peer_unreachable_maps_to_unknown_peer_not_closed() { let mapped: TransportError = GrpcTransportError::PeerUnreachable(ShardId(9)).into(); assert!( matches!(mapped, TransportError::UnknownPeer(ShardId(9))), "routing failure must surface as UnknownPeer, not a retry" ); } #[test] fn permanent_tls_failure_maps_to_permanent() { // A permanent failure (bad TLS material) must surface as the dedicated // `Permanent` variant so the shipper quarantines the peer instead of // re-shipping the same seqno forever. The reason carries the cause for // logs/alerts. let mapped: TransportError = GrpcTransportError::TlsConfig("bad CA".into()).into(); match mapped { TransportError::Permanent { reason } => { assert!( reason.contains("bad CA"), "reason must carry cause: {reason}" ); } other => panic!("expected Permanent, got {other:?}"), } } #[test] fn permanent_grpc_code_maps_to_permanent() { // Auth/authz and malformed-payload gRPC codes are permanent and must // map to `Permanent`, never the transient `Closed`. for code in [ tonic::Code::Unauthenticated, tonic::Code::PermissionDenied, tonic::Code::InvalidArgument, tonic::Code::OutOfRange, tonic::Code::Unimplemented, ] { let e = GrpcTransportError::Grpc(Box::new(tonic::Status::new(code, "x"))); let mapped: TransportError = e.into(); assert!( matches!(mapped, TransportError::Permanent { .. }), "{code:?} must map to Permanent, got {mapped:?}" ); } } #[test] fn transient_failure_maps_to_closed() { let mapped: TransportError = GrpcTransportError::SegmentRejected(ShardId(2)).into(); assert!(matches!(mapped, TransportError::Closed)); } }