tidaldb/tidal-net/src/transport.rs

//! `GrpcTransport` — implements tidalDB's `Transport` trait over gRPC.
//!
//! Each instance embeds a tokio runtime, runs a gRPC server for receiving
//! segments, and maintains a client connection pool for sending segments.
//! The sync/async bridge uses `runtime.block_on()`, which is safe because
//! callers (WAL shipper and segment receiver) run on `std::thread`, not
//! inside a tokio context.

use std::{
    collections::{HashMap, HashSet},
    sync::{
        Arc, Mutex,
        atomic::{AtomicBool, Ordering},
    },
    time::{Duration, Instant},
};

use tidaldb::replication::{
    shard::ShardId,
    transport::{Transport, TransportError, WalSegmentPayload},
};
use tokio::sync::{Notify, mpsc};

use crate::{
    client::PeerPool,
    config::GrpcTransportConfig,
    error::GrpcTransportError,
    server,
    sources::{ElectionHooks, ServingSources},
};

/// Minimum spacing between catch-up pull attempts per source shard. The gap
/// detector fires after every apply round while a hole is open; without a
/// floor it would open a stream per round.
const MIN_CATCHUP_INTERVAL: Duration = Duration::from_secs(2);

/// A gRPC-based transport for WAL segment shipping between tidalDB shards.
///
/// Implements the synchronous [`Transport`] trait by embedding a tokio runtime.
/// The gRPC server accepts incoming segments from peers and places them in an
/// internal channel. The client pool sends segments to peers via unary RPCs.
///
/// # Single-Consumer Invariant
///
/// `recv_segment` must only be called from a single thread (the segment receiver).
/// The `Mutex` around the receiver is a safety net, not a concurrency mechanism.
/// Concurrent callers would serialize on the mutex while each blocks on
/// `runtime.block_on(rx.recv())`, which is safe but wasteful.
///
/// # Deterministic receiver shutdown
///
/// A parked [`recv_segment`](Self::recv_segment) blocks on the inbound channel,
/// which only returns `None` once every `inbound_tx` sender drops — and those live
/// inside the server task. Aborting the server task on [`Drop`] eventually frees
/// them, but the receiver thread is typically a detached `std::thread`, so a clean
/// (non-process-exit) teardown could leave it parked. To make shutdown deterministic
/// the transport carries a latching `shutdown` signal that `Drop` (and the explicit
/// [`shutdown_receivers`](Self::shutdown_receivers)) trips **before** tearing the
/// runtime down: `recv_segment` checks the latch on entry and selects over the channel
/// and a wake `Notify`, so a parked receiver returns `None` immediately on shutdown and
/// its thread can exit and be joined.
pub struct GrpcTransport {
    config: GrpcTransportConfig,
    /// `Some` for the transport's whole life; taken in [`Drop`] so the runtime
    /// can be torn down with the non-blocking [`tokio::runtime::Runtime::shutdown_background`]
    /// (which consumes the runtime) instead of the default blocking `Runtime::drop`.
    runtime: Option<tokio::runtime::Runtime>,
    inbound_rx: Mutex<mpsc::Receiver<WalSegmentPayload>>,
    pool: Arc<PeerPool>,
    /// Per-peer durable marks, shared with the gRPC service (m11p3): the
    /// monotonic max of ship-ack hints (client side) and `ReportApplied`
    /// pushes (server side). Read by [`Transport::peer_applied_hint`].
    peer_applied: crate::server::PeerAppliedMap,
    /// Per-peer last-seen capability bit-field, shared with the gRPC service
    /// (m11p5 §3.1): folded from inbound `AppliedReport`s. Read by
    /// [`peer_capabilities`](Self::peer_capabilities) for the conf-change gate.
    peer_capabilities: crate::server::PeerCapabilityMap,
    /// Last frontier reported per source shard by [`Transport::notify_applied`]
    /// (dedup: unchanged rounds send nothing).
    last_reported: Mutex<HashMap<ShardId, u64>>,
    /// Source shards whose last `ReportApplied` push failed: the FIRST
    /// failure of a streak logs at WARN (a silently stalling frontier report
    /// surfaces as unexplained quorum 503s at 3am), repeats stay at debug,
    /// and recovery logs at INFO. Shared with the fire-and-forget report
    /// tasks. (`Arc`: the spawned task outlives the `&self` borrow.)
    report_failing: Arc<Mutex<HashSet<ShardId>>>,
    /// The catch-up pull machinery (single-flight + rate limit + the m11p4
    /// timer retry). `Arc` because the retry tasks outlive `&self` borrows.
    catchup: Arc<CatchupRunner>,
    /// Late-bound election hooks (m11p4), shared with the gRPC service: the
    /// transport consults them to stamp catch-up pulls with the current term
    /// and to surface higher terms observed on ship acks.
    election: Arc<std::sync::OnceLock<Arc<dyn ElectionHooks>>>,
    server_handle: tokio::task::JoinHandle<Result<(), tonic::transport::Error>>,
    /// Shutdown signal for the receiver. The [`AtomicBool`] **latches** the request
    /// so a `recv_segment` that has not yet parked still observes it on entry (no
    /// missed-wakeup race), while the [`Notify`] wakes a receiver that is *already*
    /// parked. Tripped by [`shutdown_receivers`](Self::shutdown_receivers) and by
    /// [`Drop`] before the runtime is torn down, so a parked
    /// [`recv_segment`](Self::recv_segment) returns `None` deterministically instead
    /// of waiting for every server-side `inbound_tx` sender to drop.
    shutdown: Arc<ShutdownSignal>,
}

/// Per-source-shard catch-up pull bookkeeping: at most one in-flight stream
/// per shard, spaced at least [`MIN_CATCHUP_INTERVAL`] apart, with at most
/// one scheduled timer retry.
struct CatchupState {
    in_flight: Arc<AtomicBool>,
    last_attempt: Option<Instant>,
    /// `true` while a timer retry is scheduled for this shard. One pending
    /// retry at a time: repeated failures while one is queued schedule
    /// nothing new (the queued retry re-enters the same gate anyway).
    retry_pending: Arc<AtomicBool>,
}

impl CatchupState {
    fn new() -> Self {
        Self {
            in_flight: Arc::new(AtomicBool::new(false)),
            last_attempt: None,
            retry_pending: Arc::new(AtomicBool::new(false)),
        }
    }
}

/// How one catch-up pull ended, deciding whether the timer retry arms.
enum PullOutcome {
    /// The stream completed cleanly (possibly empty = already caught up).
    /// The gap this pull was chasing is closed up to the source's snapshot
    /// end; anything newer arrives by live push (or triggers fresh gap
    /// detection). No retry.
    Complete,
    /// The local receiver is gone (shutdown unwinding). No retry.
    ReceiverGone,
    /// The pull failed — stream open refused, a mid-pull status, or an
    /// undecodable chunk. The gap is still open; arm the timer retry, because
    /// in an idle cluster NO push will ever re-trigger gap detection (the
    /// 2026-06-11 incident: followers restarted 1.5s before the leader's
    /// gRPC server, the one boot pull got `tcp connect error`, no write ever
    /// arrived, lag stayed at 136507 forever).
    Failed,
}

/// The catch-up pull machinery, shared by the event-driven trigger
/// ([`Transport::request_catchup`]) and the m11p4 timer-retry tasks.
struct CatchupRunner {
    pool: Arc<PeerPool>,
    /// Pulled stream chunks enter the SAME inbound channel as live unary
    /// ships, so the apply path is identical for both (m11p2).
    inbound_tx: mpsc::Sender<WalSegmentPayload>,
    shutdown: Arc<ShutdownSignal>,
    states: Mutex<HashMap<ShardId, CatchupState>>,
    /// Delay before a failed pull is re-attempted by the timer (config:
    /// `catchup_retry_interval`).
    retry_interval: Duration,
    /// This node's applied-frontier reader, when the embedder wired one: a
    /// timer retry pulls from the CURRENT contiguous frontier instead of the
    /// failed attempt's (possibly stale) start seqno. `None` (bare
    /// transports, tests) falls back to the original seqno — correct either
    /// way, since the receiver gates idempotently; fresh is just cheaper.
    applied: Option<Arc<dyn crate::sources::AppliedSource>>,
    /// Election hooks (m11p4): pulls are stamped with the puller's current
    /// term, and every received chunk's leadership claim is gated BEFORE the
    /// inbound channel — a stale source can never inject history.
    election: Arc<std::sync::OnceLock<Arc<dyn ElectionHooks>>>,
    /// Late-bound `snapshot-required` refusal sink (m11p5 §2.4): a pull whose
    /// open is refused with an `x-tidal-catchup: snapshot-required` trailer
    /// invokes it so the node latches a reseed marker. Absent → today's
    /// behavior (log + the standing retry timer).
    snapshot_required: Arc<std::sync::OnceLock<Arc<dyn crate::sources::SnapshotRequiredSink>>>,
    /// Late-bound COMPLETED-pull sink: the positive evidence that discharges a
    /// reseed marker. Absent → a completed pull discharges nothing (safe: the
    /// marker then waits for a snapshot reseed).
    catchup_served: Arc<std::sync::OnceLock<Arc<dyn crate::sources::CatchupServedSink>>>,
}

impl CatchupRunner {
    /// Gate (shutdown, single-flight, rate limit) and spawn one catch-up
    /// pull. `handle` is the transport's runtime — passed in because the
    /// event path calls this from a plain `std::thread` while the retry path
    /// calls it from inside that same runtime.
    ///
    /// Returns whether a pull was actually started. The event path ignores
    /// it (a skip means someone else is already chasing the gap), but the
    /// TIMER path must re-arm on a skip: its one-shot retry is the gap's
    /// only remaining wake-up in an idle cluster, so letting the rate limit
    /// or an in-flight pull silently consume it would re-create the exact
    /// stranding this timer exists to fix (the in-flight pull it deferred to
    /// may itself fail after the timer already fired).
    fn try_start(
        self: &Arc<Self>,
        handle: &tokio::runtime::Handle,
        from_shard: ShardId,
        from_seqno: u64,
    ) -> bool {
        if self.shutdown.is_requested() {
            return false;
        }
        // Single-flight + rate limit per source shard.
        let in_flight = {
            let mut map = self
                .states
                .lock()
                .unwrap_or_else(std::sync::PoisonError::into_inner);
            let state = map.entry(from_shard).or_insert_with(CatchupState::new);
            if state.in_flight.load(Ordering::Acquire) {
                return false;
            }
            if let Some(last) = state.last_attempt
                && last.elapsed() < MIN_CATCHUP_INTERVAL
            {
                return false;
            }
            state.last_attempt = Some(Instant::now());
            state.in_flight.store(true, Ordering::Release);
            let flag = Arc::clone(&state.in_flight);
            drop(map);
            flag
        };

        tracing::info!(
            shard = from_shard.0,
            from_seqno,
            "replication gap open; pulling catch-up stream from source"
        );
        let runner = Arc::clone(self);
        handle.spawn(async move {
            let outcome = runner.run_pull(from_shard, from_seqno).await;
            in_flight.store(false, Ordering::Release);
            if matches!(outcome, PullOutcome::Failed) {
                runner.schedule_retry(from_shard, from_seqno);
            }
        });
        true
    }

    /// Open the stream and drain it into the inbound channel.
    async fn run_pull(&self, from_shard: ShardId, from_seqno: u64) -> PullOutcome {
        // The puller's current term (m11p4): the source serves only a
        // same-term pull, and every received chunk re-proves its claim.
        let term = self.election.get().map_or(0, |hooks| hooks.self_claim().0);
        let mut stream = match self.pool.stream_from(from_shard, from_seqno, term).await {
            Ok(stream) => stream,
            Err(e) => {
                tracing::warn!(
                    shard = from_shard.0,
                    from_seqno,
                    error = %e,
                    retry_in = ?self.retry_interval,
                    "catch-up stream open failed; will retry on the next \
                     detected gap or the retry timer, whichever first"
                );
                // A refusal can also arrive on OPEN (a deposed/stale-term
                // source rejects before the stream spawns); classify its
                // trailer the same way so a snapshot-required open-refusal
                // latches the reseed marker (m11p5 §2.4).
                if let GrpcTransportError::Grpc(status) = &e {
                    self.handle_catchup_trailer(from_shard, from_seqno, status);
                }
                return PullOutcome::Failed;
            }
        };
        let mut chunks = 0u64;
        loop {
            match stream.message().await {
                Ok(Some(msg)) => match WalSegmentPayload::try_from(msg) {
                    Ok(payload) => {
                        // Term fence on the PULL path (m11p4, design-review
                        // C14): a chunk whose leadership claim is stale —
                        // e.g. a deposed source racing its own step-down —
                        // aborts the pull before the inbound channel.
                        if let Some(hooks) = self.election.get()
                            && let Err(rejection) = hooks.observe_leader_claim(
                                payload.term,
                                payload.leader_region,
                                payload.id.seqno,
                            )
                        {
                            tracing::warn!(
                                shard = from_shard.0,
                                chunk_term = payload.term,
                                ?rejection,
                                "catch-up chunk refused by the term gate; aborting pull \
                                 (the retry timer re-pulls once the term is joined)"
                            );
                            return PullOutcome::Failed;
                        }
                        chunks += 1;
                        // Bounded send = natural backpressure: the
                        // puller pauses while the receiver drains.
                        if self.inbound_tx.send(payload).await.is_err() {
                            return PullOutcome::ReceiverGone;
                        }
                    }
                    Err(e) => {
                        tracing::error!(
                            shard = from_shard.0,
                            error = e,
                            retry_in = ?self.retry_interval,
                            "catch-up stream chunk failed to convert; aborting pull"
                        );
                        return PullOutcome::Failed;
                    }
                },
                Ok(None) => {
                    tracing::info!(
                        shard = from_shard.0,
                        from_seqno,
                        chunks,
                        "catch-up stream complete"
                    );
                    // POSITIVE EVIDENCE (the only sound reseed-marker discharge
                    // short of a snapshot install): the source streamed the whole
                    // requested range and every chunk was applied. A frontier
                    // comparison cannot substitute for this — a term join re-bases
                    // the applied frontier onto the new leader's stream, so it can
                    // leap over history the node never received.
                    if let Some(sink) = self.catchup_served.get() {
                        sink.catchup_served(from_shard, from_seqno);
                    }
                    return PullOutcome::Complete;
                }
                Err(status) => {
                    // FAILED_PRECONDITION is the source's structured "this
                    // log can NEVER serve you that range" (segment format
                    // unknown after a rolling upgrade, compacted history, no
                    // durable WAL — m11p4). Name the remedy instead of
                    // logging it like a transient. The timer still retries:
                    // the condition clears when an operator reseeds the
                    // source or completes the upgrade, and until then the
                    // repeating log line is the visibility this follower's
                    // stalled replication deserves.
                    if status.code() == tonic::Code::FailedPrecondition {
                        tracing::error!(
                            shard = from_shard.0,
                            from_seqno,
                            %status,
                            retry_in = ?self.retry_interval,
                            "catch-up unservable from the source's WAL: this \
                             follower needs a snapshot (m11p5) or an operator \
                             reseed; replication stays degraded until then"
                        );
                        // m11p5 §2.4: a `snapshot-required` trailer latches the
                        // node's reseed marker (the reseed runs on the next
                        // boot). A `rejoin`/`stepping-down` trailer (a plain
                        // election term-mismatch) or an absent trailer (pre-p5
                        // source) does NOT — that conflation would poison the
                        // fleet with reseed markers during an ordinary
                        // election. The retry timer fires REGARDLESS.
                        self.handle_catchup_trailer(from_shard, from_seqno, &status);
                    } else {
                        tracing::error!(
                            shard = from_shard.0,
                            from_seqno,
                            %status,
                            retry_in = ?self.retry_interval,
                            "catch-up stream failed mid-pull; will re-pull on \
                             the next detected gap or the retry timer"
                        );
                    }
                    return PullOutcome::Failed;
                }
            }
        }
    }

    /// Classify a `FAILED_PRECONDITION` catch-up refusal by its
    /// `x-tidal-catchup` trailer (m11p5 §2.4) and, when it is
    /// `snapshot-required`, invoke the late-bound sink so the node latches its
    /// reseed marker. Any other value — `rejoin`, `stepping-down` — or an
    /// absent trailer (a pre-m11p5 source) is conservatively a no-op: only a
    /// genuine snapshot-required refusal may latch the marker. The standing
    /// retry timer is unaffected (the caller arms it regardless).
    fn handle_catchup_trailer(&self, from_shard: ShardId, from_seqno: u64, status: &tonic::Status) {
        let Some(value) = status.metadata().get(crate::server::CATCHUP_TRAILER) else {
            return; // pre-m11p5 source / untyped refusal: no latch
        };
        if value.as_bytes() != b"snapshot-required" {
            return; // rejoin / stepping-down / unknown: not a reseed condition
        }
        let Some(sink) = self.snapshot_required.get() else {
            // No sink wired (bare transport): the marker mechanism lives in
            // the node (stage B). The log line above is the visibility.
            return;
        };
        tracing::error!(
            shard = from_shard.0,
            from_seqno,
            "catch-up refused snapshot-required; latching the reseed marker \
             (the reseed runs on the next boot)"
        );
        sink.snapshot_required(from_shard, from_seqno);
    }

    /// Arm the timer retry for `from_shard` (m11p4): after
    /// [`retry_interval`](Self::retry_interval), re-enter [`try_start`] from
    /// the freshest known frontier. At most one timer is armed per shard.
    ///
    /// Must run inside the transport's runtime (it `tokio::spawn`s); both
    /// callers are pull tasks, which are.
    fn schedule_retry(self: &Arc<Self>, from_shard: ShardId, last_from_seqno: u64) {
        if self.shutdown.is_requested() {
            return;
        }
        let pending = {
            let mut map = self
                .states
                .lock()
                .unwrap_or_else(std::sync::PoisonError::into_inner);
            let state = map.entry(from_shard).or_insert_with(CatchupState::new);
            let flag = Arc::clone(&state.retry_pending);
            drop(map);
            flag
        };
        if pending.swap(true, Ordering::AcqRel) {
            return; // a retry is already scheduled for this shard
        }
        let runner = Arc::clone(self);
        tokio::spawn(async move {
            tokio::time::sleep(runner.retry_interval).await;
            pending.store(false, Ordering::Release);
            if runner.shutdown.is_requested() {
                return;
            }
            // Pull from the CURRENT contiguous frontier when readable (live
            // pushes may have advanced it during the wait); never below the
            // failed attempt's start, so a 0-reporting frontier (nothing
            // applied yet / unknown shard) cannot regress the request.
            let from = runner.applied.as_ref().map_or(last_from_seqno, |a| {
                a.applied_seqno(from_shard)
                    .saturating_add(1)
                    .max(last_from_seqno)
            });
            let started = runner.try_start(&tokio::runtime::Handle::current(), from_shard, from);
            if !started {
                // The timer's wake-up was consumed by the rate limit or an
                // in-flight pull. Re-arm: if that pull succeeds the extra
                // retry costs one empty stream open; if it fails, its own
                // re-arm dedups against this one via `retry_pending`. Either
                // way the gap keeps a standing wake-up until a pull
                // completes — the liveness property this timer exists for.
                // Logged so an operator tracing a stalled follower can see
                // the timer loop alive between default-level pull failures.
                tracing::debug!(
                    shard = from_shard.0,
                    from_seqno = from,
                    retry_in = ?runner.retry_interval,
                    "catch-up timer wake-up skipped (rate limit / pull in \
                     flight); re-armed"
                );
                runner.schedule_retry(from_shard, from);
            }
        });
    }
}

/// Latching shutdown signal: an [`AtomicBool`] that survives the not-yet-parked race
/// plus a [`Notify`] to wake an already-parked waiter. Held by [`GrpcTransport`] and
/// tripped on [`Drop`] / [`shutdown_receivers`](GrpcTransport::shutdown_receivers).
///
/// `pub(crate)` so the gRPC server's mTLS accept loop ([`crate::server`]) can
/// share the same latch — the accept loop and the serve future both exit on it,
/// so a transport `Drop` tears the whole inbound path down deterministically.
pub(crate) struct ShutdownSignal {
    requested: AtomicBool,
    notify: Notify,
}

impl ShutdownSignal {
    pub(crate) fn new() -> Self {
        Self {
            requested: AtomicBool::new(false),
            notify: Notify::new(),
        }
    }

    /// Latch the shutdown request and wake any already-parked receiver.
    ///
    /// `Release` so the latch is visible to a receiver that subsequently reads it
    /// with `Acquire` on `recv_segment` entry; `notify_waiters` covers a receiver
    /// already parked on `notified()` at the moment this is called.
    fn request(&self) {
        self.requested.store(true, Ordering::Release);
        self.notify.notify_waiters();
    }

    /// Whether shutdown has been requested. `Acquire` pairs with the `Release` in
    /// [`request`](Self::request) so a not-yet-parked receiver observes the latch.
    pub(crate) fn is_requested(&self) -> bool {
        self.requested.load(Ordering::Acquire)
    }

    /// Resolve once shutdown is requested (m11p7 mTLS serve loop).
    ///
    /// Registers on the [`Notify`] via `enable()` BEFORE the final flag re-check
    /// so a `request()` racing this call is either observed by the re-check or
    /// delivered by `notify_waiters` to the now-registered waiter — never both
    /// missed. Used as the `serve_with_incoming_shutdown` signal and the accept
    /// loop's exit arm.
    pub(crate) async fn wait(&self) {
        if self.requested.load(Ordering::Acquire) {
            return;
        }
        let notified = self.notify.notified();
        tokio::pin!(notified);
        // `enable()` registers the waiter immediately (the future otherwise only
        // registers on first poll), closing the gap between the check above and
        // the await below.
        notified.as_mut().enable();
        if self.requested.load(Ordering::Acquire) {
            return;
        }
        notified.await;
    }
}

/// Install a process-wide rustls [`CryptoProvider`] before tonic's TLS
/// builders run.
///
/// tidal-net pulls rustls only transitively (via tonic), with no
/// provider feature in its own dependency closure, so under a narrow build the
/// process-level default is absent and rustls panics (could not automatically
/// determine the process-level [`CryptoProvider`]); under a wider build two
/// providers can be present, which is equally ambiguous. Installing aws-lc-rs
/// explicitly here — idempotently, ignoring the error when another component
/// (e.g. a reqwest-based client elsewhere in the process) already set a default
/// — removes that fragility for the mTLS tests, a standalone tidal-server, and
/// the replication path alike. See `tidal-net/BUILD.bazel`.
///
/// Public so a binary can install the provider at the TOP of `main()`, BEFORE
/// any code builds a rustls client. The seed-join / reseed boot path
/// (`tidal-server`) constructs a blocking `reqwest` (rustls) HTTPS client on a
/// dedicated boot thread *before* `GrpcTransport::new` (which calls this) runs —
/// so without an up-front install that client panics. Idempotent and cheap;
/// safe to call unconditionally.
pub fn ensure_crypto_provider() {
    static ONCE: std::sync::Once = std::sync::Once::new();
    ONCE.call_once(|| {
        let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
    });
}

/// Spawn the m11p7 cert-rotation reloader on `handle`: poll the TLS files every
/// `interval`, and on a content change hot-swap the server identity (`resolver`)
/// and rebuild the outbound peer channels (`pool`). Exits when `shutdown` trips.
/// A reload/rebuild failure keeps the prior material and is logged — the node
/// never goes dark mid-rotation.
fn spawn_cert_reloader(
    handle: &tokio::runtime::Handle,
    resolver: Arc<crate::tls::DynamicCertResolver>,
    tls: &crate::config::TlsConfig,
    pool: Arc<PeerPool>,
    shutdown: Arc<ShutdownSignal>,
    interval: Duration,
    local_shard: u16,
) {
    let reloader = crate::tls::ServerCertReloader::new(
        resolver,
        tls.clone(),
        crate::tls::ServerCertReloader::fingerprint(tls),
    );
    handle.spawn(async move {
        loop {
            tokio::time::sleep(interval).await;
            if shutdown.is_requested() {
                return;
            }
            match reloader.poll_once() {
                Ok(false) => {}
                Ok(true) => match pool.rebuild_all() {
                    Ok(()) => tracing::info!(
                        shard = local_shard,
                        "TLS material rotated: server cert hot-swapped, peer channels rebuilt"
                    ),
                    Err(e) => tracing::warn!(
                        shard = local_shard,
                        error = %e,
                        "server cert hot-swapped, but peer-channel rebuild failed; outbound \
                         peers keep their prior client cert until the next rotation"
                    ),
                },
                Err(e) => tracing::warn!(
                    shard = local_shard,
                    error = %e,
                    "TLS rotation poll failed; keeping the current cert"
                ),
            }
        }
    });
}

impl GrpcTransport {
    /// Create and start a new gRPC transport.
    ///
    /// This starts a gRPC server on `config.listen_addr` and builds lazy
    /// connections to all configured peers.
    ///
    /// # Errors
    ///
    /// Returns an error if the server or client pool fails to initialize.
    pub fn new(config: GrpcTransportConfig) -> Result<Self, GrpcTransportError> {
        Self::new_with_sources(config, ServingSources::default())
    }

    /// Create and start a new gRPC transport with node-side serving sources
    /// (m11p2): an applied-seqno reader for ack piggybacking and a WAL
    /// read-back source for the `StreamSegments` catch-up path.
    ///
    /// # Errors
    ///
    /// Returns an error if the server or client pool fails to initialize.
    pub fn new_with_sources(
        config: GrpcTransportConfig,
        sources: ServingSources,
    ) -> Result<Self, GrpcTransportError> {
        ensure_crypto_provider();

        // Validate numeric invariants BEFORE building anything (C15). In
        // particular `mpsc::channel(config.channel_capacity)` below panics on a
        // zero capacity (tokio asserts buffer > 0), so a `channel_capacity == 0`
        // config must be caught here as a typed `Internal` error rather than
        // panicking deep in the constructor. `PeerPool::new` validates again
        // (idempotent, cheap); doing it up front is what guards the real
        // constructor ordering the config doc promises.
        config.validate()?;

        let runtime = tokio::runtime::Builder::new_multi_thread()
            .worker_threads(2)
            .enable_all()
            .thread_name(format!("tidal-grpc-{}", config.local_shard))
            .build()
            .map_err(|e| GrpcTransportError::Internal(format!("tokio runtime: {e}")))?;

        let (inbound_tx, inbound_rx) = mpsc::channel(config.channel_capacity);

        // Build server and client pool inside the runtime context.
        // TLS setup in tonic requires a tokio reactor to be available.
        let server_tx = inbound_tx.clone();
        let peer_applied: crate::server::PeerAppliedMap = Arc::new(Mutex::new(HashMap::new()));
        let server_map = Arc::clone(&peer_applied);
        let peer_capabilities: crate::server::PeerCapabilityMap =
            Arc::new(Mutex::new(HashMap::new()));
        let server_caps = Arc::clone(&peer_capabilities);
        // The catch-up retry timer reads this node's applied frontier through
        // the same source the server piggybacks on acks (m11p4).
        let applied_for_catchup = sources.applied.clone();
        // The election hooks cell is shared with the gRPC service (late-bound
        // by the embedding application alongside the applied sink).
        let election = Arc::clone(&sources.election);
        // The snapshot-required sink cell (m11p5 §2.4): the catch-up pull
        // surfaces a `snapshot-required` trailer to it so the node latches a
        // reseed marker. Late-bound, shared with the gRPC service.
        let snapshot_required = Arc::clone(&sources.snapshot_required);
        // The positive-evidence counterpart: a completed pull is what discharges a
        // reseed marker. Same late-bound sharing as `snapshot_required`.
        let catchup_served = Arc::clone(&sources.catchup_served);

        // The shutdown latch is created up front (m11p7): the gRPC server's mTLS
        // accept loop and its serve future both exit on it, so it must exist
        // before `start_server` spawns them.
        let shutdown = Arc::new(ShutdownSignal::new());

        // m11p7 cert rotation: when TLS is configured, build the hot-swappable
        // server identity resolver from the initial cert. The server is served
        // over a custom tokio-rustls acceptor fed by this resolver (NOT tonic's
        // fixed `.tls_config()`), so a later rotation swaps the cert with zero
        // connection drop. A bad initial cert fails construction loudly here.
        let server_resolver = match &config.tls {
            Some(tls) => {
                let initial = crate::tls::load_certified_key(&tls.server_cert, &tls.server_key)?;
                Some(Arc::new(crate::tls::DynamicCertResolver::new(initial)))
            }
            None => None,
        };

        let server_shutdown = Arc::clone(&shutdown);
        let resolver_for_server = server_resolver.clone();
        let (server_handle, pool) = runtime.block_on(async {
            let handle = server::start_server(
                &config,
                server_tx,
                sources,
                server_map,
                server_caps,
                resolver_for_server,
                server_shutdown,
            )?;
            let pool = PeerPool::new(&config)?;
            Ok::<_, GrpcTransportError>((handle, pool))
        })?;
        let pool = Arc::new(pool);

        // m11p7 cert-rotation reloader: when TLS is on, poll the cert files on a
        // timer and hot-swap the server resolver + rebuild peer channels when the
        // content changes (a k8s secret mount swaps file content behind a stable
        // path). Cert/bearer rotation without restart.
        if let (Some(resolver), Some(tls)) = (server_resolver, config.tls.clone()) {
            spawn_cert_reloader(
                runtime.handle(),
                resolver,
                &tls,
                Arc::clone(&pool),
                Arc::clone(&shutdown),
                config.rotation_poll_interval,
                config.local_shard.0,
            );
        }

        let catchup = Arc::new(CatchupRunner {
            pool: Arc::clone(&pool),
            inbound_tx,
            shutdown: Arc::clone(&shutdown),
            states: Mutex::new(HashMap::new()),
            retry_interval: config.catchup_retry_interval,
            applied: applied_for_catchup,
            election: Arc::clone(&election),
            snapshot_required,
            catchup_served,
        });

        Ok(Self {
            config,
            runtime: Some(runtime),
            inbound_rx: Mutex::new(inbound_rx),
            pool,
            peer_applied,
            peer_capabilities,
            last_reported: Mutex::new(HashMap::new()),
            report_failing: Arc::new(Mutex::new(HashSet::new())),
            catchup,
            election,
            server_handle,
            shutdown,
        })
    }

    /// A handle for the election driver's outbound RPCs (m11p4): vote and
    /// heartbeat fan-outs plus the transfer `TimeoutNow`, all fired as async
    /// tasks on this transport's runtime with per-call timeouts — callable
    /// from any thread, never blocking the caller. Results arrive on the
    /// caller's channel as [`ElectionNetEvent`]s.
    #[must_use]
    pub fn election_net(&self) -> ElectionNet {
        ElectionNet {
            pool: Arc::clone(&self.pool),
            handle: self.runtime().handle().clone(),
            shutdown: Arc::clone(&self.shutdown),
        }
    }

    /// This peer's last-seen capability bit-field (m11p5 §3.1), or `None` if
    /// no `AppliedReport` from it has been folded yet. The leader's
    /// conf-change gate (stage C) reads this to refuse appending kind-4
    /// records until every voter reports the [`CAP_KIND4_MEMBERSHIP`] bit.
    ///
    /// [`CAP_KIND4_MEMBERSHIP`]: crate::CAP_KIND4_MEMBERSHIP
    #[must_use]
    pub fn peer_capabilities(&self, peer: ShardId) -> Option<u64> {
        self.peer_capabilities
            .lock()
            .unwrap_or_else(std::sync::PoisonError::into_inner)
            .get(&peer)
            .copied()
    }

    /// Whether this leader has had a live gRPC round-trip with `peer` within
    /// `window`: an accepted ship OR a backpressure reply, both of which prove
    /// the peer's gRPC server is up. A genuine transport error (dead/partitioned
    /// peer) never refreshes the stamp, so it goes stale and this returns
    /// `false`.
    ///
    /// The leader's `/cluster/status` aggregator consults this when a peer's
    /// `/cluster/status/local` HTTP probe times out: under a sustained quorum
    /// write-burst (large 1536-dim HNSW inserts), a follower's HTTP control-plane
    /// can be momentarily starved while replication keeps flowing, which the
    /// old probe-only check misread as a partition. Fresh gRPC contact means the
    /// peer is SLOW-but-alive (report `reachable`), not partitioned. A peer with
    /// no recent contact stays honestly `reachable: false` — the genuine-partition
    /// contract the chaos suite asserts is preserved (a real TCP severance kills
    /// BOTH the HTTP probe and the gRPC ship).
    #[must_use]
    pub fn peer_grpc_fresh(&self, peer: ShardId, window: Duration) -> bool {
        self.pool.peer_grpc_fresh(peer, window)
    }

    /// Add a peer's gRPC connection at runtime (m11p5 §3.3 conf-change). The
    /// channel is LAZY, so the insert costs no DNS resolution or connect: a
    /// DNS-named peer re-resolves on its first RPC and on every reconnect. The
    /// new connection inherits this transport's config (TLS/timeouts/codec
    /// limits). `addr` is a bare `host:port`.
    ///
    /// Shared with the [`ElectionNet`] and [`CatchupRunner`] (both hold the
    /// same `Arc<PeerPool>`), so a peer added here is immediately reachable by
    /// election RPCs, ships, and catch-up pulls.
    ///
    /// # Errors
    ///
    /// Returns [`GrpcTransportError`] if the URI is malformed or the client
    /// TLS configuration cannot be built.
    pub fn add_peer(&self, shard: ShardId, addr: &str) -> Result<(), GrpcTransportError> {
        // `PeerPool::add_peer` builds a tonic channel via `connect_lazy`, which
        // registers background tasks and therefore REQUIRES a Tokio reactor in
        // scope. A membership conf-change applies this from the election driver
        // thread — a plain `std::thread` with NO ambient runtime — so enter the
        // transport's OWN embedded runtime first (without it, `connect_lazy`
        // panics with "there is no reactor running"). The guard is dropped as
        // soon as `add_peer` returns; no task is spawned that outlives it.
        let _guard = self.runtime().enter();
        self.pool.add_peer(shard, addr)
    }

    /// Remove a peer's gRPC connection at runtime (m11p5 §3.3 conf-change):
    /// drops the lazy channel. Returns whether a peer was present. After
    /// removal, every RPC to `shard` surfaces as
    /// [`TransportError::PeerUnreachable`]-equivalent at the call site.
    pub fn remove_peer(&self, shard: ShardId) -> bool {
        // Enter the embedded runtime for the same reason as `add_peer`: dropping
        // a tonic channel tears down its background connection task, which is
        // reactor-aware. A conf-change removal applies from the election thread
        // (no ambient runtime), so scope the transport's own runtime here.
        let _guard = self.runtime().enter();
        self.pool.remove_peer(shard)
    }

    /// A `FetchSnapshot` server-stream from `shard` (m11p5 §2): the joiner/
    /// reseed catch-up path. `from_seqno` is the puller's frontier+1; `term`
    /// stamps the request for the source's same-term fence. Bypasses the
    /// circuit breaker for the same reason as [`stream_from`](crate::client::PeerPool::stream_from).
    ///
    /// The returned stream yields a header chunk first (needed / manifest)
    /// then file chunks; the caller (stage B node-side install) verifies each
    /// file's BLAKE3 against the manifest and advances its frontier to the
    /// snapshot seq before resuming `StreamSegments`.
    ///
    /// # Errors
    ///
    /// Returns [`GrpcTransportError`] if the peer is unknown or the stream
    /// cannot be opened (including `Unimplemented` from a pre-m11p5 peer).
    pub fn fetch_snapshot(
        &self,
        shard: ShardId,
        from_seqno: u64,
        term: u64,
    ) -> Result<tonic::Streaming<crate::proto::SnapshotChunk>, GrpcTransportError> {
        self.runtime()
            .block_on(self.pool.fetch_snapshot(shard, from_seqno, term))
    }

    /// Send a `JoinCluster` request to a known peer (m11p5 §3.3): the
    /// operator/manual `/cluster/join` forward to the leader once the transport
    /// exists. Blocks on this transport's runtime; callable from any thread.
    ///
    /// # Errors
    ///
    /// Returns [`GrpcTransportError`] if the peer is unknown or the RPC fails
    /// (including `Unimplemented` from a pre-m11p5 peer). A non-leader refusal
    /// is carried IN the response body (`accepted=false`), not as an error.
    pub fn join_cluster(
        &self,
        to: ShardId,
        request: crate::proto::JoinRequest,
    ) -> Result<crate::proto::JoinResponse, GrpcTransportError> {
        self.runtime().block_on(self.pool.join_cluster(to, request))
    }

    /// The embedded tokio runtime.
    ///
    /// # Infallible by construction
    ///
    /// The `expect` here can never fire on any production path. `runtime` is set to
    /// `Some` exactly once, in [`new`](Self::new), and is only ever moved out in
    /// [`Drop::drop`] via `self.runtime.take()`. `Drop` runs at most once and is the
    /// final use of `self`; no method (`send_segment`, `recv_segment`, …) can observe
    /// `self` after `Drop` has begun, so every call to `runtime()` sees `Some`. The
    /// `Option` exists purely so `Drop` can move the runtime out and tear it down with
    /// the non-blocking `shutdown_background` (see the field doc on `runtime`), not to
    /// model a genuinely-absent runtime — hence `expect` over fallible propagation: a
    /// `None` here would be a memory-safety/lifecycle bug in this module, not a runtime
    /// condition a caller could handle.
    const fn runtime(&self) -> &tokio::runtime::Runtime {
        self.runtime
            .as_ref()
            .expect("runtime is Some for the transport's whole lifetime; only Drop takes it")
    }

    /// Signal any current or future [`recv_segment`](Self::recv_segment) to return `None`.
    ///
    /// Latches the shutdown request and wakes a receiver already blocked in
    /// `recv_segment` so it returns `None` — its exit condition per the [`Transport`]
    /// contract — letting the receiver thread exit and be joined. Because the request
    /// latches, a receiver that has not yet parked also observes it on its next entry.
    ///
    /// # Why this exists separately from [`Drop`]
    ///
    /// The receiver thread typically holds its own `Arc<dyn Transport>` clone, so
    /// [`GrpcTransport::drop`] cannot fire until that thread has already exited and
    /// released its `Arc` — a cycle: the thread parks in `recv_segment` and never
    /// releases the `Arc`, so `Drop` never runs, so the thread is never woken. Break
    /// the cycle by calling `shutdown_receivers` from the owner's shutdown path (which
    /// holds a *different* `Arc`) BEFORE joining the receiver thread and dropping its
    /// `Arc`. The notify-trip in [`Drop`] remains a backstop for the non-cyclic case
    /// (e.g. a transport owned outright, as in this crate's tests).
    ///
    /// Idempotent: the request latches, so any current or future `recv_segment`
    /// returns `None`.
    pub fn shutdown_receivers(&self) {
        self.shutdown.request();
    }

    /// Whether the embedded gRPC serve loop has terminated, for ANY reason.
    ///
    /// The serve task runs for the transport's whole lifetime; a `true` here
    /// means the listener stopped accepting — either because the transport is
    /// shutting down or because the serve loop failed (in which case the failure
    /// was already logged at `error!` inside the task; see [`server::start_server`]).
    ///
    /// Because this conflates a clean shutdown with a silent death, prefer
    /// [`serve_loop_died`](Self::serve_loop_died) for liveness decisions — it is
    /// the one that distinguishes the two (C14).
    #[must_use]
    pub fn server_terminated(&self) -> bool {
        self.server_handle.is_finished()
    }

    /// Whether the gRPC serve loop has died WITHOUT a shutdown being requested.
    ///
    /// This is the load-bearing liveness signal (C14): on a follower the receive
    /// side dies silently when the listener stops, a TLS handshake task panics,
    /// or the reactor is torn down. In all those cases the spawned server task
    /// ends, its `inbound_tx` drops, and [`recv_segment`](Self::recv_segment)
    /// returns `None` — *exactly* as it does on a clean
    /// [`shutdown_receivers`](Self::shutdown_receivers). The two are otherwise
    /// indistinguishable, so a dead follower would look like an intentionally
    /// stopped one.
    ///
    /// `serve_loop_died()` resolves the ambiguity: it is `true` only when the
    /// serve task has finished AND no shutdown was requested. Health checks and
    /// the cluster control plane poll this so a follower whose receive side has
    /// silently died is observable and demoted, not a black hole; a clean
    /// shutdown returns `false` here even though [`server_terminated`](Self::server_terminated)
    /// is `true`.
    #[must_use]
    pub fn serve_loop_died(&self) -> bool {
        self.server_handle.is_finished() && !self.shutdown.is_requested()
    }

    /// Assert we are not inside a tokio runtime (`block_on` would panic).
    #[cfg(debug_assertions)]
    fn assert_not_in_async_context() {
        debug_assert!(
            tokio::runtime::Handle::try_current().is_err(),
            "GrpcTransport methods must not be called from within a tokio runtime; \
             use std::thread instead"
        );
    }
}

impl Transport for GrpcTransport {
    fn send_segment(&self, to: ShardId, payload: WalSegmentPayload) -> Result<(), TransportError> {
        #[cfg(debug_assertions)]
        Self::assert_not_in_async_context();

        // Validate payload size before sending.
        if payload.bytes.len() > self.config.max_payload_bytes {
            return Err(TransportError::PayloadTooLarge {
                size: payload.bytes.len(),
                max: self.config.max_payload_bytes,
            });
        }

        let (applied, responder_term) = self
            .runtime()
            .block_on(self.pool.send_to(to, payload))
            .map_err(TransportError::from)?;
        crate::server::fold_peer_applied(&self.peer_applied, to, applied);
        // A follower answering from a higher term is this sender's step-down
        // signal (m11p4) — a deposed leader learns its term is stale from its
        // own outbound traffic even if it never hears a heartbeat.
        if responder_term > 0
            && let Some(hooks) = self.election.get()
        {
            let (current, _) = hooks.self_claim();
            if responder_term > current {
                hooks.on_observed_term(responder_term);
            }
        }
        Ok(())
    }

    fn peer_applied_hint(&self, peer: ShardId) -> u64 {
        self.peer_applied
            .lock()
            .unwrap_or_else(std::sync::PoisonError::into_inner)
            .get(&peer)
            .copied()
            .unwrap_or(0)
    }

    fn notify_applied(&self, source_shard: ShardId, applied: u64) {
        // Push this node's durable frontier to the stream's source (m11p3
        // quorum acks): fire-and-forget, once per ADVANCED apply round (the
        // dedup map drops unchanged rounds), fully decoupled from ship acks
        // — so the leader's commit index stays fresh even when its outbound
        // ships stall (gap-parked follower, pull-based catch-up, quiet
        // leader). A lost report self-corrects on the next advanced round;
        // marks are monotonic on the receiving side.
        if self.shutdown.is_requested() || source_shard == self.config.local_shard {
            return;
        }
        let mut last = self
            .last_reported
            .lock()
            .unwrap_or_else(std::sync::PoisonError::into_inner);
        let entry = last.entry(source_shard).or_insert(0);
        if applied <= *entry {
            return;
        }
        *entry = applied;
        drop(last);
        let pool = Arc::clone(&self.pool);
        let reporter = self.config.local_shard;
        let failing = Arc::clone(&self.report_failing);
        // The reporter's current term (m11p4): the source folds this report
        // into its quorum commit index ONLY when it matches the leadership
        // the index was activated with.
        let reporter_term = self.election.get().map_or(0, |hooks| hooks.self_claim().0);
        self.runtime().spawn(async move {
            match pool
                .report_applied(source_shard, reporter, applied, reporter_term)
                .await
            {
                Ok(()) => {
                    let recovered = failing
                        .lock()
                        .unwrap_or_else(std::sync::PoisonError::into_inner)
                        .remove(&source_shard);
                    if recovered {
                        tracing::info!(
                            source = source_shard.0,
                            applied,
                            "applied-frontier reports recovered"
                        );
                    }
                }
                Err(e) => {
                    // Best-effort by design: the next advanced round
                    // re-reports, and ship-ack hints keep flowing regardless.
                    // But a STREAK of failures must be visible at default log
                    // levels — a silently stalling frontier report shows up
                    // as unexplained quorum 503s on the leader.
                    let first_of_streak = failing
                        .lock()
                        .unwrap_or_else(std::sync::PoisonError::into_inner)
                        .insert(source_shard);
                    if first_of_streak {
                        tracing::warn!(
                            source = source_shard.0,
                            applied,
                            error = %e,
                            "applied-frontier report failed; the source's quorum \
                             freshness degrades to ship-ack hints until this \
                             recovers (will retry on next advance)"
                        );
                    } else {
                        tracing::debug!(
                            source = source_shard.0,
                            applied,
                            error = %e,
                            "applied-frontier report still failing (will retry \
                             on next advance)"
                        );
                    }
                }
            }
        });
    }

    fn request_catchup(&self, from_shard: ShardId, from_seqno: u64) {
        // All gating (shutdown, single-flight, rate limit) lives in the
        // runner, shared with the m11p4 timer-retry path: a failed pull arms
        // a one-shot timer that re-pulls from the fresh frontier, so an IDLE
        // cluster self-heals without waiting for a push that never comes. A
        // skip (`false`) means another pull or its timer already owns the
        // gap — the event path needs no follow-up of its own.
        let _ = self
            .catchup
            .try_start(self.runtime().handle(), from_shard, from_seqno);
    }

    fn recv_segment(&self) -> Option<WalSegmentPayload> {
        #[cfg(debug_assertions)]
        Self::assert_not_in_async_context();

        let mut rx = match self.inbound_rx.lock() {
            Ok(rx) => rx,
            Err(poisoned) => {
                tracing::error!("inbound_rx lock poisoned; recovering guard");
                poisoned.into_inner()
            }
        };
        let shutdown = &self.shutdown;
        // Fast path: if shutdown was already requested before we ever parked, the
        // latched flag short-circuits to `None`. This closes the missed-wakeup race
        // where `shutdown_receivers`/`Drop` ran between a prior `recv_segment`
        // returning and this call constructing its `notified()` future.
        if shutdown.is_requested() {
            return None;
        }
        // Otherwise select over an inbound segment and the shutdown notify so a
        // parked receiver returns `None` the instant shutdown is signaled, rather
        // than waiting for every server-side `inbound_tx` sender to drop. `None`
        // from either arm is the receiver's exit condition (see the `Transport`
        // trait contract), so a tripped shutdown looks exactly like channel close.
        self.runtime().block_on(async {
            // Register the `notified()` future BEFORE the final flag re-check so a
            // `request()` that runs concurrently either (a) is seen by this re-check
            // or (b) finds us already registered and wakes us — never both-missed.
            let notified = shutdown.notify.notified();
            tokio::pin!(notified);
            if shutdown.is_requested() {
                return None;
            }
            tokio::select! {
                // Bias toward draining already-queued segments before observing
                // shutdown, so a shutdown that races with in-flight segments does
                // not silently discard one already sitting in the channel.
                biased;
                segment = rx.recv() => segment,
                () = &mut notified => None,
            }
        })
    }

    fn try_recv_segment(&self) -> Option<WalSegmentPayload> {
        #[cfg(debug_assertions)]
        Self::assert_not_in_async_context();

        // Non-blocking drain for receiver-side group-commit coalescing
        // (m11p1). No runtime hop: tokio's mpsc `try_recv` is synchronous.
        // Shutdown short-circuits exactly like `recv_segment`'s fast path.
        if self.shutdown.is_requested() {
            return None;
        }
        let mut rx = match self.inbound_rx.lock() {
            Ok(rx) => rx,
            Err(poisoned) => {
                tracing::error!("inbound_rx lock poisoned; recovering guard");
                poisoned.into_inner()
            }
        };
        rx.try_recv().ok()
    }

    fn peer_breaker_state(&self, peer: ShardId) -> u8 {
        // Read-only breaker peek for the self-heal gauge (m11p8) — never admits
        // the half-open probe.
        self.pool.breaker_state(peer).as_gauge()
    }

    fn local_shard(&self) -> ShardId {
        self.config.local_shard
    }
}

impl Drop for GrpcTransport {
    fn drop(&mut self) {
        // 1. Trip the shutdown notify FIRST, while the runtime is still alive, so a
        //    parked `recv_segment` wakes on its `shutdown.notified()` arm and returns
        //    `None` deterministically — its receiver thread can then exit and join
        //    instead of leaking until process exit. `notify_waiters` wakes a waiter
        //    that is already parked; `recv_segment` re-creates its future on each call
        //    so there is no missed-wakeup window for a not-yet-parked receiver (a
        //    fresh `recv_segment` after Drop sees a closed channel once the server
        //    task is aborted below). The latch in `request()` also makes any future
        //    `recv_segment` short-circuit to `None` without parking.
        self.shutdown.request();

        // 2. Abort the server task so its `inbound_tx` is released (so a receiver
        //    that calls `recv_segment` AFTER Drop also observes channel close), then
        //    tear the runtime down WITHOUT blocking.
        //
        //    A `GrpcTransport` can be dropped from within an async context — e.g.
        //    cluster shutdown unwinding on the axum reactor thread — where the
        //    default blocking `Runtime::drop` panics ("Cannot drop a runtime in a
        //    context where blocking is not allowed"). `shutdown_background` returns
        //    immediately and reclaims the worker threads off-thread, so `Drop` is
        //    safe in any context.
        self.server_handle.abort();
        if let Some(runtime) = self.runtime.take() {
            runtime.shutdown_background();
        }
    }
}

/// One completed (or failed) election RPC, delivered to the driver's inbox.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ElectionNetEvent {
    /// A peer answered our pre-vote/vote.
    VoteReply {
        from: ShardId,
        prevote: bool,
        term: u64,
        granted: bool,
        /// The typed removed signal (m11p5 §3.3): the voter's applied roster
        /// lists THIS candidate as `Removed`. The driver flips readiness 503 +
        /// suppresses campaigning, exempt from the reseed marker.
        removed: bool,
    },
    /// The vote RPC failed (unreachable, timeout, or `Unimplemented` from a
    /// pre-m11p4 peer) — counts as not-granted.
    VoteUnreachable { from: ShardId, prevote: bool },
    /// A peer answered our heartbeat.
    HeartbeatReply {
        from: ShardId,
        term: u64,
        accepted: bool,
        /// The typed removed signal (m11p5 §3.3): the responder's applied
        /// roster lists THIS node (the heartbeat sender) as `Removed`. The
        /// driver flips readiness 503 + suppresses campaigning, no reseed.
        removed: bool,
    },
    /// The heartbeat RPC failed — no ack for the check-quorum lease.
    HeartbeatUnreachable { from: ShardId },
    /// The transfer `TimeoutNow` was delivered (`accepted` = the target
    /// started an election).
    TimeoutNowDelivered { target: ShardId, accepted: bool },
    /// The transfer `TimeoutNow` could not be delivered.
    TimeoutNowUnreachable { target: ShardId },
}

/// Per-RPC deadline for election traffic: well under the heartbeat interval
/// and the election timeout, so one stalled peer can never serialize a
/// fan-out round.
const ELECTION_RPC_TIMEOUT: Duration = Duration::from_secs(1);

/// The election driver's outbound surface (m11p4).
///
/// Fire-and-forget fan-outs on the transport's runtime, results delivered as
/// [`ElectionNetEvent`]s on the driver's channel. Callable from any thread;
/// never blocks.
///
/// Deliberately bypasses the circuit breaker: an election must be able to
/// probe a peer the data-plane breaker quarantined, and the traffic is rare
/// and self-limiting.
pub struct ElectionNet {
    pool: Arc<PeerPool>,
    handle: tokio::runtime::Handle,
    shutdown: Arc<ShutdownSignal>,
}

impl ElectionNet {
    /// Fan `request` to every peer in `peers` as a pre-vote/vote round.
    pub fn fan_votes(
        &self,
        peers: &[ShardId],
        request: crate::proto::VoteRequest,
        tx: &std::sync::mpsc::Sender<ElectionNetEvent>,
    ) {
        for &peer in peers {
            if self.shutdown.is_requested() {
                return;
            }
            let pool = Arc::clone(&self.pool);
            let tx = tx.clone();
            let prevote = request.prevote;
            self.handle.spawn(async move {
                let outcome =
                    tokio::time::timeout(ELECTION_RPC_TIMEOUT, pool.request_vote(peer, request))
                        .await;
                let event = match outcome {
                    Ok(Ok(reply)) => ElectionNetEvent::VoteReply {
                        from: peer,
                        prevote,
                        term: reply.term,
                        granted: reply.granted,
                        removed: reply.removed,
                    },
                    Ok(Err(_)) | Err(_) => ElectionNetEvent::VoteUnreachable {
                        from: peer,
                        prevote,
                    },
                };
                let _ = tx.send(event);
            });
        }
    }

    /// Fan one heartbeat round to every peer.
    pub fn fan_heartbeats(
        &self,
        peers: &[ShardId],
        request: &crate::proto::HeartbeatRequest,
        tx: &std::sync::mpsc::Sender<ElectionNetEvent>,
    ) {
        for &peer in peers {
            if self.shutdown.is_requested() {
                return;
            }
            let pool = Arc::clone(&self.pool);
            let req = request.clone();
            let tx = tx.clone();
            self.handle.spawn(async move {
                let outcome =
                    tokio::time::timeout(ELECTION_RPC_TIMEOUT, pool.heartbeat(peer, req)).await;
                let event = match outcome {
                    Ok(Ok(reply)) => ElectionNetEvent::HeartbeatReply {
                        from: peer,
                        term: reply.term,
                        accepted: reply.accepted,
                        removed: reply.removed,
                    },
                    Ok(Err(_)) | Err(_) => ElectionNetEvent::HeartbeatUnreachable { from: peer },
                };
                let _ = tx.send(event);
            });
        }
    }

    /// Deliver a transfer `TimeoutNow` to `target`.
    pub fn send_timeout_now(
        &self,
        target: ShardId,
        request: crate::proto::TimeoutNowRequest,
        tx: &std::sync::mpsc::Sender<ElectionNetEvent>,
    ) {
        if self.shutdown.is_requested() {
            return;
        }
        let pool = Arc::clone(&self.pool);
        let tx = tx.clone();
        self.handle.spawn(async move {
            let outcome =
                tokio::time::timeout(ELECTION_RPC_TIMEOUT, pool.timeout_now(target, request)).await;
            let event = match outcome {
                Ok(Ok(reply)) => ElectionNetEvent::TimeoutNowDelivered {
                    target,
                    accepted: reply.accepted,
                },
                Ok(Err(_)) | Err(_) => ElectionNetEvent::TimeoutNowUnreachable { target },
            };
            let _ = tx.send(event);
        });
    }
}

/// Factory for building a set of [`GrpcTransport`] instances, one per shard.
///
/// Analogous to `InProcessTransportFactory` but for gRPC connections.
pub struct GrpcTransportFactory;

impl GrpcTransportFactory {
    /// Build one `GrpcTransport` per configuration.
    ///
    /// Each transport gets its own tokio runtime, gRPC server, and client pool.
    ///
    /// # Errors
    ///
    /// Returns an error if any transport fails to initialize.
    pub fn build(
        configs: Vec<GrpcTransportConfig>,
    ) -> Result<HashMap<ShardId, GrpcTransport>, GrpcTransportError> {
        let mut result = HashMap::new();
        for config in configs {
            let shard = config.local_shard;
            let transport = GrpcTransport::new(config)?;
            result.insert(shard, transport);
        }
        Ok(result)
    }
}

#[cfg(test)]
#[allow(clippy::unwrap_used)] // test assertions on known-good fixtures
mod tests {
    use std::net::SocketAddr;

    use super::*;

    /// Bind port 0 to obtain a free, OS-assigned address (tonic cannot bind 0
    /// directly, so we resolve a concrete port up front).
    fn free_addr() -> SocketAddr {
        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
        listener.local_addr().unwrap()
    }

    /// C15: `GrpcTransport::new` with `channel_capacity == 0` must return a typed
    /// `Internal` error, NOT panic. The panic used to happen at
    /// `mpsc::channel(0)` BEFORE `PeerPool::new` ran `validate()`, so the only
    /// existing test (which called `PeerPool::new` directly) masked it. This
    /// drives the real constructor ordering.
    #[test]
    #[allow(clippy::significant_drop_tightening)]
    fn new_with_zero_channel_capacity_returns_internal_not_panic() {
        let config = GrpcTransportConfig {
            local_shard: ShardId(0),
            listen_addr: free_addr(),
            channel_capacity: 0,
            insecure: true,
            ..Default::default()
        };
        let result = GrpcTransport::new(config);
        assert!(
            matches!(result, Err(GrpcTransportError::Internal(_))),
            "channel_capacity == 0 must be a typed Internal error, not a panic"
        );
    }

    /// C14: a serve loop that dies WITHOUT a shutdown request must be observable
    /// via `serve_loop_died()`, distinct from a clean `shutdown_receivers()`.
    /// We abort the server task to model a silent serve-loop death and assert the
    /// transport reports `serve_loop_died() == true` while a transport that was
    /// cleanly shut down reports `false` (even though `server_terminated()` is
    /// `true` for both).
    #[test]
    #[allow(clippy::significant_drop_tightening)]
    fn serve_loop_died_distinguishes_silent_death_from_clean_shutdown() {
        // --- silent death: abort the serve task, no shutdown requested ---
        let dead = GrpcTransport::new(GrpcTransportConfig {
            local_shard: ShardId(0),
            listen_addr: free_addr(),
            insecure: true,
            ..Default::default()
        })
        .unwrap();
        // Kill the serve loop the way a listener-death / reactor-teardown would.
        dead.server_handle.abort();
        // Wait for the abort to take effect.
        let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
        while !dead.server_terminated() && std::time::Instant::now() < deadline {
            std::thread::sleep(std::time::Duration::from_millis(10));
        }
        assert!(
            dead.server_terminated(),
            "the aborted serve task must report server_terminated()"
        );
        assert!(
            dead.serve_loop_died(),
            "a serve loop that died WITHOUT a shutdown request must report \
             serve_loop_died() == true (C14)"
        );

        // --- clean shutdown: request shutdown, then the task ends ---
        let clean = GrpcTransport::new(GrpcTransportConfig {
            local_shard: ShardId(1),
            listen_addr: free_addr(),
            insecure: true,
            ..Default::default()
        })
        .unwrap();
        clean.shutdown_receivers();
        clean.server_handle.abort();
        let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
        while !clean.server_terminated() && std::time::Instant::now() < deadline {
            std::thread::sleep(std::time::Duration::from_millis(10));
        }
        assert!(clean.server_terminated());
        assert!(
            !clean.serve_loop_died(),
            "a serve loop that ended AFTER shutdown was requested is a clean \
             shutdown, not a silent death (C14)"
        );
    }
}