tidaldb/tidal/src/replication/shipper.rs

//! WAL segment shipper: polls for sealed segments and ships them to peers.
//!
//! The shipper runs in a background thread, polling the WAL directory for
//! sealed segments (those with a newer segment after them) and sending each
//! to all configured peer shards via the [`Transport`] trait.

use std::{collections::HashMap, path::PathBuf, sync::Arc, thread::JoinHandle, time::Duration};

use crossbeam::channel::bounded;
use dashmap::DashMap;

use crate::{
    replication::{
        RegionId, WalSegmentId,
        shard::ShardId,
        transport::{Transport, TransportError, WalSegmentPayload},
    },
    wal::format::batch::{HEADER_SIZE, encode_batch_with_shard},
};

/// Configuration for the WAL shipper background thread.
pub struct ShipperConfig {
    /// Directory containing WAL segment files.
    pub wal_dir: PathBuf,
    /// This node's shard identity.
    pub shard_id: ShardId,
    /// Peer shards to ship segments to.
    pub peer_shards: Vec<ShardId>,
    /// How often to poll for new sealed segments.
    pub poll_interval: Duration,
    /// When `true`, this shipper feeds a **community overlay** rather than a
    /// full replica: every [`SignalScope::Local`] event is stripped from each
    /// segment before it is shipped, enforcing the load-bearing M9 invariant
    /// that local-scoped signals never leave the node. Per-intent share
    /// filtering is applied earlier, at write time, via
    /// [`SharePolicy`](crate::governance::SharePolicy); the shipper only
    /// enforces the hard local/non-local boundary.
    ///
    /// `false` (the default) preserves M8 full-replication behavior: every
    /// event ships verbatim to the peer replica.
    ///
    /// [`SignalScope::Local`]: crate::governance::SignalScope::Local
    pub community_share_only: bool,
}

impl Default for ShipperConfig {
    fn default() -> Self {
        Self {
            wal_dir: PathBuf::from("wal"),
            shard_id: ShardId::SINGLE,
            peer_shards: vec![],
            poll_interval: Duration::from_secs(2),
            community_share_only: false,
        }
    }
}

/// Shared, operator-visible shipper state: per-peer high-water-mark and the set
/// of quarantined peers.
///
/// Previously these lived as local variables on the shipper thread's stack
/// (`peer_hwm`, `quarantined`), so an operator could neither observe which peers
/// were quarantined nor clear a quarantine without dropping and re-spawning the
/// whole shipper. Hoisting them into an `Arc<ShipperState>` shared with
/// [`WalShipperHandle`] gives the control plane read accessors
/// ([`quarantined_peers`](Self::quarantined_peers), [`peer_hwm`](Self::peer_hwm))
/// and a [`clear_quarantine`](Self::clear_quarantine) method an operator
/// endpoint can call after fixing a peer's config — no shipper restart required.
///
/// Both maps are [`DashMap`]s so the shipper thread and an operator thread can
/// touch them concurrently without a coarse lock. The shipper advances a peer's
/// HWM only on its own successful send and re-reads the quarantine set at the
/// top of each peer's run, so a concurrent `clear_quarantine` takes effect on
/// the next poll.
#[derive(Debug, Default)]
pub struct ShipperState {
    /// Per-peer high-water-mark: the highest seqno each peer has acknowledged.
    peer_hwm: DashMap<ShardId, u64>,
    /// Peers quarantined after a PERMANENT send failure. Value is the seqno the
    /// peer was stuck on when quarantined (for operator diagnostics).
    quarantined: DashMap<ShardId, u64>,
}

impl ShipperState {
    /// Build a fresh state with every configured peer at HWM 0 and no
    /// quarantines.
    #[must_use]
    fn new(peers: &[ShardId]) -> Self {
        let peer_hwm = DashMap::new();
        for &p in peers {
            peer_hwm.insert(p, 0);
        }
        Self {
            peer_hwm,
            quarantined: DashMap::new(),
        }
    }

    /// The high-water-mark (last acknowledged seqno) for `peer`, or 0 if unknown.
    #[must_use]
    pub fn peer_hwm(&self, peer: ShardId) -> u64 {
        self.peer_hwm.get(&peer).map_or(0, |r| *r)
    }

    /// Snapshot of every tracked peer's high-water-mark.
    #[must_use]
    pub fn peer_hwms(&self) -> HashMap<ShardId, u64> {
        self.peer_hwm
            .iter()
            .map(|r| (*r.key(), *r.value()))
            .collect()
    }

    /// `true` if `peer` is currently quarantined (permanently failed).
    #[must_use]
    pub fn is_quarantined(&self, peer: ShardId) -> bool {
        self.quarantined.contains_key(&peer)
    }

    /// The set of currently-quarantined peers, each with the seqno it was stuck
    /// on when quarantined.
    #[must_use]
    pub fn quarantined_peers(&self) -> HashMap<ShardId, u64> {
        self.quarantined
            .iter()
            .map(|r| (*r.key(), *r.value()))
            .collect()
    }

    /// Clear `peer`'s quarantine so the shipper retries it on the next poll.
    ///
    /// Intended for the operator endpoint after a peer's config/TLS/CA has been
    /// fixed. Returns `true` if the peer was quarantined (and is now cleared),
    /// `false` if it was not quarantined. The peer resumes from its existing
    /// high-water-mark, so no segment is re-shipped that the peer already acked.
    pub fn clear_quarantine(&self, peer: ShardId) -> bool {
        let was = self.quarantined.remove(&peer).is_some();
        if was {
            tracing::info!(
                peer = %peer,
                "shipper: quarantine cleared by operator; peer will be retried next poll"
            );
        }
        was
    }

    // ── Internal mutators used by the shipper thread ────────────────────────

    fn advance_hwm(&self, peer: ShardId, seqno: u64) {
        self.peer_hwm.insert(peer, seqno);
    }

    fn quarantine(&self, peer: ShardId, stuck_seqno: u64) {
        self.quarantined.insert(peer, stuck_seqno);
    }
}

/// Handle to a running WAL shipper thread.
///
/// Call [`stop`](Self::stop) to signal shutdown and join the thread.
pub struct WalShipperHandle {
    shutdown_tx: crossbeam::channel::Sender<()>,
    thread: Option<JoinHandle<()>>,
    /// Shared, operator-visible shipper state (per-peer HWM + quarantine set).
    state: Arc<ShipperState>,
}

impl WalShipperHandle {
    /// Signal the shipper to stop and join the background thread.
    pub fn stop(mut self) {
        let _ = self.shutdown_tx.send(());
        if let Some(handle) = self.thread.take() {
            let _ = handle.join();
        }
    }

    /// The shared shipper state for operator/control-plane inspection and
    /// quarantine clearing. The returned `Arc` stays valid after [`stop`](Self::stop).
    #[must_use]
    pub fn state(&self) -> Arc<ShipperState> {
        Arc::clone(&self.state)
    }

    /// Clear a peer's quarantine without restarting the shipper.
    ///
    /// Returns `true` if the peer was quarantined and is now cleared. See
    /// [`ShipperState::clear_quarantine`].
    #[must_use]
    pub fn clear_quarantine(&self, peer: ShardId) -> bool {
        self.state.clear_quarantine(peer)
    }
}

/// Spawn a background thread that polls for sealed WAL segments and ships
/// them to all peer shards.
///
/// The shipper identifies sealed segments as those with a newer segment file
/// after them in sequence order. The most recent segment is skipped because
/// it may still be actively written.
///
/// # Panics
///
/// Panics if the OS fails to spawn the background thread.
#[allow(clippy::too_many_lines)] // linear shipper setup + per-peer send/quarantine loop
pub fn spawn_shipper<T: Transport + ?Sized>(
    config: ShipperConfig,
    transport: Arc<T>,
) -> WalShipperHandle {
    let (shutdown_tx, shutdown_rx) = bounded::<()>(1);
    // Shared, operator-visible per-peer HWM + quarantine set. Owned by both the
    // handle (for inspection / clear_quarantine) and the shipper thread.
    let state = Arc::new(ShipperState::new(&config.peer_shards));
    let thread_state = Arc::clone(&state);
    let thread = std::thread::Builder::new()
        .name("tidaldb-wal-shipper".into())
        .spawn(move || {
            // Per-peer high-water-mark and the quarantine set now live in the
            // shared `thread_state` (an `Arc<ShipperState>`) instead of on this
            // thread's stack, so an operator can observe quarantined peers / HWMs
            // and clear a quarantine via the handle without restarting the
            // shipper. The shipper still advances a peer's cursor ONLY when ITS
            // own send succeeds, and re-reads the quarantine set at the top of
            // each peer's run so a concurrent clear takes effect next poll.
            //
            // A peer hits PERMANENT failure (bad TLS/CA, auth rejection,
            // malformed payload, unimplemented RPC) and is quarantined: skipped
            // on every subsequent poll, cursor frozen, until an operator clears
            // it. This trades a silent infinite retry stall for a loud,
            // actionable, bounded failure. Healthy peers are unaffected.
            let state = thread_state;
            loop {
                // Sleep for poll_interval, exit early on shutdown signal.
                match shutdown_rx.recv_timeout(config.poll_interval) {
                    Ok(()) | Err(crossbeam::channel::RecvTimeoutError::Disconnected) => {
                        tracing::debug!("wal shipper shutting down");
                        return;
                    }
                    Err(crossbeam::channel::RecvTimeoutError::Timeout) => {}
                }

                let segments = match crate::wal::segment::list_segments_for_shard(
                    &config.wal_dir,
                    config.shard_id,
                ) {
                    Ok(s) => s,
                    Err(e) => {
                        tracing::warn!(error = %e, "shipper: failed to list WAL segments");
                        continue;
                    }
                };

                // Only ship segments that have a newer segment after them
                // (the last segment may still be actively written).
                let sealed_count = if segments.len() > 1 {
                    segments.len() - 1
                } else {
                    0
                };
                let sealed = &segments[..sealed_count];

                // Cache segment bytes read this poll so peers at the same lag
                // don't each re-read the same file. Keyed by seqno. Each entry
                // carries the (possibly filtered) shippable bytes AND the
                // segment's authoritative last WAL seqno computed from the
                // ORIGINAL (pre-filter) bytes — the latter is what the receiver's
                // lag gauge needs so an all-local empty filtration still advances
                // the leader high-water-mark.
                let mut bytes_cache: HashMap<u64, (Vec<u8>, u64)> = HashMap::new();

                // Ship each peer its own contiguous run of segments, in seqno
                // order, starting just past its high-water-mark. A peer is
                // advanced ONLY when its own send succeeds; on the first
                // failure we stop that peer for this poll so it receives a
                // gap-free, in-order stream (the receiver applies strictly by
                // seqno — a skipped segment would stall it). The peer resumes
                // from the same un-acked seqno on the next poll.
                for &peer in &config.peer_shards {
                    // A quarantined peer hit a permanent failure earlier; never
                    // re-attempt it (re-shipping would fail identically). It
                    // stays skipped until an operator fixes the peer/config and
                    // restarts the shipper.
                    if state.is_quarantined(peer) {
                        continue;
                    }
                    for (seqno, path) in sealed {
                        if *seqno <= state.peer_hwm(peer) {
                            continue;
                        }

                        // Read (and cache) the segment bytes lazily. In
                        // community-overlay mode, strip local-scope events from
                        // the segment before caching so no peer ever receives a
                        // local signal. We compute the authoritative last WAL
                        // seqno from the ORIGINAL bytes BEFORE filtering, so an
                        // all-local segment (which filters to empty) still carries
                        // the leader's true high-water-mark for the receiver's lag
                        // gauge.
                        let entry = match bytes_cache.get(seqno) {
                            Some(b) => b,
                            None => match std::fs::read(path) {
                                Ok(original) => {
                                    let leader_last_seq = last_seq_in_segment(&original);
                                    let shippable = if config.community_share_only {
                                        filter_segment_drop_local(&original)
                                    } else {
                                        original
                                    };
                                    bytes_cache
                                        .entry(*seqno)
                                        .or_insert((shippable, leader_last_seq))
                                }
                                Err(e) => {
                                    tracing::warn!(
                                        error = %e,
                                        path = %path.display(),
                                        "shipper: failed to read segment"
                                    );
                                    // Cannot ship this (or any later) segment to
                                    // this peer without the bytes; stop the
                                    // peer's run so it stays gap-free.
                                    break;
                                }
                            },
                        };
                        let (segment_bytes, leader_last_seq) = entry;
                        let event_count = count_events_in_segment(segment_bytes);
                        let payload = WalSegmentPayload {
                            id: WalSegmentId::new(RegionId::SINGLE, config.shard_id, *seqno),
                            bytes: segment_bytes.clone(),
                            event_count,
                            leader_last_seq: *leader_last_seq,
                            stream_baseline: 0,
                            term: 0,
                            leader_region: 0,
                        };
                        match transport.send_segment(peer, payload) {
                            Ok(()) => {
                                tracing::debug!(
                                    seqno = *seqno,
                                    peer = %peer,
                                    "shipped WAL segment"
                                );
                                // Advance ONLY this peer's cursor on its own
                                // successful send.
                                state.advance_hwm(peer, *seqno);
                            }
                            Err(TransportError::Closed) => {
                                // Transient (peer down, circuit open, or
                                // follower backpressure): leave this peer's HWM
                                // untouched and stop its run so the un-acked
                                // segment is re-shipped, in order, next poll.
                                tracing::warn!(
                                    peer = %peer,
                                    seqno = *seqno,
                                    "shipper: transport closed, will retry peer next poll"
                                );
                                break;
                            }
                            Err(TransportError::Permanent { reason }) => {
                                // PERMANENT: re-shipping this seqno will fail
                                // identically forever (bad TLS/CA, auth
                                // rejection, malformed payload, unimplemented
                                // RPC). Quarantine the peer instead of stalling
                                // on an unbounded silent retry: stop advancing
                                // it, skip it on every future poll, and surface
                                // a LOUD, actionable error. The peer's HWM is
                                // left untouched so no segment is silently
                                // skipped. The quarantine is now operator-clearable
                                // via WalShipperHandle::clear_quarantine — no
                                // shipper restart required once the peer/config is
                                // fixed.
                                state.quarantine(peer, *seqno);
                                tracing::error!(
                                    peer = %peer,
                                    seqno = *seqno,
                                    reason = %reason,
                                    "shipper: PERMANENT transport failure — peer quarantined, \
                                     replication to this peer is STOPPED until an operator clears \
                                     the quarantine (clear_quarantine) after fixing peer/config; \
                                     retrying without a fix would not help"
                                );
                                break;
                            }
                            Err(e) => {
                                // Other failures are also non-advancing: stop the
                                // peer's run so we retry this seqno later without
                                // creating a gap.
                                tracing::warn!(
                                    error = %e,
                                    peer = %peer,
                                    seqno = *seqno,
                                    "shipper: send failed, will retry peer next poll"
                                );
                                break;
                            }
                        }
                    }
                }
            }
        })
        .expect("failed to spawn wal shipper thread");

    WalShipperHandle {
        shutdown_tx,
        thread: Some(thread),
        state,
    }
}

/// Strip every [`SignalScope::Local`] event from a WAL segment, re-encoding
/// each batch with only its shippable (non-local) events.
///
/// Used by community-overlay shippers ([`ShipperConfig::community_share_only`])
/// to enforce the invariant that local-scoped signals never leave the node.
/// Each surviving batch preserves its `batch_timestamp` and shard/region
/// identity; batches whose events are entirely local are dropped. A segment
/// with no shippable events returns empty bytes, which the receiver applies as
/// a no-op.
///
/// # Preserving the WAL position (CRITICAL: idempotency + lag boundary)
///
/// The receiver derives a batch's last WAL sequence number as
/// `first_seq + event_count - 1` and uses it both as the per-segment
/// idempotency boundary and as the value that advances
/// [`ReplicationState`](crate::replication::ReplicationState) /
/// [`ReplicationLagGauge`](crate::replication::ReplicationLagGauge). If we kept
/// the original `first_seq` while shrinking `event_count`, the derived last seq
/// would *under-report* the true WAL position by exactly the number of dropped
/// events — silently lowering the high-water-mark, breaking the
/// `lag = leader_seqno - applied_seqno` invariant, and letting a later replay
/// re-apply the segment.
///
/// To keep the boundary authoritative we **re-stamp `first_seq`** so that the
/// re-encoded batch's last seq is identical to the original batch's last seq:
///
/// ```text
/// orig_last      = orig_first_seq + orig_event_count - 1
/// new_first_seq  = orig_last - (kept_count - 1)
///                = orig_first_seq + orig_event_count - kept_count
/// new_last       = new_first_seq + kept_count - 1   ==   orig_last  ✓
/// ```
///
/// The receiver never assigns per-event seqnos — it only reads the last-seq
/// boundary — so collapsing the surviving (non-contiguous) events onto a fresh
/// contiguous `first_seq..=orig_last` range is correct and keeps replay
/// idempotent.
///
/// Corrupt or trailing bytes that fail to decode terminate the scan (matching
/// [`count_events_in_segment`]); already-decoded batches are preserved.
///
/// [`SignalScope::Local`]: crate::governance::SignalScope::Local
#[must_use]
pub fn filter_segment_drop_local(bytes: &[u8]) -> Vec<u8> {
    use crate::governance::SignalScope;

    let mut out = Vec::with_capacity(bytes.len());
    let mut offset = 0;
    while offset < bytes.len() {
        let remaining = &bytes[offset..];
        if remaining.len() < HEADER_SIZE {
            break;
        }
        match crate::wal::format::decode_batch_payload(remaining) {
            Ok((header, payload)) => {
                let batch_size = HEADER_SIZE + header.payload_len as usize;
                offset += batch_size;

                // Blob (item-metadata / embedding) batches carry global config
                // data, not scoped signals: the M9 local-scope invariant does
                // not apply, so they pass through verbatim (byte-identical,
                // checksum intact).
                let events = match payload {
                    crate::wal::format::BatchPayload::Signals(events) => events,
                    crate::wal::format::BatchPayload::ItemMetadata(_)
                    | crate::wal::format::BatchPayload::Embedding(_)
                    | crate::wal::format::BatchPayload::TermMarker(_) => {
                        out.extend_from_slice(&remaining[..batch_size]);
                        continue;
                    }
                };

                // Authoritative last WAL seq of the *original* batch, before any
                // events are dropped. This is the value the receiver must end up
                // deriving so the high-water-mark never regresses.
                let orig_last_seq = header
                    .first_seq
                    .saturating_add(u64::from(header.event_count).saturating_sub(1));

                let kept: Vec<_> = events
                    .into_iter()
                    .filter(|e| {
                        // Unknown/corrupt discriminants are conservatively
                        // dropped (never shipped) rather than panicking.
                        SignalScope::from_discriminant(e.scope).is_ok_and(SignalScope::is_shippable)
                    })
                    .collect();

                if kept.is_empty() {
                    continue;
                }

                // Re-stamp first_seq so first_seq + kept_count - 1 == orig_last_seq.
                // saturating_sub guards against a malformed header where
                // event_count under-counts the kept events (kept can never exceed
                // the decoded events, so this only clamps pathological input).
                let kept_count = kept.len() as u64;
                let new_first_seq = orig_last_seq.saturating_sub(kept_count.saturating_sub(1));

                // Re-encode the kept events, carrying the WAL-position-preserving
                // first_seq. Encoding cannot fail here: `kept` is non-empty and
                // bounded by the original batch's event count
                // (<= MAX_EVENTS_PER_BATCH).
                if let Ok(encoded) = encode_batch_with_shard(
                    &kept,
                    new_first_seq,
                    header.batch_timestamp,
                    header.shard_id,
                    header.region_id,
                ) {
                    out.extend_from_slice(&encoded);
                }
            }
            Err(_) => break,
        }
    }
    out
}

/// Compute a WAL segment's authoritative last sequence number from its
/// **original** (pre-filter) bytes: the maximum `first_seq + event_count - 1`
/// across all decodable batches.
///
/// This is the value the leader knows before any community-overlay filtering,
/// and the value the receiver feeds to the replication lag gauge's leader
/// high-water-mark. Computing it from the original bytes — not the filtered
/// ones — is load-bearing: an all-local segment filters to empty and would
/// otherwise report a leader seqno of 0, making the receiver's gauge
/// under-report the leader's progress across that segment (obs-REPL-1).
///
/// Returns `0` for an empty or fully-undecodable segment (the receiver treats
/// `0` as "unknown" and falls back to the per-batch boundaries it decodes).
/// Corrupt or trailing bytes terminate the scan (matching
/// [`count_events_in_segment`]); the last seq of the already-decoded batches is
/// returned.
fn last_seq_in_segment(bytes: &[u8]) -> u64 {
    let mut offset = 0;
    let mut last_seq = 0u64;
    while offset < bytes.len() {
        let remaining = &bytes[offset..];
        if remaining.len() < HEADER_SIZE {
            break;
        }
        // Kind-aware decode so blob (item-metadata / embedding) batches —
        // which consume one seqno each — keep advancing the boundary instead
        // of terminating the scan early.
        match crate::wal::format::decode_batch_payload(remaining) {
            Ok((header, _payload)) => {
                let batch_last_seq = header
                    .first_seq
                    .saturating_add(u64::from(header.event_count).saturating_sub(1));
                last_seq = last_seq.max(batch_last_seq);
                let batch_size = HEADER_SIZE + header.payload_len as usize;
                offset += batch_size;
            }
            Err(_) => break,
        }
    }
    last_seq
}

/// Count the total number of events in a WAL segment by parsing all batches.
fn count_events_in_segment(bytes: &[u8]) -> u64 {
    let mut offset = 0;
    let mut total = 0u64;
    while offset < bytes.len() {
        let remaining = &bytes[offset..];
        if remaining.len() < HEADER_SIZE {
            break;
        }
        match crate::wal::format::decode_batch_payload(remaining) {
            Ok((header, _payload)) => {
                total += u64::from(header.event_count);
                let batch_size = HEADER_SIZE + header.payload_len as usize;
                offset += batch_size;
            }
            Err(_) => break,
        }
    }
    total
}

#[cfg(test)]
#[allow(clippy::unwrap_used)]
mod tests {
    use std::{collections::HashSet, sync::Mutex};

    use super::*;
    use crate::wal::{
        format::batch::{EventRecord, RECORD_TYPE_SIGNAL, decode_batch, encode_batch},
        segment::segment_filename,
    };

    use crate::governance::SignalScope;

    /// A community-scoped (shippable) event.
    fn community_event(id: u64) -> EventRecord {
        EventRecord::scoped(
            id,
            RECORD_TYPE_SIGNAL,
            1.0,
            id * 100,
            SignalScope::Community(crate::governance::CommunityId(1)).discriminant(),
            0,
            1,
            0,
        )
    }

    #[test]
    fn count_events_empty() {
        assert_eq!(count_events_in_segment(&[]), 0);
    }

    #[test]
    fn drop_local_keeps_only_shippable() {
        // One local (dropped) + one community (kept) event in a batch.
        let events = vec![
            EventRecord::signal(1, RECORD_TYPE_SIGNAL, 1.0, 100),
            community_event(2),
        ];
        let bytes = encode_batch(&events, 10, 99).unwrap();
        let filtered = filter_segment_drop_local(&bytes);
        assert_eq!(count_events_in_segment(&filtered), 1);
        // Verify the surviving event is the community one.
        let (header, kept) = decode_batch(&filtered).unwrap();
        // CRITICAL fix: first_seq is re-stamped so the derived last seq
        // (first_seq + event_count - 1) still equals the ORIGINAL batch's last
        // seq (10 + 2 - 1 = 11) even though one event was dropped.
        let orig_last_seq = 10 + 2 - 1;
        let filtered_last_seq = header.first_seq + u64::from(header.event_count) - 1;
        assert_eq!(
            filtered_last_seq, orig_last_seq,
            "filtered batch must preserve the original WAL last-seq boundary"
        );
        assert_eq!(
            header.first_seq, 11,
            "first_seq re-stamped to preserve last seq"
        );
        assert_eq!(header.batch_timestamp, 99);
        assert_eq!(kept.len(), 1);
        assert_eq!(kept[0].entity_id, 2);
        assert_eq!(
            kept[0].scope,
            SignalScope::Community(crate::governance::CommunityId(1)).discriminant()
        );
    }

    #[test]
    fn drop_local_all_local_yields_empty() {
        let events = vec![
            EventRecord::signal(1, RECORD_TYPE_SIGNAL, 1.0, 100),
            EventRecord::signal(2, RECORD_TYPE_SIGNAL, 1.0, 200),
        ];
        let bytes = encode_batch(&events, 1, 1).unwrap();
        let filtered = filter_segment_drop_local(&bytes);
        assert!(
            filtered.is_empty(),
            "all-local segment must filter to empty"
        );
        assert_eq!(count_events_in_segment(&filtered), 0);
    }

    #[test]
    fn drop_local_multi_batch_preserves_shippable() {
        // Batch 1: all local (dropped). Batch 2: mixed (one kept).
        let b1 = encode_batch(
            &[EventRecord::signal(1, RECORD_TYPE_SIGNAL, 1.0, 100)],
            1,
            1,
        )
        .unwrap();
        let b2 = encode_batch(
            &[
                community_event(7),
                EventRecord::signal(8, RECORD_TYPE_SIGNAL, 1.0, 800),
            ],
            2,
            2,
        )
        .unwrap();
        let mut seg = b1;
        seg.extend(b2);
        let filtered = filter_segment_drop_local(&seg);
        assert_eq!(count_events_in_segment(&filtered), 1);
        let (header, kept) = decode_batch(&filtered).unwrap();
        assert_eq!(kept[0].entity_id, 7);
        // b2 had first_seq=2, 2 events → orig last seq = 3. After dropping the
        // local event the surviving batch must still derive last seq 3.
        let last_seq = header.first_seq + u64::from(header.event_count) - 1;
        assert_eq!(last_seq, 3, "WAL last-seq boundary preserved across filter");
    }

    #[test]
    fn drop_local_noop_when_all_shippable() {
        let events = vec![community_event(1), community_event(2)];
        let bytes = encode_batch(&events, 5, 5).unwrap();
        let filtered = filter_segment_drop_local(&bytes);
        assert_eq!(count_events_in_segment(&filtered), 2);
    }

    #[test]
    fn count_events_single_batch() {
        let events = vec![EventRecord::signal(1, RECORD_TYPE_SIGNAL, 1.0, 100)];
        let bytes = encode_batch(&events, 1, 1).unwrap();
        assert_eq!(count_events_in_segment(&bytes), 1);
    }

    #[test]
    fn count_events_multiple_batches() {
        let e1 = vec![EventRecord::signal(1, RECORD_TYPE_SIGNAL, 1.0, 100)];
        let e2: Vec<EventRecord> = (0..5)
            .map(|i| EventRecord::signal(i, RECORD_TYPE_SIGNAL, 1.0, 200))
            .collect();
        let mut bytes = encode_batch(&e1, 1, 1).unwrap();
        bytes.extend(encode_batch(&e2, 2, 2).unwrap());
        assert_eq!(count_events_in_segment(&bytes), 6);
    }

    #[test]
    fn count_events_truncated_segment() {
        let events = vec![EventRecord::signal(1, RECORD_TYPE_SIGNAL, 1.0, 100)];
        let mut bytes = encode_batch(&events, 1, 1).unwrap();
        // Append garbage that does not form a valid batch header.
        bytes.extend(&[0xFFu8; 10]);
        // Should still count the first valid batch.
        assert_eq!(count_events_in_segment(&bytes), 1);
    }

    #[test]
    fn shipper_config_default() {
        let config = ShipperConfig::default();
        assert_eq!(config.shard_id, ShardId::SINGLE);
        assert!(config.peer_shards.is_empty());
        assert_eq!(config.poll_interval, Duration::from_secs(2));
    }

    // ── Per-peer high-water-mark behavior (REPL-2) ───────────────────────

    /// A fault-injecting transport that records every (peer, seqno) it is
    /// asked to ship and fails sends to a configurable set of peers.
    ///
    /// Failing peers receive `TransportError::Closed` (the transient code the
    /// gRPC client returns for follower backpressure / unreachable peers), so
    /// the shipper must NOT advance their cursor.
    struct RecordingTransport {
        /// Peers whose sends should fail with `Closed`.
        failing: Mutex<HashSet<ShardId>>,
        /// Peers whose sends should fail with `Permanent` (e.g. a bad TLS chain
        /// that never becomes shippable). The shipper must quarantine these
        /// rather than retry them forever.
        permanent: Mutex<HashSet<ShardId>>,
        /// Per-peer ordered log of seqnos the shipper attempted to send.
        sent: Mutex<HashMap<ShardId, Vec<u64>>>,
    }

    impl RecordingTransport {
        fn new(failing: &[ShardId]) -> Self {
            Self {
                failing: Mutex::new(failing.iter().copied().collect()),
                permanent: Mutex::new(HashSet::new()),
                sent: Mutex::new(HashMap::new()),
            }
        }

        /// Build a transport whose sends to `permanent` peers fail with a
        /// non-retryable `TransportError::Permanent`.
        fn with_permanent(permanent: &[ShardId]) -> Self {
            Self {
                failing: Mutex::new(HashSet::new()),
                permanent: Mutex::new(permanent.iter().copied().collect()),
                sent: Mutex::new(HashMap::new()),
            }
        }

        fn seqnos_for(&self, peer: ShardId) -> Vec<u64> {
            self.sent
                .lock()
                .unwrap()
                .get(&peer)
                .cloned()
                .unwrap_or_default()
        }

        fn heal(&self, peer: ShardId) {
            self.failing.lock().unwrap().remove(&peer);
        }

        /// Clear a peer from the PERMANENT-failure set (the operator fixed the
        /// peer's config/TLS). Used together with `clear_quarantine` to prove a
        /// quarantined peer can recover without a shipper restart.
        fn heal_permanent(&self, peer: ShardId) {
            self.permanent.lock().unwrap().remove(&peer);
        }
    }

    impl Transport for RecordingTransport {
        fn send_segment(
            &self,
            to: ShardId,
            payload: WalSegmentPayload,
        ) -> Result<(), TransportError> {
            self.sent
                .lock()
                .unwrap()
                .entry(to)
                .or_default()
                .push(payload.id.seqno);
            if self.permanent.lock().unwrap().contains(&to) {
                Err(TransportError::Permanent {
                    reason: "test: mutually-distrusted TLS chain".into(),
                })
            } else if self.failing.lock().unwrap().contains(&to) {
                Err(TransportError::Closed)
            } else {
                Ok(())
            }
        }

        fn recv_segment(&self) -> Option<WalSegmentPayload> {
            None
        }

        fn local_shard(&self) -> ShardId {
            ShardId::SINGLE
        }
    }

    /// Write `count` sealed segments (seqnos 1..=count) plus one extra "active"
    /// segment so all `count` are considered sealed by the shipper.
    fn seed_sealed_segments(dir: &std::path::Path, count: u64) {
        for seq in 1..=count + 1 {
            let events = vec![EventRecord::signal(seq, RECORD_TYPE_SIGNAL, 1.0, seq * 100)];
            let bytes = encode_batch(&events, seq, seq).unwrap();
            let path = dir.join(segment_filename(ShardId::SINGLE, seq));
            std::fs::write(path, bytes).unwrap();
        }
    }

    /// Spin until `cond` is true or the deadline passes, polling cheaply.
    fn wait_until(deadline: Duration, mut cond: impl FnMut() -> bool) {
        let start = std::time::Instant::now();
        while start.elapsed() < deadline {
            if cond() {
                return;
            }
            std::thread::sleep(Duration::from_millis(5));
        }
    }

    #[test]
    fn failing_peer_does_not_skip_segments() {
        let dir = tempfile::tempdir().unwrap();
        seed_sealed_segments(dir.path(), 3);

        let healthy = ShardId(1);
        let broken = ShardId(2);
        let transport = Arc::new(RecordingTransport::new(&[broken]));

        let config = ShipperConfig {
            wal_dir: dir.path().to_path_buf(),
            shard_id: ShardId::SINGLE,
            peer_shards: vec![healthy, broken],
            poll_interval: Duration::from_millis(10),
            community_share_only: false,
        };
        let handle = spawn_shipper(config, Arc::clone(&transport));

        // The healthy peer should receive all three sealed segments exactly once.
        wait_until(Duration::from_secs(2), || {
            transport.seqnos_for(healthy) == vec![1, 2, 3]
        });
        assert_eq!(
            transport.seqnos_for(healthy),
            vec![1, 2, 3],
            "healthy peer should receive every sealed segment once"
        );

        // The broken peer keeps being re-sent the lowest un-acked segment (1):
        // its cursor never advanced, so it never silently skipped 2 or 3.
        wait_until(Duration::from_secs(2), || {
            transport.seqnos_for(broken).len() >= 3
        });
        let broken_sent = transport.seqnos_for(broken);
        assert!(
            broken_sent.len() >= 3,
            "broken peer should be retried, got: {broken_sent:?}"
        );
        assert!(
            broken_sent.iter().all(|&s| s == 1),
            "broken peer should keep re-receiving seqno 1 (the un-acked floor), got: {broken_sent:?}"
        );

        handle.stop();
    }

    #[test]
    fn healed_peer_catches_up() {
        let dir = tempfile::tempdir().unwrap();
        seed_sealed_segments(dir.path(), 3);

        let broken = ShardId(2);
        let transport = Arc::new(RecordingTransport::new(&[broken]));

        let config = ShipperConfig {
            wal_dir: dir.path().to_path_buf(),
            shard_id: ShardId::SINGLE,
            peer_shards: vec![broken],
            poll_interval: Duration::from_millis(10),
            community_share_only: false,
        };
        let handle = spawn_shipper(config, Arc::clone(&transport));

        // While broken, the peer only ever sees seqno 1 retried.
        wait_until(Duration::from_secs(2), || {
            transport.seqnos_for(broken).len() >= 2
        });
        assert!(
            transport.seqnos_for(broken).iter().all(|&s| s == 1),
            "broken peer should only retry seqno 1 before healing"
        );

        // Heal the peer; it must now advance through 1, 2, 3 in order.
        transport.heal(broken);
        wait_until(Duration::from_secs(2), || {
            let sent = transport.seqnos_for(broken);
            sent.contains(&1) && sent.contains(&2) && sent.contains(&3)
        });
        let sent = transport.seqnos_for(broken);
        assert!(
            sent.contains(&1) && sent.contains(&2) && sent.contains(&3),
            "healed peer should catch up to all sealed segments, got: {sent:?}"
        );

        handle.stop();
    }

    /// A peer that returns a PERMANENT failure must be quarantined: the shipper
    /// attempts it once, then stops re-shipping the same seqno forever. This is
    /// the core fix — a permanent fault (bad TLS/CA, auth rejection, malformed
    /// payload, unimplemented RPC) used to map to the transient `Closed`
    /// variant and stall replication on an unbounded silent retry of seqno 1.
    #[test]
    fn permanent_failure_quarantines_peer_no_infinite_retry() {
        let dir = tempfile::tempdir().unwrap();
        seed_sealed_segments(dir.path(), 3);

        let permanent_peer = ShardId(2);
        let transport = Arc::new(RecordingTransport::with_permanent(&[permanent_peer]));

        let config = ShipperConfig {
            wal_dir: dir.path().to_path_buf(),
            shard_id: ShardId::SINGLE,
            peer_shards: vec![permanent_peer],
            // Fast poll so several poll cycles elapse within the test window;
            // a non-quarantined peer would accumulate many retries of seqno 1.
            poll_interval: Duration::from_millis(5),
            community_share_only: false,
        };
        let handle = spawn_shipper(config, Arc::clone(&transport));

        // Wait for the first (and only) send attempt to land.
        wait_until(Duration::from_secs(2), || {
            !transport.seqnos_for(permanent_peer).is_empty()
        });

        // Let MANY poll intervals elapse. A transient-`Closed` peer would by
        // now have re-shipped seqno 1 dozens of times; a quarantined peer must
        // not be re-attempted at all.
        std::thread::sleep(Duration::from_millis(150));

        let sent = transport.seqnos_for(permanent_peer);
        assert_eq!(
            sent,
            vec![1],
            "permanent-failure peer must be quarantined after exactly one attempt \
             (no infinite retry of seqno 1), got: {sent:?}"
        );

        handle.stop();
    }

    /// Quarantining a permanently-failing peer must NOT affect a healthy peer
    /// in the same shipper: the healthy peer still receives every sealed
    /// segment exactly once.
    #[test]
    fn permanent_failure_does_not_block_healthy_peer() {
        let dir = tempfile::tempdir().unwrap();
        seed_sealed_segments(dir.path(), 3);

        let healthy = ShardId(1);
        let permanent_peer = ShardId(2);
        let transport = Arc::new(RecordingTransport::with_permanent(&[permanent_peer]));

        let config = ShipperConfig {
            wal_dir: dir.path().to_path_buf(),
            shard_id: ShardId::SINGLE,
            peer_shards: vec![healthy, permanent_peer],
            poll_interval: Duration::from_millis(5),
            community_share_only: false,
        };
        let handle = spawn_shipper(config, Arc::clone(&transport));

        // Healthy peer receives all three sealed segments exactly once.
        wait_until(Duration::from_secs(2), || {
            transport.seqnos_for(healthy) == vec![1, 2, 3]
        });
        assert_eq!(
            transport.seqnos_for(healthy),
            vec![1, 2, 3],
            "healthy peer must receive every sealed segment despite a quarantined sibling"
        );

        // The permanent peer was attempted exactly once and then quarantined.
        std::thread::sleep(Duration::from_millis(80));
        assert_eq!(
            transport.seqnos_for(permanent_peer),
            vec![1],
            "permanent peer stays quarantined to one attempt"
        );

        handle.stop();
    }

    // ── Operator-visible quarantine/HWM state (Extensibility-W) ──────────────

    /// `last_seq_in_segment` returns the maximum `first_seq + event_count - 1`
    /// across all batches, computed from the ORIGINAL (pre-filter) bytes.
    #[test]
    fn last_seq_in_segment_is_max_batch_boundary() {
        // Batch A: first_seq=10, 1 event → last 10. Batch B: first_seq=11, 3
        // events → last 13. Segment last seq = 13.
        let a = encode_batch(
            &[EventRecord::signal(1, RECORD_TYPE_SIGNAL, 1.0, 100)],
            10,
            10,
        )
        .unwrap();
        let b = encode_batch(
            &(0..3)
                .map(|i| EventRecord::signal(i, RECORD_TYPE_SIGNAL, 1.0, 200))
                .collect::<Vec<_>>(),
            11,
            11,
        )
        .unwrap();
        let mut seg = a;
        seg.extend(b);
        assert_eq!(last_seq_in_segment(&seg), 13);
        // Empty segment → 0 (unknown).
        assert_eq!(last_seq_in_segment(&[]), 0);
    }

    /// The shipper's quarantine and per-peer HWM are visible to an operator via
    /// the handle's shared `ShipperState`, and a quarantine can be cleared
    /// WITHOUT restarting the shipper — after which the (now-healed) peer is
    /// retried and catches up.
    #[test]
    fn quarantine_and_hwm_visible_and_clearable_without_restart() {
        let dir = tempfile::tempdir().unwrap();
        seed_sealed_segments(dir.path(), 3);

        let healthy = ShardId(1);
        let permanent_peer = ShardId(2);
        let transport = Arc::new(RecordingTransport::with_permanent(&[permanent_peer]));

        let config = ShipperConfig {
            wal_dir: dir.path().to_path_buf(),
            shard_id: ShardId::SINGLE,
            peer_shards: vec![healthy, permanent_peer],
            poll_interval: Duration::from_millis(5),
            community_share_only: false,
        };
        let handle = spawn_shipper(config, Arc::clone(&transport));
        let state = handle.state();

        // The healthy peer advances; its HWM becomes observable as 3.
        wait_until(Duration::from_secs(2), || state.peer_hwm(healthy) == 3);
        assert_eq!(
            state.peer_hwm(healthy),
            3,
            "operator can read the healthy peer's high-water-mark"
        );

        // The permanent peer becomes observably quarantined (stuck at seqno 1).
        wait_until(Duration::from_secs(2), || {
            state.is_quarantined(permanent_peer)
        });
        let quarantined = state.quarantined_peers();
        assert_eq!(
            quarantined.get(&permanent_peer).copied(),
            Some(1),
            "quarantine set must expose the peer and the seqno it stuck on, got: {quarantined:?}"
        );

        // Operator fixes the peer's config, then clears the quarantine via the
        // handle — no shipper restart.
        transport.heal_permanent(permanent_peer);
        assert!(
            handle.clear_quarantine(permanent_peer),
            "clear_quarantine must report it cleared a live quarantine"
        );
        assert!(
            !state.is_quarantined(permanent_peer),
            "peer must no longer be quarantined after clear"
        );

        // The peer is now retried and catches up to all sealed segments.
        wait_until(Duration::from_secs(2), || {
            state.peer_hwm(permanent_peer) == 3
        });
        assert_eq!(
            state.peer_hwm(permanent_peer),
            3,
            "cleared peer resumes shipping and catches up without a restart"
        );

        // Clearing a non-quarantined peer reports false.
        assert!(!handle.clear_quarantine(ShardId(99)));

        handle.stop();
    }

    /// In community-overlay mode an all-local segment ships empty bytes but the
    /// payload must still carry the authoritative `leader_last_seq` so the
    /// receiver's lag gauge does not under-report (obs-REPL-1).
    #[test]
    fn all_local_segment_ships_empty_with_authoritative_leader_last_seq() {
        use std::sync::Mutex as StdMutex;

        /// Captures the last payload's (`bytes_len`, `leader_last_seq`) per send.
        struct CapturingTransport {
            captured: StdMutex<Vec<(usize, u64)>>,
        }
        impl Transport for CapturingTransport {
            fn send_segment(
                &self,
                _to: ShardId,
                payload: WalSegmentPayload,
            ) -> Result<(), TransportError> {
                self.captured
                    .lock()
                    .unwrap()
                    .push((payload.bytes.len(), payload.leader_last_seq));
                Ok(())
            }
            fn recv_segment(&self) -> Option<WalSegmentPayload> {
                None
            }
            fn local_shard(&self) -> ShardId {
                ShardId::SINGLE
            }
        }

        let dir = tempfile::tempdir().unwrap();
        // Write one sealed all-local segment (file seqno 1) whose single batch
        // has first_seq=4 and 2 events → true last WAL seq = 5; plus a trailing
        // active segment so the first is considered sealed.
        let all_local = encode_batch(
            &[
                EventRecord::signal(1, RECORD_TYPE_SIGNAL, 1.0, 100),
                EventRecord::signal(2, RECORD_TYPE_SIGNAL, 1.0, 200),
            ],
            4,
            4,
        )
        .unwrap();
        std::fs::write(
            dir.path().join(segment_filename(ShardId::SINGLE, 1)),
            all_local,
        )
        .unwrap();
        std::fs::write(
            dir.path().join(segment_filename(ShardId::SINGLE, 2)),
            encode_batch(&[community_event(9)], 6, 6).unwrap(),
        )
        .unwrap();

        let peer = ShardId(1);
        let transport = Arc::new(CapturingTransport {
            captured: StdMutex::new(Vec::new()),
        });
        let config = ShipperConfig {
            wal_dir: dir.path().to_path_buf(),
            shard_id: ShardId::SINGLE,
            peer_shards: vec![peer],
            poll_interval: Duration::from_millis(5),
            community_share_only: true,
        };
        let handle = spawn_shipper(config, Arc::clone(&transport));

        wait_until(Duration::from_secs(2), || {
            !transport.captured.lock().unwrap().is_empty()
        });
        let captured = transport.captured.lock().unwrap().clone();
        handle.stop();

        let (bytes_len, leader_last_seq) = captured[0];
        assert_eq!(bytes_len, 0, "all-local segment filters to empty bytes");
        assert_eq!(
            leader_last_seq, 5,
            "empty segment must still carry the leader's true last WAL seq (4 + 2 - 1)"
        );
    }
}