//! Segment receiver: consumes WAL segments from the transport and applies
//! them to the local signal ledger.
//!
//! The receiver runs in a background thread, blocking on
//! [`Transport::recv_segment`] and replaying each batch into the shared
//! [`SignalLedger`] via `apply_replicated_event`. Idempotent replay is ensured
//! by checking the per-shard high-water-mark in [`ReplicationState`].
//!
//! # Durability (BLOCKER 6)
//!
//! Each accepted event is applied **WAL-first** on the follower:
//! `apply_replicated_event` appends the event to the follower's *own* WAL and
//! fsyncs (via the group-commit writer) before mutating in-memory state. The
//! follower's normal WAL replay on open therefore reconstructs all replicated
//! state — without this, every replicated event between the last checkpoint and
//! a follower crash was silently lost. The leader-seqno high-water-mark is
//! persisted separately by the periodic checkpoint thread (a re-shipped segment
//! whose last seq is at or below the restored high-water-mark is an idempotent
//! no-op).
//!
//! # Error handling
//!
//! If a batch fails BLAKE3 verification or structural decode, `apply_payload`
//! returns [`WalError::Corruption`]. If the follower's own WAL append fails (a
//! disk fault), it returns [`WalError::Io`]. In both cases the receiver thread
//! propagates the error immediately (it does **not** skip the payload) so that
//! [`SegmentReceiverHandle::join`] can surface it to the operator — the
//! follower must never silently acknowledge an event it failed to durably
//! record.

use std::{
    sync::{
        Arc,
        atomic::{AtomicBool, Ordering},
    },
    thread::JoinHandle,
};

use crate::{
    replication::{
        lag::ReplicationLagGauge, shard::ShardId, state::ReplicationState, transport::Transport,
    },
    schema::{EntityId, Timestamp},
    signals::{SignalLedger, SignalTypeId},
    wal::{
        error::WalError,
        format::batch::{HEADER_SIZE, decode_batch},
    },
};

/// Handle to a running segment receiver thread.
///
/// Call [`join`](Self::join) to block until the thread exits and retrieve any
/// corruption error that caused it to stop. While the node is still open, poll
/// [`died`](Self::died) (or [`is_finished`](Self::is_finished)) to observe a
/// receiver that halted on an error mid-life — without it, a dead receiver is
/// indistinguishable from a healthy one until shutdown calls `join` (C11).
pub struct SegmentReceiverHandle {
    thread: Option<JoinHandle<Result<(), WalError>>>,
    /// Latched `true` exactly when the receiver thread halts on an apply error
    /// (corrupt segment / follower-WAL IO fault) or an unexpected exit — i.e. a
    /// silent replication stall. A clean `recv_segment() == None` shutdown leaves
    /// this `false`, so it distinguishes "the follower stopped applying because
    /// something broke" from "the follower was asked to shut down" (C11). Shared
    /// with the thread, which sets it before returning the error.
    died: Arc<AtomicBool>,
}

impl SegmentReceiverHandle {
    /// Block until the receiver thread exits.
    ///
    /// # Errors
    ///
    /// Returns `Err(WalError::Corruption { .. })` if a corrupt batch was
    /// received. Returns `Err(WalError::Corruption { .. })` if the thread
    /// panicked unexpectedly.
    pub fn join(mut self) -> Result<(), WalError> {
        self.thread.take().map_or(Ok(()), |handle| {
            handle.join().unwrap_or_else(|_| {
                Err(WalError::Corruption {
                    message: "segment receiver thread panicked".into(),
                })
            })
        })
    }

    /// Whether the receiver thread has halted on an error (corrupt segment,
    /// follower-WAL IO fault, or an unexpected exit) while the node is still
    /// open.
    ///
    /// `true` means replication has silently stopped applying and the follower
    /// is frozen — health checks must report degraded. A clean shutdown
    /// (`recv_segment() == None`) does **not** set this, so it does not
    /// false-positive on intentional teardown (C11). Cheap, lock-free, and safe
    /// to poll from `health_check` while the handle is still parked in its mutex.
    #[must_use]
    pub fn died(&self) -> bool {
        self.died.load(Ordering::Acquire)
    }

    /// Whether the receiver thread's join handle has finished.
    ///
    /// Mirrors [`std::thread::JoinHandle::is_finished`]. Unlike [`died`](Self::died)
    /// this is also `true` after a clean shutdown, so health checks should prefer
    /// `died` to avoid flagging an intentional teardown as a fault.
    #[must_use]
    pub fn is_finished(&self) -> bool {
        self.thread
            .as_ref()
            .is_some_and(std::thread::JoinHandle::is_finished)
    }
}

/// Spawn a background thread that receives WAL segments and replays them
/// into the signal ledger.
///
/// The thread exits when `transport.recv_segment()` returns `None` (transport
/// closed / shutdown) **or** when a corrupt batch is detected.
///
/// When `lag_gauge` is `Some`, every received segment advances the gauge's
/// leader high-water-mark to the highest WAL sequence the leader has shipped
/// (obs-REPL-1). The gauge's `applied_seqno` comes from the same
/// `ReplicationState` this receiver advances, so `lag_segments()` reflects real
/// follower backpressure (`leader_seqno − applied_seqno`).
///
/// # Panics
///
/// Panics if the OS fails to spawn the background thread.
pub fn spawn_receiver<T: Transport + ?Sized>(
    transport: Arc<T>,
    ledger: Arc<SignalLedger>,
    replication_state: Arc<ReplicationState>,
    lag_gauge: Option<Arc<ReplicationLagGauge>>,
) -> SegmentReceiverHandle {
    // Liveness latch (C11): set true iff the thread halts on an apply error.
    // Shared with the handle so health_check can observe a silently-dead
    // receiver while the node is still open. A clean shutdown leaves it false.
    let died = Arc::new(AtomicBool::new(false));
    let thread_died = Arc::clone(&died);
    let thread = std::thread::Builder::new()
        .name("tidaldb-segment-receiver".into())
        .spawn(move || -> Result<(), WalError> {
            loop {
                let Some(payload) = transport.recv_segment() else {
                    // Clean shutdown: the transport closed / shutdown was
                    // requested. Leave the `died` latch false so health_check
                    // does not flag an intentional teardown as a fault (C11).
                    tracing::debug!("segment receiver: transport closed, shutting down");
                    return Ok(());
                };

                let shard_id = payload.id.shard_id;
                // Carry the segment's authoritative leader high-water-mark (known
                // before community-overlay filtering) so the lag gauge tracks the
                // leader even across an all-local (empty) segment that decodes to
                // zero batches. `0` means "unknown" and falls back to per-batch
                // boundaries inside apply_payload_with_leader_seq.
                let leader_seqno = (payload.leader_last_seq > 0).then_some(payload.leader_last_seq);
                if let Err(e) = apply_payload_with_leader_seq(
                    &payload.bytes,
                    shard_id,
                    &ledger,
                    &replication_state,
                    lag_gauge.as_deref(),
                    leader_seqno,
                ) {
                    // A corrupt segment / follower-WAL fault halts the receiver
                    // thread. Latch `died` BEFORE returning so a stalled replica
                    // is observable via SegmentReceiverHandle::died() while the
                    // node is still open — not just at shutdown via join(). Log at
                    // the failure site too -- mirrors wal::reader's per-corruption
                    // logging, except replication cannot skip-and-continue (a gap
                    // would diverge the replica), so we halt and surface.
                    thread_died.store(true, Ordering::Release);
                    tracing::error!(
                        shard = %shard_id,
                        error = %e,
                        "replication apply failed; receiver halting (health degraded)"
                    );
                    return Err(e);
                }
            }
        })
        .expect("failed to spawn segment receiver thread");

    SegmentReceiverHandle {
        thread: Some(thread),
        died,
    }
}

/// Apply all batches in a WAL segment payload to the signal ledger.
///
/// Idempotent: batches whose last sequence number is at or below the
/// replication state's high-water-mark for the source shard are skipped.
///
/// # Atomic payload application (WARNING fix)
///
/// The payload is applied **all-or-nothing**. We first decode and validate
/// *every* batch (BLAKE3, structural decode, and the `first_seq + event_count`
/// overflow check) into a staged list, advancing only the lag gauge's *leader*
/// high-water-mark as boundaries are observed. Only after the whole payload
/// validates do we apply staged events to the follower ledger (WAL-first) and
/// advance the *applied* high-water-mark to the payload's final boundary in one
/// step.
///
/// This means a corrupt batch *anywhere* in the payload causes the function to
/// return `Err` **before** any event is applied or any applied-seqno is
/// advanced, so the follower's ledger and idempotency boundary never reflect a
/// half-applied payload. The leader can cleanly re-ship the same un-acked
/// segment and it replays from a single consistent resume point. (Without this,
/// a later corrupt batch left earlier batches applied with the applied-seqno
/// stuck mid-payload, with no clean point to resume from.)
///
/// `leader_seqno` is the segment's authoritative last WAL seqno as known by the
/// leader *before* community-overlay filtering (see [`WalSegmentPayload`]). It
/// feeds the lag gauge's leader high-water-mark regardless of how many batches
/// survived filtering — an all-local segment ships zero batches yet still
/// advances the leader HWM so the gauge does not under-report (obs-REPL-1,
/// WARNING fix). `None` falls back to the per-batch boundaries (legacy callers
/// / tests with no authoritative value).
///
/// When `lag_gauge` is `Some`, the gauge's applied seqno tracks
/// `state.applied_seqno`, so `lag_segments()` is non-zero whenever a payload has
/// been received from the leader but not yet applied, and returns to 0 once the
/// follower catches up.
///
/// # Errors
///
/// Returns `WalError::Corruption` on any BLAKE3 or structural decode failure
/// (or a wrapped seqno boundary). The offset of the first corrupt batch is
/// included in the error message. On error, no state has been mutated.
pub fn apply_payload(
    bytes: &[u8],
    from_shard: ShardId,
    ledger: &SignalLedger,
    state: &ReplicationState,
    lag_gauge: Option<&ReplicationLagGauge>,
) -> Result<(), WalError> {
    apply_payload_with_leader_seq(bytes, from_shard, ledger, state, lag_gauge, None)
}

/// Like [`apply_payload`] but carrying the segment's authoritative leader
/// high-water-mark (the true last WAL seqno before community-overlay filtering).
///
/// See [`apply_payload`] for the atomicity and leader-seqno contract. The
/// `leader_seqno` is fed to the lag gauge independently of how many batches
/// survive filtering, so an all-local (empty) segment still advances the leader
/// HWM.
///
/// # Errors
///
/// Returns `WalError::Corruption` on any decode/checksum/overflow failure,
/// before any state is mutated.
pub fn apply_payload_with_leader_seq(
    bytes: &[u8],
    from_shard: ShardId,
    ledger: &SignalLedger,
    state: &ReplicationState,
    lag_gauge: Option<&ReplicationLagGauge>,
    leader_seqno: Option<u64>,
) -> Result<(), WalError> {
    // ── Phase 1: decode + validate the WHOLE payload before mutating state ──
    //
    // Stage every batch's events and track the highest last-seq across the
    // payload. If any batch fails decode/checksum/overflow we return here, with
    // neither the follower ledger nor the applied high-water-mark touched.
    let applied_floor = state.applied_seqno(from_shard);
    let mut staged: Vec<(SignalTypeId, EntityId, f64, Timestamp)> = Vec::new();
    // The highest last-seq seen across all decoded batches. WAL batches in a
    // segment are ascending, but taking the max (rather than the trailing
    // batch's boundary) keeps the advance order-independent and monotonic.
    let mut payload_last_seq: Option<u64> = None;
    let mut max_batch_last_seq: u64 = 0;

    let mut offset = 0;
    while offset < bytes.len() {
        let remaining = &bytes[offset..];
        if remaining.len() < HEADER_SIZE {
            break;
        }
        let (header, events) = decode_batch(remaining).map_err(|e| WalError::Corruption {
            message: format!("corrupt batch at payload offset {offset}: {e}"),
        })?;

        // The last WAL seq is `first_seq + event_count - 1`. Both operands
        // originate from peer- or disk-controlled header bytes, so a
        // corrupt/hostile `first_seq` near `u64::MAX` could wrap. A wrapped
        // seqno would corrupt the idempotency boundary and the lag gauge, so we
        // reject the batch as corruption rather than applying it.
        // (`event_count == 0` is a legal empty batch whose boundary is just
        // `first_seq`.)
        let batch_last_seq = if header.event_count > 0 {
            header
                .first_seq
                .checked_add(u64::from(header.event_count) - 1)
                .ok_or_else(|| WalError::Corruption {
                    message: format!(
                        "WAL seq overflow at payload offset {offset}: \
                         first_seq={} + event_count={} would wrap u64",
                        header.first_seq, header.event_count
                    ),
                })?
        } else {
            header.first_seq
        };
        max_batch_last_seq = max_batch_last_seq.max(batch_last_seq);
        payload_last_seq = Some(max_batch_last_seq);

        // Idempotency: a batch whose boundary is at or below the already-applied
        // floor is a re-ship; stage nothing for it but still let its boundary
        // count toward the payload's final advance (the advance is monotonic, so
        // re-applying the same boundary is a no-op).
        let already_applied = applied_floor.is_some_and(|applied| batch_last_seq <= applied);
        if !already_applied {
            for event in &events {
                staged.push((
                    SignalTypeId::new(u16::from(event.signal_type)),
                    EntityId::new(event.entity_id),
                    f64::from(event.weight),
                    Timestamp::from_nanos(event.timestamp_nanos),
                ));
            }
        }

        let batch_size = HEADER_SIZE + header.payload_len as usize;
        offset += batch_size;
    }

    // Record the leader high-water-mark. Prefer the authoritative segment-level
    // value (known before filtering, so an all-local segment that decoded zero
    // batches still advances it); otherwise fall back to the highest per-batch
    // boundary we observed. The leader's seqno is known the moment the segment
    // arrives, independent of how many batches survived filtering or the
    // idempotency check, so `lag_segments() = leader_seqno − applied_seqno` is
    // the real follower lag.
    if let Some(gauge) = lag_gauge {
        let observed = leader_seqno.unwrap_or(max_batch_last_seq);
        if observed > 0 {
            gauge.update_leader_seqno(observed);
        }
    }

    // ── Phase 2: apply staged events, then advance the applied HWM once ──
    //
    // Every staged event is appended to THIS follower's own WAL (and fsynced by
    // the group-commit writer) before its in-memory aggregate is mutated, so the
    // follower's normal WAL replay on open reconstructs the applied state. A WAL
    // append failure (disk fault) returns before the high-water-mark advances,
    // so the follower never acknowledges an event it could not durably record —
    // the re-shipped segment is replayed in full from one consistent point.
    for (signal_type_id, entity_id, weight, timestamp) in &staged {
        ledger
            .apply_replicated_event(*signal_type_id, *entity_id, *weight, *timestamp)
            .map_err(|e| {
                WalError::Io(std::io::Error::other(format!(
                    "follower WAL append failed applying replicated event \
                     (entity={}, signal_type={}): {e}",
                    entity_id.as_u64(),
                    signal_type_id.as_u16()
                )))
            })?;
    }

    // Advance the replication high-water-mark to the payload's final boundary in
    // a single step, only AFTER every staged event is durably appended above.
    // This is the leader-seqno idempotency boundary the periodic checkpoint
    // persists; a re-shipped payload at or below it is skipped before
    // re-application.
    if let Some(last) = payload_last_seq {
        state.advance(from_shard, last);
    }

    Ok(())
}

#[cfg(test)]
#[allow(clippy::unwrap_used)]
mod tests {
    use std::time::Duration;

    use super::*;
    use crate::{
        replication::{segment_id::WalSegmentId, shard::RegionId},
        schema::{DecaySpec, SchemaBuilder, Window},
        signals::NoopWalWriter,
        wal::format::batch::{EventRecord, encode_batch},
    };

    fn make_schema() -> crate::schema::Schema {
        let mut builder = SchemaBuilder::new();
        let _ = builder
            .signal(
                "view",
                crate::schema::EntityKind::Item,
                DecaySpec::Exponential {
                    half_life: Duration::from_secs(7 * 24 * 3600),
                },
            )
            .windows(&[Window::AllTime])
            .velocity(false)
            .add();
        builder.build().unwrap()
    }

    fn make_event(entity_id: u64, signal_type: u8, ts_ns: u64) -> EventRecord {
        EventRecord::signal(entity_id, signal_type, 1.0, ts_ns)
    }

    #[test]
    fn apply_payload_updates_ledger() {
        let schema = make_schema();
        let ledger = Arc::new(SignalLedger::new(schema, Box::new(NoopWalWriter)));
        let state = Arc::new(ReplicationState::new(&[ShardId::SINGLE]));

        // Resolve the signal type for "view" to get the correct type id.
        let type_id = ledger.resolve_signal_type("view").unwrap();

        let events = vec![make_event(42, type_id.as_u16() as u8, 1_000_000_000)];
        let bytes = encode_batch(&events, 1, 1).unwrap();

        apply_payload(&bytes, ShardId::SINGLE, &ledger, &state, None).unwrap();

        // Verify the ledger was updated.
        assert!(ledger.entries().contains_key(&(EntityId::new(42), type_id)));
        assert_eq!(state.applied_seqno(ShardId::SINGLE), Some(1));
    }

    #[test]
    fn apply_payload_idempotent() {
        let schema = make_schema();
        let ledger = Arc::new(SignalLedger::new(schema, Box::new(NoopWalWriter)));
        let state = Arc::new(ReplicationState::new(&[ShardId::SINGLE]));

        let type_id = ledger.resolve_signal_type("view").unwrap();
        let events = vec![make_event(42, type_id.as_u16() as u8, 1_000_000_000)];
        let bytes = encode_batch(&events, 1, 1).unwrap();

        // Apply once.
        apply_payload(&bytes, ShardId::SINGLE, &ledger, &state, None).unwrap();
        // Apply again -- should be idempotent.
        apply_payload(&bytes, ShardId::SINGLE, &ledger, &state, None).unwrap();

        assert_eq!(state.applied_seqno(ShardId::SINGLE), Some(1));
    }

    #[test]
    fn apply_payload_multiple_batches() {
        let schema = make_schema();
        let ledger = Arc::new(SignalLedger::new(schema, Box::new(NoopWalWriter)));
        let state = Arc::new(ReplicationState::new(&[ShardId::SINGLE]));

        let type_id = ledger.resolve_signal_type("view").unwrap();
        let e1 = vec![make_event(1, type_id.as_u16() as u8, 100)];
        let e2 = vec![make_event(2, type_id.as_u16() as u8, 200)];

        let mut bytes = encode_batch(&e1, 1, 100).unwrap();
        bytes.extend(encode_batch(&e2, 2, 200).unwrap());

        apply_payload(&bytes, ShardId::SINGLE, &ledger, &state, None).unwrap();

        assert!(ledger.entries().contains_key(&(EntityId::new(1), type_id)));
        assert!(ledger.entries().contains_key(&(EntityId::new(2), type_id)));
        assert_eq!(state.applied_seqno(ShardId::SINGLE), Some(2));
    }

    /// CRITICAL (REPL): a community-overlay filter must not move the WAL
    /// high-water-mark. Applying a filtered segment must advance `applied_seqno`
    /// to the *same* boundary as applying the original (unfiltered) segment,
    /// even though the filtered batch carries fewer events. This is the
    /// load-bearing invariant for both per-segment idempotency and the
    /// `lag = leader_seqno - applied_seqno` gauge.
    #[test]
    fn filtered_segment_advances_to_same_boundary_as_unfiltered() {
        use crate::governance::{CommunityId, SignalScope};
        use crate::replication::shipper::filter_segment_drop_local;

        let schema = make_schema();

        let community = |entity: u64, ts: u64| {
            EventRecord::scoped(
                entity,
                crate::wal::format::batch::RECORD_TYPE_SIGNAL,
                1.0,
                ts,
                SignalScope::Community(CommunityId(1)).discriminant(),
                0,
                0,
                0,
            )
        };
        let local = |entity: u64, ts: u64| make_event(entity, 1, ts);

        // A single batch starting at first_seq=10 with 4 events (true last seq
        // = 13); the middle two are local and will be stripped by the filter.
        let events = vec![
            community(100, 1_000_000_000),
            local(101, 1_100_000_000),
            local(102, 1_200_000_000),
            community(103, 1_300_000_000),
        ];
        let original = encode_batch(&events, 10, 1).unwrap();
        let filtered = filter_segment_drop_local(&original);
        assert!(
            !filtered.is_empty(),
            "two community events survive the filter"
        );

        // Apply the unfiltered segment to its own state.
        let ledger_a = Arc::new(SignalLedger::new(schema.clone(), Box::new(NoopWalWriter)));
        let state_a = ReplicationState::new(&[ShardId::SINGLE]);
        apply_payload(&original, ShardId::SINGLE, &ledger_a, &state_a, None).unwrap();

        // Apply the filtered segment to a fresh state.
        let ledger_b = Arc::new(SignalLedger::new(schema, Box::new(NoopWalWriter)));
        let state_b = ReplicationState::new(&[ShardId::SINGLE]);
        apply_payload(&filtered, ShardId::SINGLE, &ledger_b, &state_b, None).unwrap();

        assert_eq!(
            state_a.applied_seqno(ShardId::SINGLE),
            Some(13),
            "unfiltered segment advances to its true last seq (10 + 4 - 1)"
        );
        assert_eq!(
            state_b.applied_seqno(ShardId::SINGLE),
            state_a.applied_seqno(ShardId::SINGLE),
            "filtered segment must advance to the SAME boundary as unfiltered"
        );
    }

    /// CRITICAL (REPL): the filtered-vs-unfiltered boundary equivalence must
    /// hold for arbitrary mixes of local/community events across multiple
    /// batches — not just the single hand-picked case above.
    ///
    /// Scope of the invariant: the per-batch re-stamp guarantees a surviving
    /// batch preserves *its own* last seq. A segment's overall boundary is the
    /// last seq of its last surviving batch, so the equivalence holds whenever
    /// the trailing batch survives the filter. We therefore force the final
    /// batch to carry a community event (its boundary must survive); the
    /// interior batches are arbitrary local/community mixes. (A trailing
    /// all-local batch legitimately produces no shippable bytes, so its seqno
    /// range cannot — and should not — be reconstructed by the receiver.)
    #[test]
    fn prop_filtered_segment_preserves_boundary() {
        use proptest::prelude::*;

        use crate::governance::{CommunityId, SignalScope};
        use crate::replication::shipper::filter_segment_drop_local;

        // Each event is (is_community, entity_id). Batches are non-empty.
        let event_strategy = (any::<bool>(), 1u64..1000);
        let batch_strategy = proptest::collection::vec(event_strategy, 1..6);
        let batches_strategy = proptest::collection::vec(batch_strategy, 1..4);

        let mut runner = proptest::test_runner::TestRunner::new(proptest::test_runner::Config {
            cases: 128,
            ..Default::default()
        });
        runner
            .run(&batches_strategy, |batches| {
                let schema = make_schema();
                let community = |entity: u64| {
                    EventRecord::scoped(
                        entity,
                        crate::wal::format::batch::RECORD_TYPE_SIGNAL,
                        1.0,
                        1_000_000_000,
                        SignalScope::Community(CommunityId(1)).discriminant(),
                        0,
                        0,
                        0,
                    )
                };

                // Lay out batches with contiguous WAL seqnos: batch i gets
                // first_seq = next, occupying `len` slots.
                let mut original = Vec::new();
                let mut next_seq = 1u64;
                let last_idx = batches.len() - 1;
                for (i, batch) in batches.iter().enumerate() {
                    let mut recs: Vec<EventRecord> = batch
                        .iter()
                        .map(|&(is_comm, entity)| {
                            if is_comm {
                                community(entity)
                            } else {
                                make_event(entity, 1, 1_000_000_000)
                            }
                        })
                        .collect();
                    // Force the LAST event of the LAST batch to be community so
                    // the segment's trailing boundary always survives the filter.
                    if i == last_idx {
                        let last = recs.len() - 1;
                        recs[last] = community(9999);
                    }
                    original.extend(encode_batch(&recs, next_seq, 1).unwrap());
                    next_seq += batch.len() as u64;
                }
                let true_last_seq = next_seq - 1;

                let filtered = filter_segment_drop_local(&original);

                let ledger_a = Arc::new(SignalLedger::new(schema.clone(), Box::new(NoopWalWriter)));
                let state_a = ReplicationState::new(&[ShardId::SINGLE]);
                apply_payload(&original, ShardId::SINGLE, &ledger_a, &state_a, None).unwrap();

                let ledger_b = Arc::new(SignalLedger::new(schema, Box::new(NoopWalWriter)));
                let state_b = ReplicationState::new(&[ShardId::SINGLE]);
                apply_payload(&filtered, ShardId::SINGLE, &ledger_b, &state_b, None).unwrap();

                prop_assert_eq!(state_a.applied_seqno(ShardId::SINGLE), Some(true_last_seq));
                prop_assert_eq!(
                    state_b.applied_seqno(ShardId::SINGLE),
                    state_a.applied_seqno(ShardId::SINGLE),
                    "filtered segment must reach the same boundary as unfiltered"
                );
                Ok(())
            })
            .unwrap();
    }

    /// WARNING (REPL): a corrupt header whose `first_seq + event_count` wraps
    /// u64 must be rejected as corruption, not silently wrapped into a bogus
    /// (small) high-water-mark.
    #[test]
    fn apply_payload_seq_overflow_is_corruption() {
        let schema = make_schema();
        let ledger = Arc::new(SignalLedger::new(schema, Box::new(NoopWalWriter)));
        let state = ReplicationState::new(&[ShardId::SINGLE]);

        let type_id = ledger.resolve_signal_type("view").unwrap();
        // Encode a structurally-valid 2-event batch with first_seq = u64::MAX.
        // The checksum is computed over these bytes, so decode succeeds and the
        // batch reaches the seq arithmetic, where first_seq + (2 - 1) wraps.
        let e1 = make_event(1, type_id.as_u16() as u8, 100);
        let e2 = make_event(2, type_id.as_u16() as u8, 200);
        let bytes = encode_batch(&[e1, e2], u64::MAX, 1).unwrap();

        let result = apply_payload(&bytes, ShardId::SINGLE, &ledger, &state, None);
        assert!(
            matches!(result, Err(WalError::Corruption { .. })),
            "first_seq=u64::MAX with 2 events must overflow → Corruption, got {result:?}"
        );
        assert_eq!(
            state.applied_seqno(ShardId::SINGLE),
            Some(0),
            "overflowing batch must not advance the high-water-mark"
        );
    }

    #[test]
    fn apply_payload_corrupt_returns_error() {
        let schema = make_schema();
        let ledger = Arc::new(SignalLedger::new(schema, Box::new(NoopWalWriter)));
        let state = Arc::new(ReplicationState::new(&[ShardId::SINGLE]));

        // Build a valid batch then corrupt the checksum region.
        let type_id = ledger.resolve_signal_type("view").unwrap();
        let events = vec![make_event(7, type_id.as_u16() as u8, 500)];
        let mut bytes = encode_batch(&events, 1, 1).unwrap();
        // Flip bytes in the checksum region (bytes 32-63) to break BLAKE3.
        for b in &mut bytes[32..64] {
            *b = b.wrapping_add(1);
        }

        let result = apply_payload(&bytes, ShardId::SINGLE, &ledger, &state, None);
        assert!(
            matches!(result, Err(WalError::Corruption { .. })),
            "expected Corruption, got {result:?}"
        );
        // Ledger must NOT have been updated.
        assert!(!ledger.entries().contains_key(&(EntityId::new(7), type_id)));
    }

    /// WARNING (REPL): a payload whose LATER batch is corrupt must apply
    /// **none** of its batches and must NOT advance the applied high-water-mark.
    /// The earlier (valid) batch must not be half-applied — the leader can then
    /// cleanly re-ship the same un-acked segment and replay it from one
    /// consistent resume point.
    #[test]
    fn apply_payload_atomic_on_mid_payload_corruption() {
        let schema = make_schema();
        let ledger = Arc::new(SignalLedger::new(schema, Box::new(NoopWalWriter)));
        let state = ReplicationState::new(&[ShardId::SINGLE]);
        let type_id = ledger.resolve_signal_type("view").unwrap();

        // First batch (seqs 1..=1): valid, entity 100. Second batch (seqs
        // 2..=2): valid then corrupted so decode fails AFTER the first batch.
        let good = vec![make_event(100, type_id.as_u16() as u8, 100)];
        let bad = vec![make_event(200, type_id.as_u16() as u8, 200)];
        let mut bytes = encode_batch(&good, 1, 100).unwrap();
        let good_len = bytes.len();
        bytes.extend(encode_batch(&bad, 2, 200).unwrap());
        // Corrupt the SECOND batch's checksum region (bytes 32..64 of it).
        for b in &mut bytes[good_len + 32..good_len + 64] {
            *b = b.wrapping_add(1);
        }

        let result = apply_payload(&bytes, ShardId::SINGLE, &ledger, &state, None);
        assert!(
            matches!(result, Err(WalError::Corruption { .. })),
            "mid-payload corruption must return Corruption, got {result:?}"
        );
        // NEITHER batch may be applied: the first (valid) batch must not have
        // been committed before the second batch failed to decode.
        assert!(
            !ledger
                .entries()
                .contains_key(&(EntityId::new(100), type_id)),
            "earlier valid batch must NOT be applied when a later batch is corrupt"
        );
        assert!(
            !ledger
                .entries()
                .contains_key(&(EntityId::new(200), type_id)),
            "corrupt batch must not be applied"
        );
        // The applied high-water-mark must NOT have advanced past 0.
        assert_eq!(
            state.applied_seqno(ShardId::SINGLE),
            Some(0),
            "half-applied payload must not advance the applied high-water-mark"
        );
    }

    /// WARNING (REPL): in `community_share_only` mode a segment whose events are
    /// all local filters to empty bytes that decode to ZERO batches. The
    /// receiver must still advance the lag gauge's leader high-water-mark from
    /// the segment's authoritative `leader_last_seq` (known before filtering),
    /// so the gauge does not under-report the leader's progress across the
    /// all-local segment.
    #[test]
    fn lag_gauge_tracks_leader_hwm_across_all_local_empty_segment() {
        use crate::replication::lag::ReplicationLagGauge;

        let schema = make_schema();
        let ledger = Arc::new(SignalLedger::new(schema, Box::new(NoopWalWriter)));
        let state = Arc::new(ReplicationState::new(&[ShardId::SINGLE]));
        let gauge = Arc::new(ReplicationLagGauge::new(
            ShardId::SINGLE,
            Arc::clone(&state),
        ));

        // An all-local segment filters to empty bytes: the leader's TRUE last
        // WAL seqno for it is 5 (e.g. a batch first_seq=4 with 2 events), but the
        // shipped bytes are empty.
        let empty_bytes: Vec<u8> = Vec::new();
        apply_payload_with_leader_seq(
            &empty_bytes,
            ShardId::SINGLE,
            &ledger,
            &state,
            Some(&gauge),
            Some(5),
        )
        .unwrap();

        assert_eq!(
            gauge.leader_seqno(),
            5,
            "leader HWM must advance from the authoritative leader_last_seq even \
             when the filtered segment decodes zero batches"
        );
        // No events were applied (empty segment), so the applied seqno stays 0.
        assert_eq!(state.applied_seqno(ShardId::SINGLE), Some(0));
        // The gauge therefore reports the leader being ahead of the follower.
        assert_eq!(
            gauge.lag_segments(),
            5,
            "leader at 5, applied 0 → 5 segments behind across the empty segment"
        );
    }

    /// A minimal transport that returns one payload then signals shutdown.
    struct OneShot {
        rx: crossbeam::channel::Receiver<crate::replication::WalSegmentPayload>,
    }

    impl Transport for OneShot {
        fn send_segment(
            &self,
            _to: ShardId,
            _payload: crate::replication::WalSegmentPayload,
        ) -> Result<(), crate::replication::TransportError> {
            Ok(())
        }
        fn recv_segment(&self) -> Option<crate::replication::WalSegmentPayload> {
            self.rx.recv().ok()
        }
        fn local_shard(&self) -> ShardId {
            ShardId(1)
        }
    }

    #[test]
    fn receiver_thread_exits_on_transport_close() {
        let (tx, rx) = crossbeam::channel::bounded(4);
        let transport = Arc::new(OneShot { rx });

        let schema = make_schema();
        let ledger = Arc::new(SignalLedger::new(schema, Box::new(NoopWalWriter)));
        let state = Arc::new(ReplicationState::new(&[ShardId(0)]));

        let handle = spawn_receiver(
            Arc::clone(&transport),
            Arc::clone(&ledger),
            Arc::clone(&state),
            None,
        );

        // Send one segment.
        let type_id = ledger.resolve_signal_type("view").unwrap();
        let events = vec![make_event(99, type_id.as_u16() as u8, 100)];
        let payload_bytes = encode_batch(&events, 1, 1).unwrap();
        tx.send(crate::replication::WalSegmentPayload {
            id: WalSegmentId::new(RegionId::SINGLE, ShardId(0), 1),
            bytes: payload_bytes,
            event_count: 1,
            leader_last_seq: 1,
        })
        .unwrap();

        // Give the receiver a moment to process.
        std::thread::sleep(Duration::from_millis(50));

        // Drop sender -- receiver's recv will return None.
        drop(tx);

        // The receiver should exit gracefully.
        handle.join().unwrap();

        // Verify the segment was applied.
        assert!(ledger.entries().contains_key(&(EntityId::new(99), type_id)));
        assert_eq!(state.applied_seqno(ShardId(0)), Some(1));
    }

    #[test]
    fn receiver_thread_exits_on_corrupt_payload() {
        let (tx, rx) = crossbeam::channel::bounded(4);
        let transport = Arc::new(OneShot { rx });

        let schema = make_schema();
        let ledger = Arc::new(SignalLedger::new(schema, Box::new(NoopWalWriter)));
        let state = Arc::new(ReplicationState::new(&[ShardId(0)]));

        let handle = spawn_receiver(
            Arc::clone(&transport),
            Arc::clone(&ledger),
            Arc::clone(&state),
            None,
        );

        // Build a valid batch then corrupt it.
        let type_id = ledger.resolve_signal_type("view").unwrap();
        let events = vec![make_event(55, type_id.as_u16() as u8, 100)];
        let mut corrupt_bytes = encode_batch(&events, 1, 1).unwrap();
        for b in &mut corrupt_bytes[32..64] {
            *b = b.wrapping_add(1);
        }

        tx.send(crate::replication::WalSegmentPayload {
            id: WalSegmentId::new(RegionId::SINGLE, ShardId(0), 1),
            bytes: corrupt_bytes,
            event_count: 1,
            leader_last_seq: 1,
        })
        .unwrap();

        // Give thread time to process.
        std::thread::sleep(Duration::from_millis(50));
        drop(tx);

        let result = handle.join();
        assert!(
            matches!(result, Err(WalError::Corruption { .. })),
            "expected Corruption from join, got {result:?}"
        );
        // Entity 55 must NOT have been applied.
        assert!(!ledger.entries().contains_key(&(EntityId::new(55), type_id)));
    }

    /// C11: a receiver that halts on a corrupt segment must flip its `died`
    /// liveness latch WITHOUT the owner ever calling `join()` — so `health_check`
    /// can observe a silently-stalled follower while the node is still open,
    /// instead of the death being visible only at shutdown.
    #[test]
    fn receiver_died_latch_set_on_corrupt_without_join() {
        let (tx, rx) = crossbeam::channel::bounded(4);
        let transport = Arc::new(OneShot { rx });

        let schema = make_schema();
        let ledger = Arc::new(SignalLedger::new(schema, Box::new(NoopWalWriter)));
        let state = Arc::new(ReplicationState::new(&[ShardId(0)]));

        let handle = spawn_receiver(
            Arc::clone(&transport),
            Arc::clone(&ledger),
            Arc::clone(&state),
            None,
        );

        // Healthy until something breaks.
        assert!(!handle.died(), "fresh receiver must not report died");

        // Feed one corrupt segment.
        let type_id = ledger.resolve_signal_type("view").unwrap();
        let events = vec![make_event(55, type_id.as_u16() as u8, 100)];
        let mut corrupt_bytes = encode_batch(&events, 1, 1).unwrap();
        for b in &mut corrupt_bytes[32..64] {
            *b = b.wrapping_add(1);
        }
        tx.send(crate::replication::WalSegmentPayload {
            id: WalSegmentId::new(RegionId::SINGLE, ShardId(0), 1),
            bytes: corrupt_bytes,
            event_count: 1,
            leader_last_seq: 1,
        })
        .unwrap();

        // Poll the latch (NOT join): the receiver must halt and set `died` while
        // the channel/transport stays open and the owner never calls join().
        let deadline = std::time::Instant::now() + Duration::from_secs(5);
        while !handle.died() && std::time::Instant::now() < deadline {
            std::thread::sleep(Duration::from_millis(5));
        }
        assert!(
            handle.died(),
            "a corrupt segment must flip the receiver's died latch without join()"
        );
        // The transport is still open (tx not dropped) — the death is observable
        // purely from the latch, exactly what health_check reads.
        drop(tx);
    }

    /// C11: a CLEAN shutdown (transport closed via `recv_segment()` == None) must
    /// NOT set the `died` latch — otherwise `health_check` would flag an
    /// intentional teardown as a fault.
    #[test]
    fn receiver_died_latch_unset_on_clean_shutdown() {
        let (tx, rx) = crossbeam::channel::bounded(4);
        let transport = Arc::new(OneShot { rx });

        let schema = make_schema();
        let ledger = Arc::new(SignalLedger::new(schema, Box::new(NoopWalWriter)));
        let state = Arc::new(ReplicationState::new(&[ShardId(0)]));

        let handle = spawn_receiver(
            Arc::clone(&transport),
            Arc::clone(&ledger),
            Arc::clone(&state),
            None,
        );

        // Clean shutdown: drop the sender so recv_segment() returns None.
        drop(tx);

        // The thread exits Ok; the latch must stay false.
        let deadline = std::time::Instant::now() + Duration::from_secs(5);
        while !handle.is_finished() && std::time::Instant::now() < deadline {
            std::thread::sleep(Duration::from_millis(5));
        }
        assert!(
            handle.is_finished(),
            "clean shutdown must let the thread exit"
        );
        assert!(
            !handle.died(),
            "a clean shutdown must NOT set the died latch (no false-positive fault)"
        );
        handle.join().unwrap();
    }

    /// obs-REPL-1: a follower that has received batches from the leader but not
    /// yet applied all of them reports a NON-ZERO replication lag; the lag
    /// returns to 0 once the follower catches up. This proves the receiver
    /// feeds the lag gauge's leader high-water-mark from the real apply path,
    /// not just from test-only `update_leader_seqno` calls.
    #[test]
    fn apply_payload_feeds_lag_gauge_until_caught_up() {
        use crate::replication::lag::ReplicationLagGauge;

        let schema = make_schema();
        let ledger = Arc::new(SignalLedger::new(schema, Box::new(NoopWalWriter)));
        let state = Arc::new(ReplicationState::new(&[ShardId::SINGLE]));
        let gauge = Arc::new(ReplicationLagGauge::new(
            ShardId::SINGLE,
            Arc::clone(&state),
        ));

        let type_id = ledger.resolve_signal_type("view").unwrap();

        // First segment carries WAL seqs 1..=2 (leader high-water = 2).
        let e1 = vec![make_event(1, type_id.as_u16() as u8, 100)];
        let e2 = vec![make_event(2, type_id.as_u16() as u8, 200)];
        let mut first = encode_batch(&e1, 1, 100).unwrap();
        first.extend(encode_batch(&e2, 2, 200).unwrap());

        // Apply the first segment: leader HWM 2, applied 2 → caught up.
        apply_payload(&first, ShardId::SINGLE, &ledger, &state, Some(&gauge)).unwrap();
        assert_eq!(gauge.leader_seqno(), 2);
        assert_eq!(gauge.applied_seqno(), 2);
        assert_eq!(gauge.lag_segments(), 0, "fully applied → no lag");

        // Simulate received-but-not-yet-applied backpressure: the leader has
        // shipped up through seq 5, but the follower has only applied 2.
        // Recording the leader HWM ahead of application is exactly what the
        // receiver does for each decoded batch before the apply step.
        gauge.update_leader_seqno(5);
        assert_eq!(
            gauge.lag_segments(),
            3,
            "leader at 5, applied 2 → 3 segments behind"
        );

        // Deliver the missing segment (WAL seqs 3..=5) and apply it: caught up.
        let e3 = vec![make_event(3, type_id.as_u16() as u8, 300)];
        let e4 = vec![make_event(4, type_id.as_u16() as u8, 400)];
        let e5 = vec![make_event(5, type_id.as_u16() as u8, 500)];
        let mut catchup = encode_batch(&e3, 3, 300).unwrap();
        catchup.extend(encode_batch(&e4, 4, 400).unwrap());
        catchup.extend(encode_batch(&e5, 5, 500).unwrap());

        apply_payload(&catchup, ShardId::SINGLE, &ledger, &state, Some(&gauge)).unwrap();
        assert_eq!(gauge.leader_seqno(), 5);
        assert_eq!(gauge.applied_seqno(), 5);
        assert_eq!(gauge.lag_segments(), 0, "caught up → lag back to 0");
    }
}
