//! Simulated multi-region cluster for M8 UAT testing.
//!
//! Creates a set of ephemeral [`TidalDb`] instances wired with the real M8
//! distributed fabric. The transport layer is pluggable: by default it uses
//! in-process crossbeam channels ([`ChannelTransport`]), but callers can
//! provide external transports (e.g., [`GrpcTransport`] from `tidal-net`).
//!
//! Signal replication traverses the full path:
//!
//! ```text
//! write_signal → encode_batch → transport.send_segment()
//!                               ↓
//!                        spawn_receiver thread
//!                               ↓
//!                         apply_payload
//!                               ↓
//!                    SignalLedger::apply_wal_event
//!                    ReplicationState::advance
//! ```
//!
//! # Architecture
//!
//! * All nodes open with `NodeRole::Single` so they accept direct writes AND
//!   can be promoted to leader at any time.
//! * Every non-initial-leader region starts a `spawn_receiver` thread (via
//!   `db.start_replication(transport)`) that processes incoming WAL batches.
//! * `write_signal` encodes the event as a one-event WAL batch and ships it
//!   immediately to all non-partitioned followers via their transport.
//! * A `batch_log` records every shipped batch so `await_convergence` can
//!   re-deliver missed batches after a partition is healed.
//! * `await_convergence` ships any pending batches, then polls
//!   `ReplicationState::applied_seqno` until all active followers have caught
//!   up to the current leader's sequence number.
//!
//! # Module size (`CODING_GUIDELINES` §9)
//!
//! This file is a single cohesive concern: the `SimulatedCluster` test harness
//! and its lifecycle (`build` → `write_signal`/`await_convergence`/fault
//! injection → `Drop`-time receiver-thread join). The reusable transport types
//! (`ChannelTransport`, `RecvOnlyChannelTransport`, `BatchEntry`,
//! `redeliver_missed`) already live in the sibling `cluster_transport` module.
//! A further split would fragment the harness's public API across files and risk
//! breaking the carefully-ordered `Drop` join logic, so it is intentionally kept
//! whole. Revisit only if a genuinely separable sub-area (e.g. partition/fault
//! orchestration) grows large enough to stand alone.

use std::{
    collections::{HashMap, HashSet},
    sync::{
        Arc, Mutex, RwLock,
        atomic::{AtomicU64, Ordering},
    },
    time::{Duration, Instant},
};

use super::cluster_transport::{
    BatchEntry, ChannelTransport, RecvOnlyChannelTransport, redeliver_missed, single_event_payload,
};
use crate::{
    db::{
        TidalDb,
        config::{NodeConfig, NodeRole},
    },
    query::{retrieve::Retrieve, search::Search},
    replication::{
        SegmentReceiverHandle,
        shard::{RegionId, ShardId},
        spawn_receiver,
        transport::{Transport, WalSegmentPayload},
    },
    schema::{EntityId, Schema, Timestamp},
    signals::{NoopWalWriter, SignalLedger},
    wal::format::batch::{EventRecord, encode_batch},
};

// ── Public API ────────────────────────────────────────────────────────────

/// A simulated node in the cluster.
pub struct SimulatedNode {
    /// The region this node belongs to.
    pub region_id: RegionId,
    /// The `TidalDb` instance for this node.
    pub db: TidalDb,
}

/// Configuration for building a [`SimulatedCluster`].
pub struct ClusterConfig {
    /// Region IDs to create.
    pub regions: Vec<RegionId>,
    /// Which region hosts the leader node.
    pub leader_region: RegionId,
    /// Schema shared by all nodes.
    pub schema: Schema,
    /// Ranking profiles shared by all nodes (optional, defaults to builtins only).
    pub profiles: Vec<crate::ranking::profile::RankingProfile>,
    /// External transports keyed by follower region ID.
    ///
    /// When `None` (the default), built-in crossbeam `ChannelTransport` is used
    /// for each follower. When `Some`, the provided transports are used instead,
    /// enabling real network transport (e.g., `GrpcTransport` from `tidal-net`).
    pub transports: Option<HashMap<RegionId, Arc<dyn Transport>>>,
}

/// A simulated multi-region tidalDB cluster using the real M8 distributed
/// fabric.
///
/// Signal replication traverses the real WAL-batch encode → transport →
/// `apply_payload` → `ReplicationState::advance` pipeline instead of calling
/// `db.signal()` directly on followers.
///
/// The transport layer is pluggable via [`ClusterConfig::transports`].
pub struct SimulatedCluster {
    /// Current leader region ID. Mutable via `promote_leader`.
    leader_region: Mutex<RegionId>,
    /// All nodes indexed by region.
    nodes: HashMap<RegionId, SimulatedNode>,
    /// Per-follower transports (region → transport for WAL segment delivery).
    ///
    /// Only regions that have an active `spawn_receiver` thread appear here.
    /// This is the **send** side: `write_signal` / `redeliver_missed` ship
    /// through it. Dropping the cluster drops these senders, which closes each
    /// receiver thread's channel so it exits and its handle can be joined.
    follower_transports: HashMap<RegionId, Arc<dyn Transport>>,
    /// Join handles for the per-follower `spawn_receiver` threads.
    ///
    /// Held so the threads are *observable* rather than detached: [`Drop`] joins
    /// each one (after dropping the send-side transports that unblock them) and
    /// logs any corruption error surfaced via
    /// [`SegmentReceiverHandle::join`](crate::replication::SegmentReceiverHandle::join),
    /// instead of silently leaking the thread.
    receiver_handles: Mutex<Vec<SegmentReceiverHandle>>,
    /// `true` when followers use externally-provided (e.g. gRPC) transports.
    ///
    /// The in-process channel path is split (send-only cluster side, recv-only
    /// thread side) so dropping the cluster deterministically unblocks and joins
    /// every receiver thread. External transports share one bidirectional `Arc`
    /// and manage their own shutdown out-of-band, so blindly joining their
    /// threads in `Drop` could block; for those we observe-by-logging without a
    /// blocking join.
    external_transports: bool,
    /// All batches ever shipped, for partition-recovery re-delivery.
    batch_log: Mutex<Vec<BatchEntry>>,
    /// Per-leader signal count used as the WAL seqno for that leader's batches.
    leader_seqnos: Mutex<HashMap<RegionId, u64>>,
    /// Total signals ever written to any leader (for [`relay_log_len`]).
    total_signals: AtomicU64,
    /// Regions currently isolated by a network partition.
    partitioned_regions: Arc<RwLock<HashSet<RegionId>>>,
    /// Schema kept for reference.
    #[allow(dead_code)]
    schema: Schema,
    /// Pre-computed map: signal type name → `u8` ID used in [`EventRecord`].
    signal_type_ids: HashMap<String, u8>,
}

impl SimulatedCluster {
    /// Build a cluster from the given configuration.
    ///
    /// All nodes are created immediately in ephemeral mode. Non-leader regions
    /// have a `spawn_receiver` thread started automatically.
    ///
    /// When `config.transports` is `None`, built-in crossbeam channels are used
    /// (fast, in-process). When `Some`, the provided transports are used for
    /// segment delivery, enabling real network transport like `GrpcTransport`.
    ///
    /// # Panics
    ///
    /// Panics if any `TidalDb` fails to open or if `start_replication` fails.
    #[must_use]
    pub fn build(config: ClusterConfig) -> Self {
        // Pre-compute signal type IDs via a temporary ledger.
        //
        // The WAL `EventRecord::signal_type` field is a `u8`, so a signal type
        // whose resolved id exceeds 255 cannot be represented on the wire. A
        // blind `as u8` truncation would silently alias it onto another type
        // (corrupting replicated signals). Use a checked conversion instead: an
        // out-of-range id is dropped from the map, so `write_signal` returns a
        // clean "unknown signal type" error rather than shipping a wrong-typed
        // event. We also log it so the test author sees why the type is missing.
        let scratch_ledger = SignalLedger::new(config.schema.clone(), Box::new(NoopWalWriter));
        let signal_type_ids: HashMap<String, u8> = config
            .schema
            .signals()
            .filter_map(|def| {
                let id = scratch_ledger.resolve_signal_type(def.name()).ok()?;
                if let Ok(byte) = u8::try_from(id.as_u16()) {
                    Some((def.name().to_string(), byte))
                } else {
                    tracing::warn!(
                        signal = def.name(),
                        type_id = id.as_u16(),
                        "SimulatedCluster: signal type id exceeds u8 WAL field; \
                         excluded from the replication write path"
                    );
                    None
                }
            })
            .collect();

        // Derive all shard IDs (one per region).
        let all_shards: Vec<ShardId> = config.regions.iter().map(|r| ShardId(r.0)).collect();

        let mut nodes: HashMap<RegionId, SimulatedNode> = HashMap::new();
        let mut follower_transports: HashMap<RegionId, Arc<dyn Transport>> = HashMap::new();
        let mut receiver_handles: Vec<SegmentReceiverHandle> = Vec::new();
        let mut leader_seqnos: HashMap<RegionId, u64> = HashMap::new();

        for &region in &config.regions {
            let shard_id = ShardId(region.0);
            let peer_shards: Vec<ShardId> = all_shards
                .iter()
                .copied()
                .filter(|&s| s != shard_id)
                .collect();

            let db = TidalDb::builder()
                .ephemeral()
                .with_schema(config.schema.clone())
                .with_profiles(config.profiles.clone())
                .with_cluster(NodeConfig {
                    role: NodeRole::Single,
                    shard_id,
                    peer_shards,
                    ..NodeConfig::default()
                })
                .open()
                .expect("ephemeral TidalDb with valid schema must open");

            // Wire a receiver for every non-leader region.
            if region != config.leader_region {
                let ledger = db
                    .ledger()
                    .expect("ephemeral db with schema must have ledger")
                    .clone();
                let rep_state = db.replication_state().clone();
                // Feed this node's lag gauge so per-node replication lag is real
                // (obs-REPL-1). The gauge shares the node's replication_state.
                let lag_gauge = Some(db.control_plane().lag_gauge());

                // `send_transport` is the cluster's ship side (stored in
                // `follower_transports`); `recv_transport` is handed to the
                // receiver thread. For the in-process path we deliberately split
                // the channel so the thread does NOT hold a sender — otherwise
                // its own `Arc` would keep the channel open and the join in
                // `Drop` would block forever.
                let (send_transport, recv_transport): (Arc<dyn Transport>, Arc<dyn Transport>) =
                    if let Some(ref ext) = config.transports {
                        // External transport (e.g. GrpcTransport) is bidirectional
                        // and manages its own shutdown; share one Arc for both.
                        let t = ext
                            .get(&region)
                            .unwrap_or_else(|| {
                                panic!("no external transport for follower region {region}")
                            })
                            .clone();
                        (t.clone(), t)
                    } else {
                        // Default: built-in crossbeam channel, split into a
                        // send-only (cluster) and recv-only (thread) endpoint.
                        let (tx, rx) = crossbeam::channel::bounded::<WalSegmentPayload>(1024);
                        let send: Arc<dyn Transport> = Arc::new(ChannelTransport {
                            local_shard: shard_id,
                            tx,
                            rx: rx.clone(),
                        });
                        let recv: Arc<dyn Transport> = Arc::new(RecvOnlyChannelTransport {
                            local_shard: shard_id,
                            rx,
                        });
                        (send, recv)
                    };

                let handle = spawn_receiver(recv_transport, ledger, rep_state, lag_gauge);
                receiver_handles.push(handle);
                follower_transports.insert(region, send_transport);
            }

            nodes.insert(
                region,
                SimulatedNode {
                    region_id: region,
                    db,
                },
            );
            leader_seqnos.insert(region, 0);
        }

        Self {
            leader_region: Mutex::new(config.leader_region),
            nodes,
            follower_transports,
            receiver_handles: Mutex::new(receiver_handles),
            external_transports: config.transports.is_some(),
            batch_log: Mutex::new(Vec::new()),
            leader_seqnos: Mutex::new(leader_seqnos),
            total_signals: AtomicU64::new(0),
            partitioned_regions: Arc::new(RwLock::new(HashSet::new())),
            schema: config.schema,
            signal_type_ids,
        }
    }

    /// The current leader region ID.
    #[must_use]
    pub fn leader_region(&self) -> RegionId {
        *self
            .leader_region
            .lock()
            .unwrap_or_else(std::sync::PoisonError::into_inner)
    }

    /// Reference to the leader node.
    ///
    /// # Panics
    ///
    /// Panics if the leader region has no node.
    #[must_use]
    pub fn leader(&self) -> &SimulatedNode {
        let region = self.leader_region();
        self.nodes
            .get(&region)
            .expect("leader region must have a node")
    }

    /// Reference to a node by region.
    ///
    /// # Panics
    ///
    /// Panics if no node exists for the region.
    #[must_use]
    pub fn node(&self, region: RegionId) -> &SimulatedNode {
        self.nodes
            .get(&region)
            .unwrap_or_else(|| panic!("no node for region {region}"))
    }

    /// Write a signal to the cluster leader and ship it to active followers.
    ///
    /// The signal is immediately applied to the leader's `TidalDb`. A one-event
    /// WAL batch is encoded and shipped via each follower's transport to all
    /// non-partitioned followers with active receivers. The batch is also
    /// recorded in the `batch_log` for partition-recovery re-delivery.
    ///
    /// # Errors
    ///
    /// Returns `TidalError` if the signal type is not registered in the schema
    /// or if the leader write fails.
    // The `leader_seqnos` guard is deliberately held across the whole
    // seqno-bump + encode + leader-write region so no other writer observes an
    // in-flight seqno before a rollback (atomicity); clippy's drop-tightening
    // suggestion would break that invariant.
    #[allow(clippy::significant_drop_tightening)]
    pub fn write_signal(
        &self,
        signal_type: &str,
        entity_id: EntityId,
        weight: f64,
    ) -> crate::Result<()> {
        let ts = Timestamp::now();
        let leader_region = self.leader_region();
        let leader_shard = ShardId(leader_region.0);

        // Resolve the type id (fallible, non-mutating) BEFORE touching the
        // leader, so an unknown signal type returns `Err` via `?` with no state
        // change.
        let type_id = *self.signal_type_ids.get(signal_type).ok_or_else(|| {
            crate::TidalError::invalid_input(format!("unknown signal type '{signal_type}'"))
        })?;

        let events = [EventRecord::signal(
            entity_id.as_u64(),
            type_id,
            weight as f32,
            ts.as_nanos(),
        )];

        // Commit the seqno, encode the batch, and apply the leader write while
        // holding the `leader_seqnos` lock so the three are atomic with respect
        // to one another and to concurrent writers. The seqno is bumped first
        // (so it is unique under concurrency) but rolled back if either the
        // encode (a future event-count/size limit) or the leader write fails —
        // so a failure never leaves the leader ahead of an un-shipped batch, and
        // never burns a seqno that no follower could reach (which would hang
        // `await_convergence`). Because the whole fallible region is inside the
        // lock, no other writer can observe the in-flight seqno before rollback.
        let (seqno, bytes) = {
            let mut seqnos = self
                .leader_seqnos
                .lock()
                .unwrap_or_else(std::sync::PoisonError::into_inner);
            let slot = seqnos.entry(leader_region).or_insert(0);
            *slot += 1;
            let seqno = *slot;

            // Encode; on failure roll back the unused seqno before returning.
            let bytes = match encode_batch(&events, seqno, ts.as_nanos()) {
                Ok(bytes) => bytes,
                Err(e) => {
                    *slot -= 1;
                    return Err(crate::TidalError::internal("write_signal", e.to_string()));
                }
            };

            // Apply the leader write; on failure roll back the unused seqno.
            // `slot` is still the live borrow, so no re-entry into `seqnos`.
            if let Err(e) = self.nodes[&leader_region]
                .db
                .signal(signal_type, entity_id, weight, ts)
            {
                *slot -= 1;
                return Err(e);
            }

            (seqno, bytes)
        };

        // Ship immediately to non-partitioned followers via their transport.
        let partitioned = self
            .partitioned_regions
            .read()
            .unwrap_or_else(std::sync::PoisonError::into_inner)
            .clone();

        for (&region, transport) in &self.follower_transports {
            if region == leader_region || partitioned.contains(&region) {
                continue;
            }
            // Share the payload construction with `redeliver_missed` so the eager
            // ship path and the recovery path can never drift on the load-bearing
            // `event_count`/`leader_last_seq` invariants (the receiver's monotonic
            // `advance` relies on `leader_last_seq == seqno`).
            let payload = single_event_payload(leader_shard, seqno, bytes.clone());
            // The eager ship is BEST-EFFORT: the leader write is already durable
            // and recorded in `batch_log`, so a failed ship is recovered by
            // `await_convergence` / `heal_region` re-delivery. We do NOT fail the
            // write on a ship error (that would be the wrong contract — the data
            // is durably on the leader). But a swallowed ship error is invisible
            // to an operator, so surface it as a WARN: a follower that keeps
            // failing to receive is observable here (and reconciled later) rather
            // than silently lagging. See `redeliver_missed`.
            if let Err(e) = transport.send_segment(ShardId(region.0), payload) {
                tracing::warn!(
                    region = region.0,
                    leader_shard = leader_shard.0,
                    seqno,
                    error = %e,
                    "eager follower ship failed; leader write is durable and queued for \
                     re-delivery (await_convergence / heal_region)"
                );
            }
        }

        // Record in batch_log for partition-recovery re-delivery.
        self.batch_log
            .lock()
            .unwrap_or_else(std::sync::PoisonError::into_inner)
            .push(BatchEntry {
                source_shard: leader_shard,
                seqno,
                bytes,
            });

        self.total_signals.fetch_add(1, Ordering::Relaxed);
        Ok(())
    }

    /// Write a signal directly to a specific region (bypassing the leader).
    ///
    /// Used to simulate partitioned writes during split-brain scenarios.
    ///
    /// # Panics
    ///
    /// Panics if the signal write fails.
    pub fn write_signal_to_region(
        &self,
        region: RegionId,
        signal_type: &str,
        entity_id: EntityId,
        weight: f64,
    ) {
        let ts = Timestamp::now();
        self.nodes
            .get(&region)
            .unwrap_or_else(|| panic!("no node for region {region}"))
            .db
            .signal(signal_type, entity_id, weight, ts)
            .expect("signal write must succeed");
    }

    /// Wait for all non-partitioned followers to apply all pending WAL batches.
    ///
    /// * Ships any batches that were missed during a partition (re-delivery from
    ///   the `batch_log`).
    /// * Polls `ReplicationState::applied_seqno` for each active follower until
    ///   it reaches the current leader's latest seqno.
    ///
    /// # Panics
    ///
    /// Panics if convergence is not reached within `timeout`.
    pub fn await_convergence(&self, timeout: Duration) {
        let deadline = Instant::now() + timeout;

        loop {
            assert!(
                Instant::now() <= deadline,
                "convergence timeout: cluster did not converge within {timeout:?}"
            );

            let leader_region = self.leader_region();
            let leader_shard = ShardId(leader_region.0);

            let target_seqno = self
                .leader_seqnos
                .lock()
                .unwrap_or_else(std::sync::PoisonError::into_inner)
                .get(&leader_region)
                .copied()
                .unwrap_or(0);

            let partitioned = self
                .partitioned_regions
                .read()
                .unwrap_or_else(std::sync::PoisonError::into_inner)
                .clone();

            // Re-deliver any batches missed during a partition.
            //
            // Snapshot the batch log under the lock, then release it BEFORE the
            // (blocking) `redeliver_missed` calls. `redeliver_missed` ships
            // through a transport whose `send_segment` can block on a bounded
            // channel; holding `batch_log` across that block would stall every
            // concurrent `write_signal` (which must take the same lock to record
            // its batch), turning a slow follower into a cluster-wide write
            // stall. The snapshot is a one-shot clone per poll, off the hot
            // write path.
            let log_snapshot: Vec<BatchEntry> = {
                let log = self
                    .batch_log
                    .lock()
                    .unwrap_or_else(std::sync::PoisonError::into_inner);
                log.iter()
                    .map(|e| BatchEntry {
                        source_shard: e.source_shard,
                        seqno: e.seqno,
                        bytes: e.bytes.clone(),
                    })
                    .collect()
            };
            for (&rgn, transport) in &self.follower_transports {
                if rgn == leader_region || partitioned.contains(&rgn) {
                    continue;
                }
                redeliver_missed(transport.as_ref(), &self.nodes[&rgn].db, &log_snapshot);
            }

            // Check whether all active non-partitioned followers have converged.
            let mut all_converged = true;
            for (&region, node) in &self.nodes {
                if region == leader_region || partitioned.contains(&region) {
                    continue;
                }
                // Only check regions that have an active receiver thread.
                if !self.follower_transports.contains_key(&region) {
                    continue;
                }
                let applied = node
                    .db
                    .replication_state()
                    .applied_seqno(leader_shard)
                    .unwrap_or(0);
                if applied < target_seqno {
                    all_converged = false;
                    break;
                }
            }

            if all_converged {
                return;
            }
            std::thread::sleep(Duration::from_millis(1));
        }
    }

    /// Read decay score from a specific region's node.
    #[must_use]
    pub fn read_decay_score(
        &self,
        region: RegionId,
        entity_id: EntityId,
        signal_type: &str,
    ) -> Option<f64> {
        self.nodes.get(&region).and_then(|n| {
            n.db.read_decay_score(entity_id, signal_type, 0)
                .ok()
                .flatten()
        })
    }

    /// Broadcast item metadata to all nodes.
    pub fn write_item_with_metadata(
        &self,
        entity_id: EntityId,
        metadata: &HashMap<String, String>,
    ) -> crate::Result<()> {
        for node in self.nodes.values() {
            node.db.write_item_with_metadata(entity_id, metadata)?;
        }
        Ok(())
    }

    /// Broadcast embedding writes to all nodes.
    pub fn write_item_embedding(
        &self,
        entity_id: EntityId,
        embedding: &[f32],
    ) -> crate::Result<()> {
        for node in self.nodes.values() {
            node.db.write_item_embedding(entity_id, embedding)?;
        }
        Ok(())
    }

    /// Run a RETRIEVE query against a region.
    pub fn retrieve(
        &self,
        region: RegionId,
        query: &Retrieve,
    ) -> crate::Result<crate::query::retrieve::Results> {
        self.node(region).db.retrieve(query)
    }

    /// Run a SEARCH query against a region.
    pub fn search(
        &self,
        region: RegionId,
        query: &Search,
    ) -> crate::Result<crate::query::search::SearchResults> {
        self.node(region).db.search(query)
    }

    /// Item count helper for health checks.
    pub fn item_count(&self, region: RegionId) -> u64 {
        self.node(region).db.item_count()
    }

    /// Current leader seqno.
    pub fn leader_seqno(&self) -> u64 {
        let leader = self.leader_region();
        self.leader_seqnos
            .lock()
            .unwrap_or_else(std::sync::PoisonError::into_inner)
            .get(&leader)
            .copied()
            .unwrap_or(0)
    }

    /// Mark a region as partitioned.
    pub fn partition_region(&self, region: RegionId) {
        let mut partitions = self
            .partitioned_regions
            .write()
            .unwrap_or_else(std::sync::PoisonError::into_inner);
        partitions.insert(region);
    }

    /// Heal a partitioned region and re-deliver any missed WAL batches.
    pub fn heal_region(&self, region: RegionId) {
        {
            let mut partitions = self
                .partitioned_regions
                .write()
                .unwrap_or_else(std::sync::PoisonError::into_inner);
            partitions.remove(&region);
        }
        if let Some(transport) = self.follower_transports.get(&region) {
            let log = self
                .batch_log
                .lock()
                .unwrap_or_else(std::sync::PoisonError::into_inner);
            redeliver_missed(transport.as_ref(), &self.nodes[&region].db, &log);
        }
    }

    /// Whether a region is currently partitioned.
    pub fn is_partitioned(&self, region: RegionId) -> bool {
        self.partitioned_regions
            .read()
            .unwrap_or_else(std::sync::PoisonError::into_inner)
            .contains(&region)
    }

    /// Total number of signals ever written to any leader (replaces relay log
    /// length in the old signal-replay implementation).
    ///
    /// `total_signals` is a free-standing monotonic counter — it does not gate
    /// access to any other memory — so it is incremented and read with
    /// `Relaxed`. The previous `Acquire` load here was mismatched: the
    /// `fetch_add` store side uses `Relaxed`, so there is no paired `Release` for
    /// the `Acquire` to synchronize with, making the stronger ordering pure
    /// overhead with no happens-before guarantee to back it.
    #[must_use]
    pub fn relay_log_len(&self) -> u64 {
        self.total_signals.load(Ordering::Relaxed)
    }

    /// Number of WAL batches applied from the **initial** leader (`ShardId(0)`)
    /// on a specific region's replication state.
    ///
    /// This is equivalent to the event count for all tests that do not involve
    /// leader promotion, since each signal produces exactly one WAL batch.
    ///
    /// # Limitation (post-promotion)
    ///
    /// The queried shard is hardcoded to `ShardId(0)` — the *initial* leader —
    /// not the current one. `applied_seqno` is per source shard, so after a
    /// [`promote_leader`](Self::promote_leader) call the new leader ships under
    /// its own `ShardId`, and this method keeps returning only the count from the
    /// old leader's stream — i.e. it silently undercounts. Callers in a promotion
    /// scenario must instead read
    /// [`replication_state().applied_seqno(current_leader_shard)`] directly so the
    /// shard matches the live leader. Bounding the hardcode behind a doc note here
    /// rather than reading the live leader shard keeps the existing no-promotion
    /// callers' counts stable.
    #[must_use]
    pub fn applied_count(&self, region: RegionId) -> u64 {
        self.nodes.get(&region).map_or(0, |n| {
            n.db.replication_state()
                .applied_seqno(ShardId(0))
                .unwrap_or(0)
        })
    }

    /// Access the shared partitioned-regions set (for fault injection via
    /// [`crate::testing::faults::NetworkPartition`]).
    #[must_use]
    pub const fn partitioned_regions(&self) -> &Arc<RwLock<HashSet<RegionId>>> {
        &self.partitioned_regions
    }

    /// Promote a follower to leader.
    ///
    /// After promotion `write_signal` routes writes to the new leader and ships
    /// batches to all other regions that have active receivers. The new leader
    /// must already exist as a node in the cluster.
    ///
    /// # Panics
    ///
    /// Panics if `new_leader` does not correspond to an existing node.
    pub fn promote_leader(&self, new_leader: RegionId) {
        assert!(
            self.nodes.contains_key(&new_leader),
            "cannot promote non-existent region {new_leader}"
        );
        let mut leader = self
            .leader_region
            .lock()
            .unwrap_or_else(std::sync::PoisonError::into_inner);
        *leader = new_leader;
    }

    /// All region IDs in the cluster.
    #[must_use]
    pub fn regions(&self) -> Vec<RegionId> {
        self.nodes.keys().copied().collect()
    }
}

impl Drop for SimulatedCluster {
    /// Joins every `spawn_receiver` thread so none are silently leaked.
    ///
    /// Order matters: we first drop the send-side transports so each in-process
    /// channel closes (the receiver thread's recv-only endpoint then returns
    /// `None` and the thread exits), then join each handle. Any corruption error
    /// a receiver surfaced is logged rather than swallowed — a leaked, panicked,
    /// or stalled replica is now observable in test output.
    fn drop(&mut self) {
        // Drop the send side first so the in-process receiver threads unblock.
        self.follower_transports.clear();

        let handles = std::mem::take(
            &mut *self
                .receiver_handles
                .lock()
                .unwrap_or_else(std::sync::PoisonError::into_inner),
        );

        if self.external_transports {
            // External transports share one bidirectional Arc and shut down
            // out-of-band; the receiver thread still holds a recv-capable Arc,
            // so a blocking join here could hang. Drop the handles (detaching
            // the threads, which exit when the external transport closes) and
            // record that we did so rather than silently leaking them.
            tracing::debug!(
                count = handles.len(),
                "SimulatedCluster: detaching external-transport receiver threads on drop"
            );
            drop(handles);
            return;
        }

        // In-process path: the channels are now closed, so each thread is
        // exiting. Join every one and surface any corruption it halted on.
        for handle in handles {
            match handle.join() {
                Ok(()) => {}
                Err(e) => {
                    tracing::warn!(
                        error = %e,
                        "SimulatedCluster: receiver thread halted on corruption"
                    );
                }
            }
        }
    }
}
