tidaldb/tidal-server/src/cluster/reseed.rs

//! Boot-time snapshot install + swap-recovery (m11p5 §2.2–§2.7).
//!
//! When a node's data dir carries a durable `reseed_required` marker, it does
//! NOT open the existing engine and serve degraded forever — instead, BEFORE
//! constructing [`ShardReplica`], it:
//!
//! 1. **Recovers any interrupted prior swap** (§2.3) — this runs FIRST, before
//!    `ElectionStore` classification, because classifying first re-opens restart
//!    amnesia (a mid-rename crash would otherwise classify `Fresh` at term 0 and
//!    re-vote in an already-voted term).
//! 2. **Discovers the current leader** (§2.7) by polling
//!    `GET /cluster/status/local` across the topology's HTTP addresses (plus the
//!    durable membership cache when present), with bounded backoff.
//! 3. **Adopts the discovered term** durably (persist-before-act) when it is
//!    above the local durable term.
//! 4. **Fetches a snapshot** from the leader's advertised gRPC address into a
//!    sibling staging dir, verifies every file's BLAKE3 + manifest completeness,
//!    copies the node's own identity files INTO staging, writes a `COMPLETE`
//!    sentinel + a `reseed-install-pending` install sentinel, fsyncs, then
//!    performs the §2.3 swap (rename old aside → rename staging in → fsync parent
//!    → delete discard).
//!
//! A marker boot that cannot complete the handshake within a bounded window
//! **falls back to opening the existing data dir** — degraded, marker latched,
//! voting enabled (§2.2; the reseed needs a leader and the leader may need this
//! node's vote, so blocking boot forever would deadlock the cluster). The
//! post-open seed (§2.6) is driven by the install sentinel inside
//! [`ShardReplica::new`], not here.
//!
//! # The swap protocol (§2.3, review-corrected ordering)
//!
//! ```text
//! staging = <parent>/<name>.reseed-staging   (same filesystem as data_dir)
//! discard = <parent>/<name>.discard
//!
//! fetch → staging
//! verify BLAKE3 + completeness
//! copy own election_state + membership cache INTO staging   (§2.3 step 2)
//! write reseed-install-pending sentinel INTO staging        (crash-idempotent §2.6 seed)
//! write COMPLETE sentinel INTO staging; fsync               (§2.3 step 3)
//! rename(data_dir → discard)                                (§2.3 step 4)
//! rename(staging → data_dir)
//! fsync(parent)
//! delete discard
//! ```
//!
//! The reseed marker lives in the OLD dir, which becomes `.discard` and is
//! deleted; the new dir (from staging, which the leader's manifest never
//! includes a marker in) simply has none. Nothing recreates it on the success
//! path.

use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};

use tidaldb::replication::{
    ElectionStore, HardState, ReseedMarker, ReseedMarkerStore, ReseedReason, ShardId,
};

use super::topology::{GrpcTlsSpec, TopologySpec};
use crate::error::{Result, ServerError};

/// The `COMPLETE` sentinel inside staging (§2.3 step 3): its presence means the
/// staged dir is fully fetched + verified + identity-copied and the swap may be
/// (re)done. Without it, a staging dir is an interrupted fetch and is deleted.
const COMPLETE_SENTINEL: &str = "COMPLETE";

/// The install sentinel copied INTO staging BEFORE the swap (§2.6).
///
/// Its presence in the canonical data dir AFTER open tells
/// `ShardReplica::new` to run the post-open seed exactly once
/// (crash-idempotent: a crash before the seed's sentinel-delete re-runs the same
/// seed against the unchanged WAL).
pub const INSTALL_PENDING_SENTINEL: &str = "reseed-install-pending";

/// Staging dir suffix (a sibling of `data_dir` in its parent, §2.3).
const STAGING_SUFFIX: &str = "reseed-staging";

/// Discard dir suffix (the old `data_dir`, renamed aside during the swap).
const DISCARD_SUFFIX: &str = "discard";

/// Default handshake window before a marker boot falls back to opening the
/// existing data dir (§2.2). Env-overridable for tests via
/// `TIDAL_RESEED_HANDSHAKE_MS`.
const DEFAULT_HANDSHAKE_MS: u64 = 60_000;

/// Per-status-poll HTTP timeout (short — the discovery loop iterates).
const STATUS_POLL_TIMEOUT: Duration = Duration::from_millis(500);

/// Backoff bounds for the discovery loop.
const BACKOFF_MIN: Duration = Duration::from_millis(250);
const BACKOFF_MAX: Duration = Duration::from_secs(5);

/// The set of swap-related paths derived from a data dir.
///
/// All three (staging, discard, and `data_dir` itself) live in `data_dir`'s
/// PARENT on the SAME filesystem, so the swap renames are atomic (`EXDEV` /
/// mount-root failures surface loudly at reseed time, never silently — §2.3).
#[derive(Debug, Clone)]
pub struct SwapDirs {
    /// The canonical data dir (`--data-dir`).
    pub data_dir: PathBuf,
    /// `<parent>/<name>.reseed-staging`.
    pub staging: PathBuf,
    /// `<parent>/<name>.discard`.
    pub discard: PathBuf,
}

impl SwapDirs {
    /// Derive the swap paths for `data_dir`.
    ///
    /// # Errors
    ///
    /// [`ServerError::Cluster`] when `data_dir` has no parent (it is a
    /// filesystem/volume mount root) or no final component — the §2.3 constraint
    /// that the data dir must be a subdirectory of a mount (e.g. `/data/db`, the
    /// k8s manifests' layout), so staging/discard can be same-filesystem
    /// siblings.
    pub fn derive(data_dir: &Path) -> Result<Self> {
        let parent = data_dir
            .parent()
            .filter(|p| !p.as_os_str().is_empty())
            .ok_or_else(|| {
                ServerError::Cluster(format!(
                    "reseed swap requires --data-dir ({}) to be a SUBDIRECTORY of a mount, not a \
                 filesystem/volume root: the swap renames the data dir itself, so staging and \
                 .discard must be same-filesystem siblings in its parent (§2.3). The k8s \
                 manifests mount the PVC at /data and set --data-dir /data/db.",
                    data_dir.display()
                ))
            })?;
        let name = data_dir.file_name().ok_or_else(|| {
            ServerError::Cluster(format!(
                "reseed swap: --data-dir ({}) has no final path component",
                data_dir.display()
            ))
        })?;
        let mut staging = name.to_os_string();
        staging.push(format!(".{STAGING_SUFFIX}"));
        let mut discard = name.to_os_string();
        discard.push(format!(".{DISCARD_SUFFIX}"));
        Ok(Self {
            data_dir: data_dir.to_path_buf(),
            staging: parent.join(staging),
            discard: parent.join(discard),
        })
    }
}

/// Recover from any swap interrupted by a crash (§2.3 boot recovery).
///
/// Runs BEFORE `ElectionStore` classification (the review found that classifying
/// first re-opens restart amnesia: a mid-rename crash leaves `data_dir` absent,
/// which a naive classifier would read as a fresh node at term 0 and re-vote).
///
/// The crash windows and their idempotent redo:
/// - staging WITHOUT `COMPLETE` → an interrupted fetch: delete staging.
/// - staging WITH `COMPLETE` + `data_dir` present → the crash landed between
///   writing `COMPLETE` and the first rename: redo the swap from step 4.
/// - staging WITH `COMPLETE` + `data_dir` ABSENT → the mid-rename crash (old dir
///   already renamed aside, staging not yet renamed in): complete the rename.
/// - stray `.discard` (any case) → a swap that crashed after `staging → data_dir`
///   but before the discard delete: delete it.
///
/// Every window is an idempotent redo; the identity files travel inside staging
/// so there is never an instant at which the canonical path lacks them post-swap.
///
/// # Errors
///
/// [`ServerError::Cluster`] on a rename/delete failure (loud, naming the §2.3
/// constraint when it is an `EXDEV`/mount-root cross-device error).
pub fn recover_swap(dirs: &SwapDirs) -> Result<()> {
    let staging_exists = dirs.staging.exists();
    let staging_complete = dirs.staging.join(COMPLETE_SENTINEL).exists();
    let data_exists = dirs.data_dir.exists();

    if staging_exists {
        if staging_complete {
            if data_exists {
                // Crash between COMPLETE and the first rename: the old dir is
                // still canonical. Redo the swap from step 4 (rename old aside,
                // rename staging in, fsync parent, delete discard).
                tracing::warn!(
                    staging = %dirs.staging.display(),
                    data_dir = %dirs.data_dir.display(),
                    "reseed swap recovery: a COMPLETE staging dir and a present data dir — \
                     redoing the swap from step 4 (idempotent)"
                );
                complete_swap(dirs)?;
            } else {
                // The mid-rename crash: old dir already renamed aside (to
                // discard or gone), staging not yet renamed in. Complete the
                // rename of staging → data_dir, fsync parent, delete any discard.
                tracing::warn!(
                    staging = %dirs.staging.display(),
                    "reseed swap recovery: COMPLETE staging with an ABSENT data dir (mid-rename \
                     crash) — completing the rename so a naive classifier never reads Fresh"
                );
                rename_same_fs(&dirs.staging, &dirs.data_dir)?;
                fsync_parent(&dirs.data_dir)?;
                delete_discard(dirs)?;
            }
        } else {
            // Interrupted fetch (no COMPLETE): the staged bytes are partial and
            // unverified — delete the whole staging dir and let the boot loop
            // retry or fall back.
            tracing::warn!(
                staging = %dirs.staging.display(),
                "reseed swap recovery: staging dir WITHOUT a COMPLETE sentinel (interrupted \
                 fetch) — deleting it"
            );
            remove_dir_all_loud(&dirs.staging, "interrupted staging")?;
        }
    }

    // A stray discard from a swap that crashed after the second rename but before
    // the discard delete. Safe to remove unconditionally (it is the OLD data dir,
    // already superseded by the new canonical one).
    if dirs.discard.exists() {
        tracing::warn!(
            discard = %dirs.discard.display(),
            "reseed swap recovery: deleting a stray .discard (post-swap cleanup)"
        );
        delete_discard(dirs)?;
    }

    Ok(())
}

/// The §2.3 step-4 swap, factored so both the live install and the recovery
/// redo run the identical sequence: rename `data_dir` aside → rename staging in
/// → fsync parent → delete discard.
fn complete_swap(dirs: &SwapDirs) -> Result<()> {
    // A leftover discard from a prior partial swap would make the first rename
    // fail (target exists); clear it first.
    if dirs.discard.exists() {
        delete_discard(dirs)?;
    }
    rename_same_fs(&dirs.data_dir, &dirs.discard)?;
    rename_same_fs(&dirs.staging, &dirs.data_dir)?;
    fsync_parent(&dirs.data_dir)?;
    delete_discard(dirs)?;
    Ok(())
}

fn delete_discard(dirs: &SwapDirs) -> Result<()> {
    remove_dir_all_loud(&dirs.discard, "discard")
}

fn remove_dir_all_loud(path: &Path, what: &str) -> Result<()> {
    match std::fs::remove_dir_all(path) {
        Ok(()) => Ok(()),
        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
        Err(e) => Err(ServerError::Cluster(format!(
            "reseed swap: failed to delete {what} dir {}: {e}",
            path.display()
        ))),
    }
}

/// Rename `from` → `to` on the SAME filesystem, surfacing `EXDEV` loudly with
/// the §2.3 constraint named.
fn rename_same_fs(from: &Path, to: &Path) -> Result<()> {
    std::fs::rename(from, to).map_err(|e| {
        // `EXDEV` (raw OS error 18 on Linux/macOS) means the rename crossed a
        // filesystem boundary — exactly the mount-root mistake §2.3 forbids.
        let is_exdev = e.raw_os_error() == Some(18);
        if is_exdev {
            ServerError::Cluster(format!(
                "reseed swap: rename {} → {} crossed a filesystem boundary (EXDEV). The §2.3 \
                 constraint REQUIRES --data-dir to be a subdirectory of a mount so staging and \
                 .discard are same-filesystem siblings (k8s: mount the PVC at /data, \
                 --data-dir /data/db). Refusing to copy across devices — fix the mount layout.",
                from.display(),
                to.display()
            ))
        } else {
            ServerError::Cluster(format!(
                "reseed swap: rename {} → {} failed: {e}",
                from.display(),
                to.display()
            ))
        }
    })
}

/// Durably fsync a directory's entry (the parent of `child`), so a swap's
/// renames survive a crash.
fn fsync_parent(child: &Path) -> Result<()> {
    let Some(parent) = child.parent() else {
        return Ok(());
    };
    let dir = std::fs::File::open(parent).map_err(|e| {
        ServerError::Cluster(format!(
            "reseed swap: open parent {} for fsync failed: {e}",
            parent.display()
        ))
    })?;
    dir.sync_all().map_err(|e| {
        ServerError::Cluster(format!(
            "reseed swap: fsync parent {} failed: {e}",
            parent.display()
        ))
    })?;
    Ok(())
}

/// Majority of `total_voters` voters (true `floor(n/2)+1`), matching the m11p5
/// §3.0 `majority()` fix — even sizes are mandatory transit states, so this must
/// be correct for every n.
#[must_use]
pub const fn majority(total_voters: usize) -> usize {
    total_voters / 2 + 1
}

/// Whether a self-restart for reseed is SAFE given the node's local view of how
/// many OTHER voters are alive (§2.4 quorum refusal).
///
/// A reseed self-restart removes THIS node from the live set while it is down.
/// It is only safe if the remaining voters can still form a quorum without it:
/// `alive_voters_excluding_self >= majority(total_voters)`. Otherwise exiting
/// during (e.g.) a 2-voter window is total write unavailability, so the
/// self-restart is REFUSED.
///
/// Note `total_voters` is the FULL voter count INCLUDING this node; the
/// remaining voters must form a majority of the whole set (the set size does
/// not shrink just because one node restarts — it is expected back).
#[must_use]
pub const fn self_restart_quorum_safe(
    alive_voters_excluding_self: usize,
    total_voters: usize,
) -> bool {
    alive_voters_excluding_self >= majority(total_voters)
}

/// The handshake window (env-overridable for tests).
fn handshake_window() -> Duration {
    let ms = std::env::var("TIDAL_RESEED_HANDSHAKE_MS")
        .ok()
        .and_then(|v| v.trim().parse::<u64>().ok())
        .unwrap_or(DEFAULT_HANDSHAKE_MS);
    Duration::from_millis(ms)
}

/// What the discovery loop learned about the current leader (§2.7).
#[derive(Debug, Clone)]
struct LeaderInfo {
    /// The leader's advertised gRPC `host:port` (the snapshot fetch target).
    grpc_addr: String,
    /// The leadership term the snapshot fetch + subsequent stream pulls stamp.
    term: u64,
    /// The discovered leader's `RegionId` (its 0-based topology index). Persisted
    /// in the install sentinel so the §2.6 post-open seed AND the post-install
    /// catch-up pull target the DISCOVERED leader's shard — not the boot
    /// topology leader, which for a reseeded ex-leader is the node ITSELF
    /// (m11p5 §2, the install-boot catch-up correction).
    region: u16,
}

/// One candidate the discovery loop polls: its HTTP status address plus the gRPC
/// address it advertises (so a `is_leader` answer immediately yields the fetch
/// target without a second lookup), and its `RegionId` (its 0-based topology
/// index) so a discovered leader's shard is known without a second lookup.
#[derive(Debug, Clone)]
struct Candidate {
    region: u16,
    http_addr: String,
    grpc_addr: String,
}

/// The outcome of a boot-time install attempt.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum InstallOutcome {
    /// A snapshot was installed (the swap completed). `ShardReplica::new`
    /// will find the install sentinel and run the §2.6 post-open seed.
    Installed,
    /// The leader reported `needed=false` (the live WAL still serves our
    /// frontier): the marker was cleared; open the existing dir and let the
    /// runtime path re-latch if catch-up still fails (self-correcting, §2.2d).
    NotNeeded,
    /// The handshake could not complete within the bounded window: fall back to
    /// opening the existing data dir, degraded, marker still latched, voting
    /// enabled (§2.2 review-blocker fix — never block boot forever).
    FellBack,
}

/// Run the boot-time snapshot install for a marker boot (§2.2–§2.7).
///
/// Returns the outcome; `ShardReplica::new` is constructed afterward in
/// every case (a fresh dir after `Installed`, the existing dir otherwise).
///
/// # Errors
///
/// [`ServerError::Cluster`] only for UNRECOVERABLE faults (a corrupt reseed
/// marker, a swap that crossed a filesystem boundary, a failed durable persist).
/// A reachability failure is NOT an error — it returns [`InstallOutcome::FellBack`]
/// so boot proceeds (the cluster may need this node's vote, §2.2).
pub fn run_boot_install(
    topology: &TopologySpec,
    region_name: &str,
    data_dir: &Path,
    shard: ShardId,
) -> Result<InstallOutcome> {
    run_boot_install_with(topology, region_name, data_dir, shard, handshake_window())
}

/// Run the boot-time reseed install for EVERY shard group this region hosts,
/// each against that group's OWN data subdir (the m11p6 layout) and with
/// SHARD-TARGETED leader discovery.
///
/// Fixes the multi-shard reseed loop: the per-group reseed marker + the divergent
/// WAL live in `<data_dir>/shard-{:05}` (`ShardReplica`'s own dir), but the legacy
/// single-call `run_boot_install(.., data_dir)` loaded the marker from the PARENT
/// dir, found none, returned `NotNeeded`, and never healed — so a divergent shard
/// re-quarantined and self-restarted forever. This visits each hosted group's
/// subdir (S==1 resolves to `data_dir` verbatim, preserving the byte-for-byte
/// legacy path) and discovers/dials THAT shard's leader (not the default group's).
///
/// # Errors
///
/// Propagates an unrecoverable install fault (see [`run_boot_install`]) or a
/// topology resolution error.
pub fn run_boot_install_for_region(
    topology: &TopologySpec,
    region_name: &str,
    data_dir: &Path,
) -> Result<Vec<(ShardId, InstallOutcome)>> {
    let groups = topology.resolve_shard_groups()?;
    let single = groups.len() == 1;
    let mut outcomes = Vec::new();
    for group in &groups {
        // Only groups THIS region replicates have local data to heal.
        if !group.replicas.iter().any(|r| r.name == region_name) {
            continue;
        }
        let group_dir = if single {
            data_dir.to_path_buf()
        } else {
            data_dir.join(super::node::shard_subdir(group.shard))
        };
        // A group subdir may not exist yet on a never-run node; the install's
        // recover_swap + marker load both tolerate an absent/empty dir (=>
        // NotNeeded), so this is safe to call unconditionally per hosted group.
        let outcome = run_boot_install(topology, region_name, &group_dir, group.shard)?;
        outcomes.push((group.shard, outcome));
    }
    Ok(outcomes)
}

/// [`run_boot_install`] with an explicit handshake window.
///
/// The window is the bounded time before the degraded fallback (§2.2). The
/// public entry point derives it from `TIDAL_RESEED_HANDSHAKE_MS` (or the 60 s
/// default); this variant lets a test bound it without mutating process env (the
/// crate forbids `unsafe`, so `set_var` is not an option).
///
/// # Errors
///
/// Same as [`run_boot_install`].
pub fn run_boot_install_with(
    topology: &TopologySpec,
    region_name: &str,
    data_dir: &Path,
    shard: ShardId,
    handshake_window: Duration,
) -> Result<InstallOutcome> {
    let dirs = SwapDirs::derive(data_dir)?;

    // (a) Swap-recovery FIRST — before any ElectionStore classification (§2.3).
    recover_swap(&dirs)?;

    // The marker may have been consumed by a recovered swap (its OLD dir, with
    // the marker, became .discard and was deleted). Re-read after recovery.
    let marker_store = ReseedMarkerStore::new(data_dir);
    let Some(marker) = marker_store
        .load()
        .map_err(|e| ServerError::Cluster(format!("refusing to boot: {e}")))?
    else {
        // No marker (or a recovered swap consumed it): nothing to install — the
        // caller opens the existing/fresh dir normally (NotNeeded = no install).
        return Ok(InstallOutcome::NotNeeded);
    };

    tracing::info!(
        region = region_name,
        reason = marker.reason.as_str(),
        from_seqno = marker.from_seqno,
        "boot-time reseed marker present: discovering a leader to snapshot-install from (§2.7)"
    );

    let candidates = candidates_for(topology, region_name, shard)?;
    if candidates.is_empty() {
        tracing::error!(
            "reseed boot: no peer candidates in the topology to discover a leader from; \
             falling back to opening the existing data dir (degraded, marker latched)"
        );
        return Ok(InstallOutcome::FellBack);
    }

    let tls = own_grpc_tls(topology, region_name);
    // m11p7: a TLS cluster's reseed discovery dials `https://` and trusts the
    // cluster CA, matching the rest of the inter-node HTTP plane.
    super::forward::set_inter_node_https(tls.is_some());
    let api_key = std::env::var("TIDAL_API_KEY").ok();
    let deadline = Instant::now() + handshake_window;
    let mut backoff = BACKOFF_MIN;

    // The blocking reqwest client for status polls (this runs on a dedicated
    // boot thread, not a reactor). A fresh runtime per fetch is acceptable — the
    // reseed is a rare boot operation, not a hot path.
    let mut status_builder = reqwest::blocking::Client::builder().timeout(STATUS_POLL_TIMEOUT);
    if let Some(t) = &tls {
        let Some(ca) = std::fs::read(&t.ca_cert)
            .ok()
            .and_then(|pem| reqwest::Certificate::from_pem(&pem).ok())
        else {
            tracing::error!("reseed boot: reading the inter-node CA cert failed; falling back");
            return Ok(InstallOutcome::FellBack);
        };
        status_builder = status_builder.add_root_certificate(ca);
    }
    let status_client = match status_builder.build() {
        Ok(c) => c,
        Err(e) => {
            tracing::error!(error = %e, "reseed boot: status client build failed; falling back");
            return Ok(InstallOutcome::FellBack);
        }
    };

    while Instant::now() < deadline {
        let Some(leader) = discover_leader(&status_client, &candidates, api_key.as_deref(), shard)
        else {
            std::thread::sleep(backoff);
            backoff = (backoff * 2).min(BACKOFF_MAX);
            continue;
        };

        // (c) Adopt the discovered term durably BEFORE pulling (§2.7
        // persist-before-act) when it exceeds our durable term. A failed durable
        // persist is unrecoverable (we cannot prove the term) — surface loudly.
        adopt_term_if_higher(data_dir, leader.term)?;

        // (d) Fetch the snapshot into staging and swap it in.
        match attempt_install(&dirs, &leader, marker, tls.as_ref()) {
            Ok(AttemptResult::Installed) => return Ok(InstallOutcome::Installed),
            Ok(AttemptResult::NotNeeded) => {
                // needed=false: the stream refused us at from_seqno previously,
                // but the leader now says the live WAL covers it — clear the
                // marker and retry the normal boot (self-correcting §2.2d). The
                // runtime path re-latches if catch-up still breaks.
                marker_store.clear().map_err(|e| {
                    ServerError::Cluster(format!(
                        "reseed boot: clear marker after needed=false: {e}"
                    ))
                })?;
                tracing::info!(
                    "reseed boot: leader reports the live WAL still covers our frontier \
                     (needed=false); cleared the marker and opening the existing data dir \
                     (self-correcting — the runtime re-latches if catch-up still fails)"
                );
                return Ok(InstallOutcome::NotNeeded);
            }
            Ok(AttemptResult::Retry) => {
                // A term-fence abort (an election landed mid-fetch) or a
                // transient transport failure: re-run discovery, re-stamp,
                // retry — bounded by the handshake deadline.
                std::thread::sleep(backoff);
                backoff = (backoff * 2).min(BACKOFF_MAX);
            }
            Err(e) => return Err(e),
        }
    }

    tracing::error!(
        window_ms = handshake_window.as_millis(),
        "reseed boot: could not complete the snapshot handshake within the bounded window; \
         falling back to opening the existing data dir (degraded, marker latched, VOTING \
         ENABLED — the cluster may need this node's vote to elect the leader the reseed needs, \
         §2.2). The reseed retries on the next boot."
    );
    Ok(InstallOutcome::FellBack)
}

/// Install a snapshot from an ALREADY-DISCOVERED leader (m11p5 §2.7 / §3.4).
///
/// The seed-join path. Unlike [`run_boot_install`], the caller has learned the
/// leader from a `JoinCluster` response (or the membership cache), so there is no
/// discovery loop and no reseed marker — just the §2.3 staging/verify/swap.
///
/// Returns:
/// - `Ok(Some(snapshot_seq))` — a snapshot was streamed, verified, and swapped
///   in; the new data dir carries the install sentinel (recording
///   `leader_region`) so `ShardReplica::new` runs the §2.6 post-open seed.
/// - `Ok(None)` — the leader reported `needed=false` (the live WAL still serves
///   `from_seqno`): nothing was installed, proceed empty (the small-cluster case
///   where the stream serves from 1).
///
/// `recover_swap` runs FIRST (a crash during a prior seed-join install's swap is
/// recovered idempotently, exactly as a marker boot's).
///
/// # Errors
///
/// [`ServerError::Cluster`] for an unrecoverable fault (a swap that crossed a
/// filesystem boundary, a failed durable persist, a fatal local filesystem
/// state). A transport/verification failure is RETRYABLE — surfaced as
/// `Ok(Err(reason))` is NOT used here; the caller's seed loop re-discovers and
/// re-invokes on a retryable failure, which this maps into a typed error the
/// caller distinguishes via [`SeedInstallError`].
pub fn install_snapshot_from_leader(
    data_dir: &Path,
    leader_grpc_addr: &str,
    leader_region: u16,
    term: u64,
    from_seqno: u64,
    tls: Option<&tidal_net::config::TlsConfig>,
) -> std::result::Result<Option<u64>, SeedInstallError> {
    let dirs = SwapDirs::derive(data_dir).map_err(SeedInstallError::Fatal)?;
    // Recover any interrupted prior swap before staging a fresh one (idempotent).
    recover_swap(&dirs).map_err(SeedInstallError::Fatal)?;

    let leader = LeaderInfo {
        grpc_addr: leader_grpc_addr.to_string(),
        term,
        region: leader_region,
    };
    // A synthetic marker carrying the resume seqno (the seed-join install has no
    // durable marker; this only conveys `from_seqno` into the shared
    // `attempt_install` path, which reads `marker.from_seqno`).
    let marker = ReseedMarker {
        reason: ReseedReason::Operator,
        from_seqno,
    };
    match attempt_install(&dirs, &leader, marker, tls) {
        Ok(AttemptResult::Installed) => {
            // Recover the installed artifact's recorded seq from the sentinel the
            // swap wrote into the (now canonical) data dir.
            let seq = read_install_sentinel(data_dir)
                .map_err(SeedInstallError::Fatal)?
                .map_or(0, |s| s.snapshot_seq);
            Ok(Some(seq))
        }
        Ok(AttemptResult::NotNeeded) => Ok(None),
        Ok(AttemptResult::Retry) => Err(SeedInstallError::Retry),
        Err(e) => Err(SeedInstallError::Fatal(e)),
    }
}

/// The outcome class of [`install_snapshot_from_leader`].
#[derive(Debug)]
pub enum SeedInstallError {
    /// Re-discover the leader and retry within the join window (term fence,
    /// transport error, verification miss).
    Retry,
    /// Unrecoverable (a swap that crossed a filesystem boundary, a failed
    /// durable persist): boot cannot proceed.
    Fatal(ServerError),
}

impl std::fmt::Display for SeedInstallError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            Self::Retry => write!(f, "snapshot install retryable (re-discover the leader)"),
            Self::Fatal(e) => write!(f, "snapshot install fatal: {e}"),
        }
    }
}

/// The candidate poll set (§2.7) for the reseed of shard group `shard`: the
/// OTHER replicas of THAT group, each carrying the group's PER-SHARD gRPC address
/// (the resolved replica's `grpc_addr` already includes the `+shard_id` port
/// offset) plus the replica's node-level HTTP address (the `:9500` plane is
/// per-node, not per-shard). Shard-targeting is load-bearing: a parent-default
/// poll would discover the WRONG shard's leader and install foreign data/term
/// into shard `shard`'s subdir.
///
/// # Errors
///
/// Propagates a topology resolution error.
fn candidates_for(
    topology: &TopologySpec,
    region_name: &str,
    shard: ShardId,
) -> Result<Vec<Candidate>> {
    let groups = topology.resolve_shard_groups()?;
    let Some(group) = groups.iter().find(|g| g.shard == shard) else {
        return Ok(Vec::new());
    };
    // The node-level HTTP address for a region (the `:9500` status/forward plane
    // is shared by every shard the node hosts; only the gRPC port is per-shard).
    let http_for = |name: &str| {
        topology
            .regions
            .iter()
            .find(|r| r.name == name)
            .and_then(|r| r.http_addr.clone())
    };
    Ok(group
        .replicas
        .iter()
        .filter(|r| r.name != region_name)
        .filter_map(|r| {
            http_for(&r.name).map(|http| Candidate {
                region: r.region.0,
                http_addr: http,
                grpc_addr: r.grpc_addr.clone(),
            })
        })
        .collect())
}

/// This region's own gRPC TLS material (the snapshot fetch dials the leader with
/// the same TLS posture this node advertises — the topology's `grpc_tls` block).
fn own_grpc_tls(
    topology: &TopologySpec,
    region_name: &str,
) -> Option<tidal_net::config::TlsConfig> {
    topology
        .regions
        .iter()
        .find(|r| r.name == region_name)
        .and_then(|r| r.grpc_tls.as_ref())
        .map(GrpcTlsSpec::to_tls_config)
}

/// Poll every candidate's `/cluster/status/local`; return the first that reports
/// itself the leader, with its advertised gRPC address + term.
fn discover_leader(
    client: &reqwest::blocking::Client,
    candidates: &[Candidate],
    api_key: Option<&str>,
    shard: ShardId,
) -> Option<LeaderInfo> {
    // The `?shard=N` selector makes `/cluster/status/local` report SHARD N's
    // leadership (status_local resolves `replica_for(sel.shard_id())`); without it
    // a multi-leader sharded node answers for its default (lowest-id) group, so we
    // would discover the wrong shard's leader/term.
    let path = format!("/cluster/status/local?shard={}", shard.0);
    for cand in candidates {
        let url = super::forward::peer_url(&cand.http_addr, &path);
        let mut req = client.get(&url);
        if let Some(key) = api_key {
            req = req.bearer_auth(key);
        }
        let Ok(resp) = req.send() else { continue };
        if !resp.status().is_success() {
            continue;
        }
        let Ok(json) = resp.json::<serde_json::Value>() else {
            continue;
        };
        let is_leader = json
            .get("is_leader")
            .and_then(serde_json::Value::as_bool)
            .unwrap_or(false);
        if !is_leader {
            continue;
        }
        let term = json
            .get("term")
            .and_then(serde_json::Value::as_u64)
            .unwrap_or(0);
        return Some(LeaderInfo {
            grpc_addr: cand.grpc_addr.clone(),
            term,
            region: cand.region,
        });
    }
    None
}

/// Adopt `term` durably (persist `{term, voted_for: None}`) when it is above the
/// node's durable term (§2.7 persist-before-act). A corrupt `election_state`
/// refuses boot (a reseed does NOT restore term knowledge; the m11p4 rule
/// stands, §2.5).
fn adopt_term_if_higher(data_dir: &Path, term: u64) -> Result<()> {
    let store = ElectionStore::new(data_dir);
    let wal_exists = data_dir
        .join("wal")
        .read_dir()
        .map(|mut it| it.next().is_some())
        .unwrap_or(false);
    let durable_term = match store.load(wal_exists) {
        Ok(tidaldb::replication::BootState::Rejoin(h)) => h.current_term,
        Ok(_) => 0, // Fresh / StateFileLost: term 0.
        Err(e) => {
            return Err(ServerError::Cluster(format!(
                "reseed boot: refusing to adopt a discovered term — {e} (a reseed does not \
                 restore term knowledge; the m11p4 corrupt-state rule stands, §2.5)"
            )));
        }
    };
    if term > durable_term {
        store
            .persist(HardState {
                current_term: term,
                voted_for: None,
            })
            .map_err(|e| {
                ServerError::Cluster(format!(
                    "reseed boot: persist discovered term {term} (persist-before-act) failed: {e}"
                ))
            })?;
        tracing::info!(
            durable_term,
            adopted = term,
            "reseed boot: adopted the discovered leadership term durably before pulling (§2.7)"
        );
    }
    Ok(())
}

/// One install attempt's result.
enum AttemptResult {
    /// The snapshot streamed, verified, and swapped in.
    Installed,
    /// The leader reported `needed=false`.
    NotNeeded,
    /// A retryable failure (term fence, transport error, verification miss):
    /// re-discover and retry within the handshake window.
    Retry,
}

/// Fetch a snapshot from `leader` into staging, verify, copy identity files,
/// write the sentinels, and swap (§2.3/§2.6/§2.7).
fn attempt_install(
    dirs: &SwapDirs,
    leader: &LeaderInfo,
    marker: ReseedMarker,
    tls: Option<&tidal_net::config::TlsConfig>,
) -> Result<AttemptResult> {
    // A stale staging dir from a prior aborted attempt: clear it.
    remove_dir_all_loud(&dirs.staging, "stale staging")?;
    std::fs::create_dir_all(&dirs.staging).map_err(|e| {
        ServerError::Cluster(format!(
            "reseed boot: create staging {} failed: {e}",
            dirs.staging.display()
        ))
    })?;

    // The leader's shard is its own shard; for the single-shard-per-region
    // mapping the snapshot's source shard is the leader's. We dial the leader's
    // advertised gRPC addr; the server-side handler resolves its own shard.
    // The puller stamps the discovered term and requests from the marker's
    // recorded from_seqno (§2.7 / §2.2d honesty: the stream already refused us
    // there).
    let fetch = fetch_into_staging(
        &leader.grpc_addr,
        tls,
        marker.from_seqno,
        leader.term,
        &dirs.staging,
    );

    match fetch {
        Ok(FetchResult::Needed { snapshot_seq }) => {
            // Copy the node's own identity files INTO staging (§2.3 step 2),
            // then the install sentinel (carrying the DISCOVERED leader's region
            // AND term so the post-open seed + catch-up pull target its shard —
            // not the boot topology leader's — and durably synthesize the term
            // marker), then COMPLETE, fsync, swap.
            copy_identity_into_staging(&dirs.data_dir, &dirs.staging)?;
            write_install_sentinel(&dirs.staging, snapshot_seq, leader.region, leader.term)?;
            write_complete_sentinel(&dirs.staging)?;
            complete_swap(dirs)?;
            tracing::info!(
                snapshot_seq,
                term = leader.term,
                "reseed boot: snapshot installed and swapped in; the post-open seed runs once \
                 ShardReplica opens the new data dir (§2.6)"
            );
            Ok(AttemptResult::Installed)
        }
        Ok(FetchResult::NotNeeded) => {
            remove_dir_all_loud(&dirs.staging, "needed=false staging")?;
            Ok(AttemptResult::NotNeeded)
        }
        Err(FetchError::Retryable(why)) => {
            tracing::warn!(reason = %why, "reseed boot: snapshot fetch retryable; re-discovering");
            remove_dir_all_loud(&dirs.staging, "retryable staging")?;
            Ok(AttemptResult::Retry)
        }
        Err(FetchError::Fatal(why)) => Err(ServerError::Cluster(format!(
            "reseed boot: snapshot install failed unrecoverably: {why}"
        ))),
    }
}

/// The result of streaming a snapshot into staging.
enum FetchResult {
    /// A snapshot was streamed + verified into staging.
    Needed { snapshot_seq: u64 },
    /// The leader reported `needed=false`.
    NotNeeded,
}

/// A snapshot fetch error.
enum FetchError {
    /// Re-discover the leader and retry (term fence, transport, verification).
    Retryable(String),
    /// Unrecoverable (an impossible local filesystem state).
    Fatal(String),
}

/// Stream the snapshot from `grpc_addr` into `staging`, verifying each file's
/// BLAKE3 against the manifest and the manifest's completeness.
///
/// Runs a dedicated current-thread tokio runtime (this is a rare boot operation
/// off the main reactor — `fetch_snapshot_standalone` is async).
// The streaming decode + per-file verification is one linear flow; splitting it
// would scatter the integrity-contract ordering (decode → write → hash → verify
// completeness → fsync).
#[allow(clippy::too_many_lines)]
fn fetch_into_staging(
    grpc_addr: &str,
    tls: Option<&tidal_net::config::TlsConfig>,
    from_seqno: u64,
    term: u64,
    staging: &Path,
) -> std::result::Result<FetchResult, FetchError> {
    use std::collections::HashMap;
    use std::io::Write;
    use tidal_net::client::fetch_snapshot_standalone;
    use tidal_net::proto::snapshot_chunk::Chunk;

    let runtime = tokio::runtime::Builder::new_current_thread()
        .enable_all()
        .build()
        .map_err(|e| FetchError::Retryable(format!("build fetch runtime: {e}")))?;

    runtime.block_on(async move {
        // The `FetchSnapshot` handler stages from the leader's OWN wired
        // `SnapshotSource` (its own shard) regardless of the requested shard id —
        // unlike `StreamSegments`, it does not route by shard. So the request's
        // `shard_id` is informational here; `from_seqno` is what drives the
        // needed/not-needed decision. We send shard 0 as the canonical value.
        let shard = ShardId(0);
        let mut stream = fetch_snapshot_standalone(grpc_addr, tls, shard, from_seqno, term)
            .await
            .map_err(|e| FetchError::Retryable(format!("open snapshot stream: {e}")))?;

        // Decode the header chunk first; then file chunks in contiguous order.
        let mut header: Option<tidal_net::proto::SnapshotHeader> = None;
        let mut open_files: HashMap<String, std::fs::File> = HashMap::new();
        let mut hashers: HashMap<String, blake3::Hasher> = HashMap::new();
        let mut sizes: HashMap<String, u64> = HashMap::new();

        loop {
            let msg = match stream.message().await {
                Ok(Some(m)) => m,
                Ok(None) => break,
                Err(status) => {
                    return Err(FetchError::Retryable(format!(
                        "snapshot stream error ({}): {}",
                        status.code(),
                        status.message()
                    )));
                }
            };
            let Some(chunk) = msg.chunk else {
                return Err(FetchError::Retryable("snapshot chunk had no variant".into()));
            };
            match chunk {
                Chunk::Header(h) => {
                    if !h.needed {
                        return Ok(FetchResult::NotNeeded);
                    }
                    header = Some(h);
                }
                Chunk::File(f) => {
                    let entry = open_files.entry(f.path.clone());
                    let file = match entry {
                        std::collections::hash_map::Entry::Occupied(o) => o.into_mut(),
                        std::collections::hash_map::Entry::Vacant(v) => {
                            let abs = staging.join(&f.path);
                            if let Some(parent) = abs.parent() {
                                std::fs::create_dir_all(parent).map_err(|e| {
                                    FetchError::Fatal(format!(
                                        "create staging subdir {}: {e}",
                                        parent.display()
                                    ))
                                })?;
                            }
                            let file = std::fs::File::create(&abs).map_err(|e| {
                                FetchError::Fatal(format!("create staged file {}: {e}", abs.display()))
                            })?;
                            hashers.insert(f.path.clone(), blake3::Hasher::new());
                            sizes.insert(f.path.clone(), 0);
                            v.insert(file)
                        }
                    };
                    // Contiguous-offset enforcement: a gap means a corrupt/torn
                    // stream — refuse (retryable; re-fetch from scratch).
                    let written = sizes.get(&f.path).copied().unwrap_or(0);
                    if f.offset != written {
                        return Err(FetchError::Retryable(format!(
                            "snapshot file '{}' arrived out of order (offset {} != written {})",
                            f.path, f.offset, written
                        )));
                    }
                    file.write_all(&f.data).map_err(|e| {
                        FetchError::Fatal(format!("write staged file '{}': {e}", f.path))
                    })?;
                    if let Some(h) = hashers.get_mut(&f.path) {
                        h.update(&f.data);
                    }
                    *sizes.entry(f.path.clone()).or_insert(0) += f.data.len() as u64;
                }
            }
        }

        let Some(header) = header else {
            return Err(FetchError::Retryable(
                "snapshot stream ended with no header (needed=true expected)".into(),
            ));
        };

        // Completeness: every manifest entry must have streamed, with its exact
        // size and BLAKE3 (the end-to-end integrity contract — the bytes a
        // joiner installs MUST be bit-exact or the open refuses).
        for entry in &header.files {
            let got_size = sizes.get(&entry.path).copied().ok_or_else(|| {
                FetchError::Retryable(format!("manifest file '{}' never streamed", entry.path))
            })?;
            if got_size != entry.size {
                return Err(FetchError::Retryable(format!(
                    "manifest file '{}' size mismatch: got {got_size}, expected {}",
                    entry.path, entry.size
                )));
            }
            let got_hash = hashers
                .get(&entry.path)
                .map(blake3::Hasher::finalize)
                .ok_or_else(|| {
                    FetchError::Retryable(format!("manifest file '{}' had no hasher", entry.path))
                })?;
            if got_hash.as_bytes().as_slice() != entry.blake3.as_slice() {
                return Err(FetchError::Retryable(format!(
                    "manifest file '{}' BLAKE3 mismatch — refusing the install (identify-or-refuse)",
                    entry.path
                )));
            }
        }
        // Reject any streamed file NOT in the manifest (a hostile/torn extra).
        for streamed in sizes.keys() {
            if !header.files.iter().any(|e| &e.path == streamed) {
                return Err(FetchError::Retryable(format!(
                    "snapshot streamed an unmanifested file '{streamed}'"
                )));
            }
        }
        // fsync every staged file so the verified bytes are durable before the
        // sentinels / swap.
        for file in open_files.values() {
            file.sync_all()
                .map_err(|e| FetchError::Fatal(format!("fsync staged file: {e}")))?;
        }

        Ok(FetchResult::Needed {
            snapshot_seq: header.snapshot_seq,
        })
    })
}

/// Copy the node's own identity files INTO staging (§2.3 step 2): `election_state`
/// (vote safety, §2.5) and the `membership` cache. These travel inside staging so
/// the canonical path is never without them post-swap.
fn copy_identity_into_staging(data_dir: &Path, staging: &Path) -> Result<()> {
    for name in ["election_state", "membership"] {
        let src = data_dir.join(name);
        if src.exists() {
            let dst = staging.join(name);
            std::fs::copy(&src, &dst).map_err(|e| {
                ServerError::Cluster(format!(
                    "reseed boot: copy identity file {name} into staging failed: {e}"
                ))
            })?;
            if let Ok(f) = std::fs::File::open(&dst) {
                let _ = f.sync_all();
            }
        }
    }
    Ok(())
}

/// The install sentinel's body (m11p5 §2.6, extended m12 reseed-loop-fix).
///
/// It carries the artifact's recovered seq, the DISCOVERED leader's region, and
/// the artifact's captured TERM. All three drive the post-open path:
/// `ShardReplica::new` seeds the frontier, durably synthesizes the term marker
/// the artifact represents (so `wal_term_mark()` reports the artifact's term —
/// not 0 — on this boot and every reboot), and issues the post-install catch-up
/// pull against the discovered leader's shard (NOT the boot topology leader,
/// which for a reseeded ex-leader is the node itself).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct InstallSentinel {
    /// The installed artifact's recovered WAL tail seq.
    pub snapshot_seq: u64,
    /// The discovered leader's `RegionId` (its 0-based topology index). `None`
    /// when the sentinel predates this field (an interrupted older install): the
    /// caller then falls back to the boot topology leader's shard.
    pub leader_region: Option<u16>,
    /// The artifact's captured leadership TERM (the discovered leader's term at
    /// fetch time). `None` when the sentinel predates this field (a legacy 8- or
    /// 10-byte install): the post-open seed then SKIPS the durable term-marker
    /// synthesis (it cannot fabricate a term it does not know) and falls back to
    /// the historical behavior. A truthful term ≥ 1 lets the seed make
    /// `wal_term_mark()` report it, so `decide_join` classifies the reseeded
    /// shard on a real `(term, frontier)` instead of the false `(0, …)` that
    /// loops `ReseedRequired` after a checkpoint-restore empties the WAL.
    pub artifact_term: Option<u64>,
}

/// The install sentinel's on-disk size, 18 bytes: 8 bytes seq (LE), then 2
/// bytes region (LE), then 8 bytes term (LE). A 10-byte sentinel (seq + region,
/// no term) and an 8-byte one (seq only) are still read for back-compat with an
/// install that was staged by an older binary and interrupted across the upgrade.
const INSTALL_SENTINEL_SIZE: usize = 8 + 2 + 8;
/// Legacy size: 8 bytes seq + 2 bytes region, no term.
const INSTALL_SENTINEL_SIZE_NO_TERM: usize = 8 + 2;

/// Write the `reseed-install-pending` install sentinel INTO staging BEFORE the
/// swap (§2.6): it carries the artifact's recovered seq, the discovered
/// leader's region, AND the artifact's captured term, so the post-open seed
/// advances the right shard's frontier, durably synthesizes the term marker,
/// and the catch-up pull dials the discovered leader. Crash-idempotent: a crash
/// before the seed's sentinel-delete re-runs the seed against the unchanged WAL.
fn write_install_sentinel(
    staging: &Path,
    snapshot_seq: u64,
    leader_region: u16,
    artifact_term: u64,
) -> Result<()> {
    let path = staging.join(INSTALL_PENDING_SENTINEL);
    let mut bytes = [0u8; INSTALL_SENTINEL_SIZE];
    bytes[0..8].copy_from_slice(&snapshot_seq.to_le_bytes());
    bytes[8..10].copy_from_slice(&leader_region.to_le_bytes());
    bytes[10..18].copy_from_slice(&artifact_term.to_le_bytes());
    std::fs::write(&path, bytes).map_err(|e| {
        ServerError::Cluster(format!(
            "reseed boot: write install sentinel {} failed: {e}",
            path.display()
        ))
    })?;
    if let Ok(f) = std::fs::File::open(&path) {
        let _ = f.sync_all();
    }
    Ok(())
}

/// Write the `COMPLETE` sentinel INTO staging and fsync it (§2.3 step 3): its
/// presence makes the staged dir swap-ready (the recovery path keys off it).
fn write_complete_sentinel(staging: &Path) -> Result<()> {
    let path = staging.join(COMPLETE_SENTINEL);
    let f = std::fs::File::create(&path).map_err(|e| {
        ServerError::Cluster(format!(
            "reseed boot: create COMPLETE sentinel {} failed: {e}",
            path.display()
        ))
    })?;
    f.sync_all().map_err(|e| {
        ServerError::Cluster(format!("reseed boot: fsync COMPLETE sentinel failed: {e}"))
    })?;
    // fsync the staging dir entry so the sentinel is durable before the rename.
    fsync_parent(&path)?;
    Ok(())
}

/// Read the install sentinel's recorded body from `data_dir`, if present.
///
/// Called by `ShardReplica::new` after `open_region_db` to drive the §2.6
/// post-open seed AND the post-install catch-up pull. `None` ⇒ this was not an
/// install boot. An 18-byte sentinel carries the discovered leader's region AND
/// the artifact's captured term; a 10-byte one carries the region but no term
/// (`artifact_term: None`); an 8-byte one (an interrupted older install) carries
/// neither (`leader_region: None`, `artifact_term: None`) so the caller falls
/// back to the boot topology leader's shard and skips the term-marker synthesis.
///
/// # Errors
///
/// [`ServerError::Cluster`] when the sentinel exists but is unreadable or
/// malformed (a present-but-corrupt sentinel must NOT be silently ignored — it
/// would skip the seed and leave the frontier wrong).
pub fn read_install_sentinel(data_dir: &Path) -> Result<Option<InstallSentinel>> {
    let path = data_dir.join(INSTALL_PENDING_SENTINEL);
    match std::fs::read(&path) {
        Ok(bytes) => {
            // All accepted lengths begin with the 8-byte seq; the 10-byte form
            // adds the 2-byte region; the 18-byte form adds the 8-byte term. No
            // slice `expect` — every index here is bounds-checked by the length
            // match (panic-free).
            let len = bytes.len();
            if len != INSTALL_SENTINEL_SIZE && len != INSTALL_SENTINEL_SIZE_NO_TERM && len != 8 {
                return Err(ServerError::Cluster(format!(
                    "reseed install sentinel {} is malformed ({len} bytes, expected \
                     {INSTALL_SENTINEL_SIZE}, {INSTALL_SENTINEL_SIZE_NO_TERM}, or 8); the \
                     post-open seed cannot proceed",
                    path.display()
                )));
            }
            let mut seq_bytes = [0u8; 8];
            seq_bytes.copy_from_slice(&bytes[0..8]);
            let snapshot_seq = u64::from_le_bytes(seq_bytes);
            let leader_region = if len >= INSTALL_SENTINEL_SIZE_NO_TERM {
                let mut region_bytes = [0u8; 2];
                region_bytes.copy_from_slice(&bytes[8..10]);
                Some(u16::from_le_bytes(region_bytes))
            } else {
                None
            };
            let artifact_term = if len == INSTALL_SENTINEL_SIZE {
                let mut term_bytes = [0u8; 8];
                term_bytes.copy_from_slice(&bytes[10..18]);
                Some(u64::from_le_bytes(term_bytes))
            } else {
                None
            };
            Ok(Some(InstallSentinel {
                snapshot_seq,
                leader_region,
                artifact_term,
            }))
        }
        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
        Err(e) => Err(ServerError::Cluster(format!(
            "reseed install sentinel {} unreadable: {e}",
            path.display()
        ))),
    }
}

/// Delete the install sentinel after the §2.6 post-open seed is durably
/// persisted (and dir-fsync the data dir so the delete survives a crash).
///
/// # Errors
///
/// [`ServerError::Cluster`] on a delete/fsync failure.
pub fn clear_install_sentinel(data_dir: &Path) -> Result<()> {
    let path = data_dir.join(INSTALL_PENDING_SENTINEL);
    match std::fs::remove_file(&path) {
        Ok(()) => {}
        Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()),
        Err(e) => {
            return Err(ServerError::Cluster(format!(
                "reseed boot: clear install sentinel {} failed: {e}",
                path.display()
            )));
        }
    }
    fsync_parent(&path)?;
    Ok(())
}

#[cfg(test)]
#[allow(clippy::unwrap_used)]
mod tests {
    use super::*;

    /// Build a [`SwapDirs`] over a tempdir-resident `data_dir` subdirectory.
    fn dirs_in(tmp: &Path) -> SwapDirs {
        let data = tmp.join("db");
        std::fs::create_dir_all(&data).unwrap();
        SwapDirs::derive(&data).unwrap()
    }

    #[test]
    fn derive_refuses_a_mount_root() {
        // A path with no usable parent (the empty parent of a bare relative
        // name) must be refused naming the §2.3 constraint. `/` has parent `/`,
        // which is non-empty, so to exercise the refusal we use a bare name.
        let err = SwapDirs::derive(Path::new("db")).err();
        // "db" has parent "" (empty) → refused.
        assert!(err.is_some(), "a parentless data dir must be refused");
        assert!(
            err.unwrap().to_string().contains("§2.3"),
            "the refusal names the §2.3 constraint"
        );
    }

    #[test]
    fn derive_paths_are_same_fs_siblings() {
        let tmp = tempfile::tempdir().unwrap();
        let dirs = dirs_in(tmp.path());
        assert_eq!(dirs.staging.parent(), dirs.data_dir.parent());
        assert_eq!(dirs.discard.parent(), dirs.data_dir.parent());
        assert!(dirs.staging.to_string_lossy().ends_with(".reseed-staging"));
        assert!(dirs.discard.to_string_lossy().ends_with(".discard"));
    }

    /// §2.3 window: staging WITHOUT COMPLETE = interrupted fetch → deleted.
    #[test]
    fn recover_deletes_incomplete_staging() {
        let tmp = tempfile::tempdir().unwrap();
        let dirs = dirs_in(tmp.path());
        std::fs::create_dir_all(&dirs.staging).unwrap();
        std::fs::write(dirs.staging.join("partial"), b"x").unwrap();

        recover_swap(&dirs).unwrap();
        assert!(!dirs.staging.exists(), "incomplete staging is deleted");
        assert!(
            dirs.data_dir.exists(),
            "the canonical data dir is untouched"
        );
    }

    /// §2.3 window: staging WITH COMPLETE + `data_dir` present → redo the swap
    /// from step 4 (the new staging becomes canonical, old goes to
    /// discard+deleted).
    #[test]
    fn recover_redoes_swap_when_complete_and_data_present() {
        let tmp = tempfile::tempdir().unwrap();
        let dirs = dirs_in(tmp.path());
        // Mark the OLD (current) data dir so we can prove it was superseded.
        std::fs::write(dirs.data_dir.join("old-marker"), b"old").unwrap();
        // A COMPLETE staging dir with a distinct file.
        std::fs::create_dir_all(&dirs.staging).unwrap();
        std::fs::write(dirs.staging.join("new-marker"), b"new").unwrap();
        std::fs::write(dirs.staging.join(COMPLETE_SENTINEL), b"").unwrap();

        recover_swap(&dirs).unwrap();

        assert!(dirs.data_dir.exists(), "data dir present after redo");
        assert!(
            dirs.data_dir.join("new-marker").exists(),
            "the staged content became canonical"
        );
        assert!(
            !dirs.data_dir.join("old-marker").exists(),
            "the old data dir was discarded"
        );
        assert!(!dirs.staging.exists(), "staging consumed");
        assert!(!dirs.discard.exists(), "discard cleaned up");
    }

    /// §2.3 window: the mid-rename crash — COMPLETE staging + ABSENT `data_dir`
    /// → complete the rename so a naive classifier never reads Fresh.
    #[test]
    fn recover_completes_mid_rename_when_data_absent() {
        let tmp = tempfile::tempdir().unwrap();
        let dirs = dirs_in(tmp.path());
        // Simulate the crash: old data dir already renamed aside (remove it),
        // staging present + COMPLETE.
        std::fs::remove_dir_all(&dirs.data_dir).unwrap();
        std::fs::create_dir_all(&dirs.staging).unwrap();
        std::fs::write(dirs.staging.join("new-marker"), b"new").unwrap();
        std::fs::write(dirs.staging.join(COMPLETE_SENTINEL), b"").unwrap();

        recover_swap(&dirs).unwrap();

        assert!(dirs.data_dir.exists(), "the rename was completed");
        assert!(
            dirs.data_dir.join("new-marker").exists(),
            "the staged content is now canonical"
        );
        assert!(!dirs.staging.exists(), "staging consumed");
    }

    /// §2.3 window: a stray .discard from a swap that crashed after the second
    /// rename but before the discard delete → deleted, data dir untouched.
    #[test]
    fn recover_deletes_stray_discard() {
        let tmp = tempfile::tempdir().unwrap();
        let dirs = dirs_in(tmp.path());
        std::fs::write(dirs.data_dir.join("live"), b"live").unwrap();
        std::fs::create_dir_all(&dirs.discard).unwrap();
        std::fs::write(dirs.discard.join("old"), b"old").unwrap();

        recover_swap(&dirs).unwrap();

        assert!(!dirs.discard.exists(), "stray discard deleted");
        assert!(
            dirs.data_dir.join("live").exists(),
            "the live data dir is untouched"
        );
    }

    /// `recover_swap` is idempotent: a clean tree (no staging, no discard) is a
    /// no-op, and re-running after a recovery changes nothing.
    #[test]
    fn recover_is_idempotent_on_a_clean_tree() {
        let tmp = tempfile::tempdir().unwrap();
        let dirs = dirs_in(tmp.path());
        std::fs::write(dirs.data_dir.join("live"), b"live").unwrap();
        recover_swap(&dirs).unwrap();
        recover_swap(&dirs).unwrap();
        assert!(dirs.data_dir.join("live").exists());
    }

    /// §2.4 quorum-refusal arithmetic: a self-restart is safe iff the remaining
    /// voters can form a majority of the FULL voter set without this node.
    #[test]
    fn quorum_refusal_arithmetic() {
        // n=3: majority 2. With this node down, 2 others alive → safe; 1 → refuse.
        assert_eq!(majority(3), 2);
        assert!(
            self_restart_quorum_safe(2, 3),
            "3 voters, 2 others alive: safe"
        );
        assert!(
            !self_restart_quorum_safe(1, 3),
            "3 voters, 1 other alive: refuse"
        );

        // n=2: majority 2. Restarting either leaves 1 other, < 2 → ALWAYS refuse
        // (a 2-voter window cannot lose a voter without losing write quorum).
        assert_eq!(majority(2), 2);
        assert!(
            !self_restart_quorum_safe(1, 2),
            "2 voters: refuse (total outage)"
        );

        // n=5: majority 3. 3+ others alive → safe; 2 → refuse.
        assert_eq!(majority(5), 3);
        assert!(self_restart_quorum_safe(3, 5));
        assert!(self_restart_quorum_safe(4, 5));
        assert!(!self_restart_quorum_safe(2, 5));

        // n=4 (an even transit state): majority 3. 3 others alive → safe; 2 →
        // refuse (proves the even-n correctness, not the buggy n/2+1=3 vs the
        // disjoint-quorum hazard).
        assert_eq!(majority(4), 3);
        assert!(self_restart_quorum_safe(3, 4));
        assert!(!self_restart_quorum_safe(2, 4));

        // n=1 (single node): majority 1; with this node down 0 others → refuse.
        assert_eq!(majority(1), 1);
        assert!(!self_restart_quorum_safe(0, 1));
    }

    /// The install + clear sentinel round-trips (carrying the discovered leader
    /// region AND the artifact term), reads a legacy 10-byte sentinel as
    /// `artifact_term: None`, an 8-byte one as `leader_region: None` too, and
    /// refuses a malformed sentinel.
    #[test]
    fn install_sentinel_roundtrip_and_malformed() {
        let tmp = tempfile::tempdir().unwrap();
        let data = tmp.path();
        assert_eq!(read_install_sentinel(data).unwrap(), None);

        // The 18-byte sentinel carries the discovered leader's region AND term.
        write_install_sentinel(data, 4096, 2, 5).unwrap();
        assert_eq!(
            read_install_sentinel(data).unwrap(),
            Some(InstallSentinel {
                snapshot_seq: 4096,
                leader_region: Some(2),
                artifact_term: Some(5),
            })
        );

        clear_install_sentinel(data).unwrap();
        assert_eq!(read_install_sentinel(data).unwrap(), None);
        // clear is idempotent.
        clear_install_sentinel(data).unwrap();

        // A legacy 10-byte sentinel (region but no term — staged by an older
        // binary and interrupted across the upgrade) reads with artifact_term
        // None, so the seed skips the term-marker synthesis.
        let mut ten = [0u8; INSTALL_SENTINEL_SIZE_NO_TERM];
        ten[0..8].copy_from_slice(&9u64.to_le_bytes());
        ten[8..10].copy_from_slice(&1u16.to_le_bytes());
        std::fs::write(data.join(INSTALL_PENDING_SENTINEL), ten).unwrap();
        assert_eq!(
            read_install_sentinel(data).unwrap(),
            Some(InstallSentinel {
                snapshot_seq: 9,
                leader_region: Some(1),
                artifact_term: None,
            })
        );
        clear_install_sentinel(data).unwrap();

        // A legacy 8-byte sentinel (an interrupted older install) reads with
        // leader_region None — the caller falls back to the boot topology leader.
        std::fs::write(data.join(INSTALL_PENDING_SENTINEL), 7u64.to_le_bytes()).unwrap();
        assert_eq!(
            read_install_sentinel(data).unwrap(),
            Some(InstallSentinel {
                snapshot_seq: 7,
                leader_region: None,
                artifact_term: None,
            })
        );
        clear_install_sentinel(data).unwrap();

        // A malformed sentinel (wrong length) is refused, not silently ignored.
        std::fs::write(data.join(INSTALL_PENDING_SENTINEL), b"short").unwrap();
        assert!(read_install_sentinel(data).is_err());
    }

    /// `adopt_term_if_higher` persists a higher term and is a no-op for a lower
    /// one.
    #[test]
    fn adopt_term_persists_only_when_higher() {
        let tmp = tempfile::tempdir().unwrap();
        let data = tmp.path();
        // Fresh dir, no election_state, no wal → durable term 0.
        adopt_term_if_higher(data, 5).unwrap();
        let store = ElectionStore::new(data);
        let h = match store.load(false).unwrap() {
            tidaldb::replication::BootState::Rejoin(h) => h,
            other => panic!("expected Rejoin after adopt, got {other:?}"),
        };
        assert_eq!(h.current_term, 5);
        assert_eq!(h.voted_for, None);

        // A lower discovered term is a no-op (never regresses).
        adopt_term_if_higher(data, 3).unwrap();
        let h2 = match store.load(false).unwrap() {
            tidaldb::replication::BootState::Rejoin(h) => h,
            other => panic!("{other:?}"),
        };
        assert_eq!(
            h2.current_term, 5,
            "a lower term never regresses durable state"
        );
    }
}