//! The public entry point for tidalDB: [`TidalDb`] handle and [`TidalDbBuilder`].

pub(crate) mod backup;
pub mod builder;
mod capabilities;
mod cohorts;
pub(crate) mod collections;
mod communities;
pub mod config;
mod creators;
pub(crate) mod export;
#[cfg(feature = "metrics")]
pub mod http;
mod items;
pub(crate) mod metadata;
pub mod metrics;
pub(crate) mod notification_tracker;
mod open;
pub mod paths;
mod query_ops;
mod relationships;
mod remove_scope;
pub(crate) mod schema_fingerprint;
mod session_restore;
mod sessions;
mod signals;
mod state_rebuild;
pub(crate) mod storage_box;
#[cfg(any(test, feature = "test-utils"))]
pub mod temp;
mod users;
pub(crate) mod wal_bridge;

use std::sync::{
    Arc, RwLock,
    atomic::{AtomicBool, AtomicU64},
};

pub use backup::BackupInfo;
pub use builder::TidalDbBuilder;
pub use config::{Config, ConfigError, NodeConfig, StorageMode};
pub use export::UserSessionSummary;
pub(crate) use metadata::deserialize_metadata;
pub use metrics::MetricsState;
pub use paths::Paths;
use roaring::RoaringBitmap;
#[cfg(any(test, feature = "test-utils"))]
pub use temp::TempTidalHome;

use self::{open::OpenResult, state_rebuild::run_checkpoint_thread, storage_box::StorageBox};
use crate::{
    entities::{
        CoEngagementIndex, CollectionIndex, CreatorItemsBitmap, HardNegIndex, InteractionLedger,
        PreferenceVectors, UserSignalIndex, UserStateIndex,
    },
    ranking::{builtins::register_builtins, registry::ProfileRegistry},
    schema::{EntityKind, Schema},
    session::{SessionId, SessionSnapshot, SessionState},
    signals::SignalLedger,
    storage::{
        StorageEngine,
        indexes::{bitmap::BitmapIndex, range::RangeIndex},
        vector::registry::EmbeddingSlotRegistry,
    },
    wal::WalHandle,
};

mod diagnostics;
mod lifecycle;
mod replication_ops;
mod sweeper;
mod text_syncer;

use self::text_syncer::open_text_syncer;

/// A tidalDB database instance.
///
/// Created via [`TidalDb::builder()`]. Call [`close`](Self::close) for
/// explicit shutdown; [`Drop`] runs best-effort cleanup if not called.
/// `Send + Sync` -- wrap in `Arc` for multi-threaded access.
///
/// # Pending refactor: field-cluster extraction (`CODING_GUIDELINES` §9)
///
/// This struct carries ~60 fields spanning M0–M10 concerns and `mod.rs` exceeds
/// the §9 file-size guideline. The shared-default initializers were extracted
/// into [`SharedDefaults`] to de-duplicate the two constructors, but the fields
/// themselves still live flat on `TidalDb`. The next structural step is to group
/// cohesive clusters into sub-structs — candidates: `replication` (state +
/// receiver/shipper handles + bridge + tenancy + control plane), `governance`
/// (membership/community/governance/capability registries), `text` (both text
/// indexes, channels, syncer threads, flush channels, health latch), and
/// `lifecycle` (checkpoint/sweeper handles + their shutdown flags).
///
/// Each cluster's blast radius reaches OUTSIDE `mod.rs`, so the extraction must
/// be done as one whole-crate change per cluster (it rewrites every
/// `self.<field>` to `self.<cluster>.<field>` across `db/lifecycle.rs`,
/// `db/sweeper.rs`, `db/diagnostics.rs`, `db/communities.rs`, `db/backup.rs`,
/// `db/remove_scope.rs`, `db/capabilities.rs`, `db/replication_ops.rs`, ...).
/// It cannot be applied safely from a `mod.rs`-only edit. Do it incrementally,
/// one cluster per change, with the whole crate compiling between steps.
pub struct TidalDb {
    config: Config,
    closed: AtomicBool,
    metrics: Arc<MetricsState>,
    #[cfg(feature = "metrics")]
    metrics_handle: Option<http::MetricsHandle>,
    ledger: Option<Arc<SignalLedger>>,
    storage: Option<StorageBox>,
    wal: std::sync::Mutex<Option<WalHandle>>,
    last_wal_seq: Arc<AtomicU64>,
    shutdown_checkpoint: Arc<AtomicBool>,
    checkpoint_thread: std::sync::Mutex<Option<std::thread::JoinHandle<()>>>,
    // Latched true once either text syncer reports unhealthy (an unrecoverable
    // commit failure or exhausted retry budget). Mirrors the
    // `metrics.checkpoint_thread_died` latch but for text-search liveness, and is
    // owned by `TidalDb` (not `MetricsState`) so `health_check` can fold it into
    // the degraded decision without depending on `JoinHandle::is_finished` — the
    // syncer now survives commit failures, so a finished handle is not the
    // signal; `TextIndex::is_healthy()` is.
    text_index_unhealthy: Arc<AtomicBool>,
    // M2 indexes
    category_index: BitmapIndex,
    format_index: BitmapIndex,
    creator_index: BitmapIndex,
    tag_index: BitmapIndex,
    duration_index: RangeIndex<u32>,
    created_at_index: RangeIndex<u64>,
    universe: Arc<RwLock<RoaringBitmap>>,
    embedding_registry: Arc<RwLock<EmbeddingSlotRegistry>>,
    profile_registry: ProfileRegistry,
    #[allow(dead_code)]
    schema_def: Option<Schema>,
    // M3 entities
    creator_items: Arc<CreatorItemsBitmap>,
    user_state: Arc<UserStateIndex>,
    hard_negatives: Arc<HardNegIndex>,
    interaction_ledger: Arc<InteractionLedger>,
    preference_vectors: Arc<PreferenceVectors>,
    // M5 text index
    #[allow(dead_code)]
    text_index: Option<Arc<crate::text::TextIndex>>,
    text_tx: std::sync::Mutex<Option<crossbeam::channel::Sender<crate::text::PendingWrite>>>,
    text_syncer_thread: std::sync::Mutex<Option<std::thread::JoinHandle<crate::Result<()>>>>,
    creator_text_index: Option<Arc<crate::text::TextIndex>>,
    creator_text_tx:
        std::sync::Mutex<Option<crossbeam::channel::Sender<crate::text::PendingWrite>>>,
    creator_text_syncer_thread:
        std::sync::Mutex<Option<std::thread::JoinHandle<crate::Result<()>>>>,
    #[allow(dead_code)]
    text_flush_tx: Option<crossbeam::channel::Sender<crossbeam::channel::Sender<()>>>,
    #[allow(dead_code)]
    creator_text_flush_tx: Option<crossbeam::channel::Sender<crossbeam::channel::Sender<()>>>,
    // M4 sessions
    #[allow(dead_code)]
    sessions: dashmap::DashMap<SessionId, Arc<SessionState>>,
    #[allow(dead_code)]
    next_session_id: AtomicU64,
    #[allow(dead_code)]
    closed_sessions: dashmap::DashMap<SessionId, SessionSnapshot>,
    // M6 co-engagement + social graph
    co_engagement: Arc<CoEngagementIndex>,
    user_signal_index: Arc<UserSignalIndex>,
    // M6 cohort engine
    cohort_registry: Arc<crate::cohort::CohortRegistry>,
    cohort_resolver: Arc<crate::cohort::CohortResolver>,
    cohort_ledger: Arc<crate::cohort::CohortSignalLedger>,
    // M6p4 collections
    collection_index: Arc<CollectionIndex>,
    // M6p5 suggestion index
    suggestion_index: Arc<crate::query::suggest::SuggestionIndex>,
    // M6p6 notification tracker
    notification_tracker: Arc<notification_tracker::NotificationTracker>,
    // M7p2 load detector
    load_detector: Arc<crate::load::LoadDetector>,
    // M7p2 backpressure config
    backpressure_config: crate::load::BackpressureConfig,
    // M7p2 per-agent rate limiter
    rate_limiter: Arc<crate::load::RateLimiter>,
    // M7p2 session TTL sweeper
    shutdown_sweeper: Arc<AtomicBool>,
    sweeper_thread: std::sync::Mutex<Option<std::thread::JoinHandle<()>>>,
    // True when a backup is in progress. Signal writes return Backpressure
    // during the backup window. Cleared by the BackupGuard RAII drop.
    backup_in_progress: Arc<AtomicBool>,
    // M8p2 replication
    replication_state: Arc<crate::replication::state::ReplicationState>,
    receiver_handle: std::sync::Mutex<Option<crate::replication::receiver::SegmentReceiverHandle>>,
    #[allow(dead_code)] // Held for its Drop side effect (thread join on shutdown).
    shipper_handle: Option<crate::replication::WalShipperHandle>,
    // M8p4 session replication bridge (dedup + cross-region session visibility)
    session_bridge: Arc<crate::replication::SessionReplicationBridge>,
    // M8p5 control plane + multi-tenancy
    tenant_router: Arc<crate::replication::TenantRouter>,
    control_plane: Arc<crate::replication::ControlPlane>,
    // M9p2 community membership lifecycle (join/leave/rejoin + stop-forward gate)
    membership_registry: Arc<crate::governance::MembershipRegistry>,
    // M9p3 community signal aggregation with per-contributor attribution (purge)
    community_ledger: Arc<crate::governance::CommunityLedger>,
    // M10p1 community governance policy engine (versioned eligibility + weighting)
    governance_registry: Arc<crate::governance::GovernanceRegistry>,
    // M10p2 agent capability tokens (scoped read/write + atomic revocation)
    capability_registry: Arc<crate::governance::CapabilityRegistry>,
    // Directory-level exclusive lock (persistent mode only).
    // Held for the lifetime of the process; released on Drop when the
    // File handle is closed. Advisory flock prevents two processes from
    // opening the same data directory simultaneously.
    #[allow(dead_code)] // Held for its Drop side effect (flock release).
    lock_file: Option<std::fs::File>,
}

/// The subset of [`TidalDb`] fields that both constructors initialize
/// identically. Produced once by [`TidalDb::shared_defaults`] and destructured
/// into each constructor's struct literal, so the verbatim default initializers
/// live in exactly one place. Field names mirror [`TidalDb`] exactly so the
/// destructure at each call site is a one-to-one move with no renaming.
struct SharedDefaults {
    sessions: dashmap::DashMap<SessionId, Arc<SessionState>>,
    next_session_id: AtomicU64,
    closed_sessions: dashmap::DashMap<SessionId, SessionSnapshot>,
    collection_index: Arc<CollectionIndex>,
    notification_tracker: Arc<notification_tracker::NotificationTracker>,
    load_detector: Arc<crate::load::LoadDetector>,
    backpressure_config: crate::load::BackpressureConfig,
    rate_limiter: Arc<crate::load::RateLimiter>,
    shutdown_sweeper: Arc<AtomicBool>,
    sweeper_thread: std::sync::Mutex<Option<std::thread::JoinHandle<()>>>,
    backup_in_progress: Arc<AtomicBool>,
    receiver_handle: std::sync::Mutex<Option<crate::replication::receiver::SegmentReceiverHandle>>,
    shipper_handle: Option<crate::replication::WalShipperHandle>,
    session_bridge: Arc<crate::replication::SessionReplicationBridge>,
    lock_file: Option<std::fs::File>,
}

impl TidalDb {
    /// Returns a new [`TidalDbBuilder`] with default (ephemeral) configuration.
    #[must_use]
    pub fn builder() -> TidalDbBuilder {
        TidalDbBuilder::new()
    }

    /// Build a default single-shard `SessionReplicationBridge` for single-node deployments.
    ///
    /// Uses an in-process self-loop transport (`ShardId::SINGLE`). In single-node mode
    /// the bridge is never actually used for shipping (no peers), but the seqno tracker
    /// and idempotency store are exercised on replay to prevent duplicate application.
    fn make_single_node_session_bridge() -> Arc<crate::replication::SessionReplicationBridge> {
        use crate::{
            replication::{
                IdempotencyStore, InProcessSessionTransportFactory, SessionReplicationBridge,
                ShardId,
            },
            session::state::SessionSeqNoTracker,
        };
        let shards = [ShardId::SINGLE];
        let mut transports = InProcessSessionTransportFactory::new(&shards).build();
        let transport = transports
            .remove(&ShardId::SINGLE)
            .expect("transport for ShardId::SINGLE always present");
        Arc::new(SessionReplicationBridge::new(
            transport,
            Arc::new(IdempotencyStore::default_capacity()),
            Arc::new(SessionSeqNoTracker::new()),
        ))
    }

    /// Build the control plane and tenant router from ONE shared topology Arc.
    ///
    /// Both `from_config` and `from_parts` previously built the control plane and
    /// the `tenant_router` from two *different* `ClusterTopology` Arcs. In
    /// multi-tenant cluster mode that split is a correctness bug: a topology
    /// update applied through the control plane is invisible to the router that
    /// actually decides where a tenant's signals route, so the two diverge. This
    /// helper constructs a single `topo` Arc, derives one `TenantRouter` from it,
    /// installs that SAME router into the control plane, and returns both — so the
    /// `TidalDb.tenant_router` field and the control plane share one topology by
    /// construction. It also removes the verbatim duplication of the control-plane
    /// wiring across the two constructors.
    fn build_control_plane(
        replication_state: &Arc<crate::replication::state::ReplicationState>,
        shard_id: crate::replication::ShardId,
    ) -> (
        Arc<crate::replication::ControlPlane>,
        Arc<crate::replication::TenantRouter>,
    ) {
        let topo = Arc::new(RwLock::new(
            crate::replication::tenant::ClusterTopology::single(),
        ));
        let router = Arc::new(crate::replication::TenantRouter::new(Arc::clone(&topo)));
        let lag = Arc::new(crate::replication::ReplicationLagGauge::new(
            shard_id,
            Arc::clone(replication_state),
        ));
        let control_plane = Arc::new(crate::replication::ControlPlane::new(
            topo,
            Arc::clone(&router),
            lag,
        ));
        (control_plane, router)
    }

    /// Install the control plane and the node's partition-id label onto the
    /// metrics state, shared by both construction paths.
    ///
    /// `set_control_plane` needs unique ownership of the metrics `Arc` (it mutates
    /// in place), so this runs before any clone is shared. `set_partition_id`
    /// stamps the node's real `shard_id` onto the node-identity Prometheus lines so
    /// cluster nodes do not collide on the default `0` (W13). If the `Arc` was
    /// already shared (never in normal construction, since the install is ordered
    /// first), the control plane is left un-wired and we surface it loudly rather
    /// than silently reporting stale health/lag.
    fn install_metrics_wiring(
        metrics: &mut Arc<MetricsState>,
        control_plane: &Arc<crate::replication::ControlPlane>,
        shard_id: crate::replication::ShardId,
    ) {
        if let Some(m) = Arc::get_mut(metrics) {
            m.set_control_plane(Arc::clone(control_plane));
            m.set_partition_id(shard_id.0);
        } else {
            tracing::error!(
                "control plane install failed: metrics Arc was already shared before \
                 set_control_plane; replication-lag and cluster-health readers are un-wired"
            );
        }
    }

    /// The fields whose initializers are byte-for-byte identical between
    /// `from_config` and `from_parts`.
    ///
    /// Both constructors used to inline these ~18 default initializers verbatim,
    /// so the two struct literals had to be kept in lock-step by hand (DRY tax,
    /// `CODING_GUIDELINES` §9). Centralizing them here means each constructor only
    /// spells out the fields that genuinely differ (opened storage/ledger/WAL,
    /// the in-memory indexes, and the control-plane/replication wiring); the
    /// shared remainder is destructured from one `SharedDefaults::new()` call.
    ///
    /// This deliberately does NOT carry session/text channel state: `from_parts`
    /// installs real text-syncer bundles where `from_config` installs `None`, so
    /// those fields stay at the call site. Everything here is a pure default that
    /// is constructed identically regardless of how the DB was opened.
    fn shared_defaults() -> SharedDefaults {
        SharedDefaults {
            sessions: dashmap::DashMap::new(),
            next_session_id: AtomicU64::new(1),
            closed_sessions: dashmap::DashMap::new(),
            collection_index: Arc::new(CollectionIndex::new()),
            notification_tracker: Arc::new(notification_tracker::NotificationTracker::new()),
            load_detector: Arc::new(crate::load::LoadDetector::new(
                crate::load::DegradationThresholds::default(),
            )),
            backpressure_config: crate::load::BackpressureConfig::default(),
            rate_limiter: Arc::new(crate::load::RateLimiter::new(
                crate::load::RateLimiterConfig::default(),
            )),
            shutdown_sweeper: Arc::new(AtomicBool::new(false)),
            sweeper_thread: std::sync::Mutex::new(None),
            backup_in_progress: Arc::new(AtomicBool::new(false)),
            receiver_handle: std::sync::Mutex::new(None),
            shipper_handle: None,
            session_bridge: Self::make_single_node_session_bridge(),
            lock_file: None,
        }
    }

    /// Construct a `TidalDb` without a schema (M0 compatibility mode).
    ///
    /// `metrics` must be uniquely owned (no other `Arc` clones yet): this
    /// constructor installs the control plane into it via `Arc::get_mut` so
    /// the health and replication-lag readers observe real cluster state. The
    /// optional metrics HTTP server is started by the caller afterward via
    /// [`set_metrics_handle`](Self::set_metrics_handle).
    #[allow(clippy::missing_const_for_fn)]
    pub(crate) fn from_config(config: Config, mut metrics: Arc<MetricsState>) -> Self {
        let mut profile_registry = ProfileRegistry::new();
        // Register builtins even in no-schema mode so health_check and metrics
        // still work. Builtins always validate, so an error here is structurally
        // impossible — but log it rather than silently dropping the Result so a
        // future builtin that fails validation is observable instead of leaving
        // the registry quietly missing profiles.
        if let Err(e) = register_builtins(&mut profile_registry) {
            tracing::error!(
                error = %e,
                "failed to register builtin ranking profiles in no-schema mode; \
                 profile-dependent queries may be unavailable"
            );
        }

        let cohort_registry = Arc::new(crate::cohort::CohortRegistry::new());
        let cohort_resolver = Arc::new(crate::cohort::CohortResolver::new(Arc::clone(
            &cohort_registry,
        )));

        // Build the control plane (single-node) and install it on the metrics
        // state so health and replication-lag readers observe real state. The
        // tenant router shares the control plane's topology Arc (see
        // `build_control_plane`).
        let rep_state = Arc::new(crate::replication::state::ReplicationState::single());
        let shard = crate::replication::ShardId::SINGLE;
        let (control_plane, tenant_router) = Self::build_control_plane(&rep_state, shard);
        Self::install_metrics_wiring(&mut metrics, &control_plane, shard);

        let SharedDefaults {
            sessions,
            next_session_id,
            closed_sessions,
            collection_index,
            notification_tracker,
            load_detector,
            backpressure_config,
            rate_limiter,
            shutdown_sweeper,
            sweeper_thread,
            backup_in_progress,
            receiver_handle,
            shipper_handle,
            session_bridge,
            lock_file,
        } = Self::shared_defaults();

        Self {
            config,
            closed: AtomicBool::new(false),
            metrics,
            #[cfg(feature = "metrics")]
            metrics_handle: None,
            ledger: None,
            storage: None,
            wal: std::sync::Mutex::new(None),
            last_wal_seq: Arc::new(AtomicU64::new(0)),
            shutdown_checkpoint: Arc::new(AtomicBool::new(false)),
            checkpoint_thread: std::sync::Mutex::new(None),
            text_index_unhealthy: Arc::new(AtomicBool::new(false)),
            category_index: BitmapIndex::new("category"),
            format_index: BitmapIndex::new("format"),
            creator_index: BitmapIndex::new("creator"),
            tag_index: BitmapIndex::new("tags"),
            duration_index: RangeIndex::new("duration"),
            created_at_index: RangeIndex::new("created_at"),
            universe: Arc::new(RwLock::new(RoaringBitmap::new())),
            embedding_registry: Arc::new(RwLock::new(EmbeddingSlotRegistry::new())),
            profile_registry,
            schema_def: None,
            creator_items: Arc::new(CreatorItemsBitmap::new()),
            user_state: Arc::new(UserStateIndex::new()),
            hard_negatives: Arc::new(HardNegIndex::new()),
            interaction_ledger: Arc::new(InteractionLedger::new()),
            // Cold-start (no schema): use the shared default so the fallback
            // cannot drift from the schema-aware open branches (see open.rs).
            preference_vectors: Arc::new(PreferenceVectors::new(
                self::open::DEFAULT_PREFERENCE_DIM,
            )),
            text_index: None,
            text_tx: std::sync::Mutex::new(None),
            text_syncer_thread: std::sync::Mutex::new(None),
            creator_text_index: None,
            creator_text_tx: std::sync::Mutex::new(None),
            creator_text_syncer_thread: std::sync::Mutex::new(None),
            text_flush_tx: None,
            creator_text_flush_tx: None,
            sessions,
            next_session_id,
            closed_sessions,
            co_engagement: Arc::new(CoEngagementIndex::new()),
            user_signal_index: Arc::new(UserSignalIndex::new()),
            cohort_registry,
            cohort_resolver,
            cohort_ledger: Arc::new(crate::cohort::CohortSignalLedger::empty()),
            collection_index,
            suggestion_index: Arc::new(crate::query::suggest::SuggestionIndex::new()),
            notification_tracker,
            load_detector,
            backpressure_config,
            rate_limiter,
            shutdown_sweeper,
            sweeper_thread,
            backup_in_progress,
            // obs-REPL-1: share the SAME Arc the control-plane lag gauge was
            // built from (above) instead of constructing a second, divergent
            // ReplicationState. No-schema mode never starts replication today, but
            // a divergent Arc here is a latent failover/lag-reporting landmine.
            // The `from_config_shares_one_replication_state_arc` test pins that
            // this field and the control-plane lag gauge observe one
            // `ReplicationState`.
            replication_state: rep_state,
            receiver_handle,
            shipper_handle,
            session_bridge,
            lock_file,
            tenant_router,
            control_plane,
            membership_registry: Arc::new(crate::governance::MembershipRegistry::new()),
            community_ledger: Arc::new(crate::governance::CommunityLedger::empty()),
            governance_registry: Arc::new(crate::governance::GovernanceRegistry::new()),
            capability_registry: Arc::new(crate::governance::CapabilityRegistry::new()),
        }
    }

    /// Construct a `TidalDb` from all opened components.
    ///
    /// `metrics` must be uniquely owned (no other `Arc` clones yet): this
    /// constructor installs the control plane into it via `Arc::get_mut` so
    /// the health and replication-lag readers observe real cluster state. The
    /// optional metrics HTTP server is started by the caller afterward via
    /// [`set_metrics_handle`](Self::set_metrics_handle).
    #[allow(
        clippy::too_many_arguments,
        clippy::missing_const_for_fn,
        clippy::too_many_lines
    )]
    pub(crate) fn from_parts(
        config: Config,
        mut metrics: Arc<MetricsState>,
        result: OpenResult,
    ) -> Self {
        let OpenResult {
            storage,
            ledger,
            wal,
            last_seq,
            profile_registry,
            category_index,
            format_index,
            creator_index,
            tag_index,
            duration_index,
            created_at_index,
            universe,
            embedding_registry,
            schema_def,
            creator_items,
            user_state,
            hard_negatives,
            interaction_ledger,
            preference_vectors,
            session_events,
        } = result;

        let ledger = Some(ledger);
        let storage = Some(storage);

        // Wrap the (already-restored) preference vectors in an Arc up-front so the
        // periodic checkpoint thread can share the SAME instance the read path uses
        // — restored in open.rs, then checkpointed here so the loss window is bounded.
        let preference_vectors = Arc::new(preference_vectors);

        // M6 co-engagement index: restore from durable storage if available.
        let co_engagement = Arc::new(CoEngagementIndex::new());
        if let Some(ref sb) = storage
            && let Err(e) = co_engagement.restore(sb.items_engine())
        {
            tracing::warn!(
                error = %e,
                "co-engagement restore failed; starting from empty state"
            );
        }
        // M6 user signal index: ephemeral by design; not persisted.
        let user_signal_index = Arc::new(UserSignalIndex::new());

        // M6 cohort engine: initialize from schema and restore from checkpoint.
        let cohort_registry = Arc::new(crate::cohort::CohortRegistry::new());

        // Reload persisted cohort definitions (persistent mode only).
        if let Some(ref sb) = storage {
            let prefix = crate::storage::entity_tag_prefix(
                crate::schema::EntityId::new(0),
                crate::storage::Tag::CohortDef,
            );
            let iter = sb.items_engine().scan_prefix(&prefix);
            for (_, value) in iter.flatten() {
                if let Some(def) = crate::cohort::deserialize_cohort_def(&value) {
                    // Skip if already registered (should not happen on fresh open,
                    // but guards against duplicate keys in storage).
                    if cohort_registry.get(&def.name).is_none() {
                        let _ = cohort_registry.define(def);
                    }
                }
            }
        }

        // M9p2 community memberships: restore lifecycle rows from Tag::Membership.
        let membership_registry = Arc::new(crate::governance::MembershipRegistry::new());
        if let Some(ref sb) = storage {
            let prefix = crate::storage::entity_tag_prefix(
                crate::schema::EntityId::new(0),
                crate::storage::Tag::Membership,
            );
            let iter = sb.items_engine().scan_prefix(&prefix);
            for (_, value) in iter.flatten() {
                if let Some(m) = crate::governance::MembershipRegistry::deserialize_row(&value) {
                    membership_registry.put(m);
                }
            }
        }
        // M9p3 community aggregation ledger: restore per-contributor entries
        // from the Tag::Community checkpoint (it cannot rebuild from the WAL,
        // whose community events carry no contributor identity).
        let community_ledger = Arc::new(crate::governance::CommunityLedger::new(&schema_def));
        if let Some(ref sb) = storage {
            community_ledger.restore(sb.items_engine());

            // Replay durable purge tombstones over the restored aggregate.
            //
            // PRIVACY WINDOW: a tombstone is written BEFORE the in-memory
            // aggregate is rematerialized, but the community checkpoint is only
            // flushed on the lifecycle path. A crash after the purge but before
            // the next checkpoint restores the PRE-purge `Tag::Community`
            // snapshot above and silently resurrects the purged user's data.
            // Scanning every `Tag::PurgeTombstone` row and re-applying it here
            // closes that window: `apply_purge_tombstone` re-removes the
            // contributions and re-establishes the watermark, and is idempotent
            // so a purge that already survived in the checkpoint is a no-op.
            let prefix = crate::storage::entity_tag_prefix(
                crate::schema::EntityId::new(0),
                crate::storage::Tag::PurgeTombstone,
            );
            let mut replayed = 0u64;
            for (_, value) in sb.items_engine().scan_prefix(&prefix).flatten() {
                if let Some(t) = crate::governance::PurgeTombstone::from_bytes(&value) {
                    community_ledger.apply_purge_tombstone(&t);
                    replayed += 1;
                }
            }

            // Replay agent-scope purge tombstones (M10p3). Same crash window as
            // the user tombstones above: a remove-by-agent that crashed before the
            // next community checkpoint must not resurrect the revoked agent's
            // contributions. `apply_agent_purge_tombstone` re-purges by writer and
            // is idempotent, so a purge already reflected in the checkpoint is a
            // no-op.
            let agent_prefix = crate::storage::entity_tag_prefix(
                crate::schema::EntityId::new(0),
                crate::storage::Tag::AgentPurgeTombstone,
            );
            let mut agent_replayed = 0u64;
            for (_, value) in sb.items_engine().scan_prefix(&agent_prefix).flatten() {
                if let Some(t) = crate::governance::AgentPurgeTombstone::from_bytes(&value) {
                    community_ledger.apply_agent_purge_tombstone(&t);
                    agent_replayed += 1;
                }
            }
            if replayed > 0 || agent_replayed > 0 {
                tracing::info!(
                    user_tombstones = replayed,
                    agent_tombstones = agent_replayed,
                    "replayed purge tombstones over restored community aggregate"
                );
            }
        }

        // M10p1 governance policies: restore versioned policies from Tag::GovernancePolicy.
        let governance_registry = Arc::new(crate::governance::GovernanceRegistry::new());
        if let Some(ref sb) = storage {
            let prefix = crate::storage::entity_tag_prefix(
                crate::schema::EntityId::new(0),
                crate::storage::Tag::GovernancePolicy,
            );
            for (_, value) in sb.items_engine().scan_prefix(&prefix).flatten() {
                if let Some(p) = crate::governance::GovernancePolicy::from_bytes(&value) {
                    governance_registry.insert_restored(p);
                }
            }
        }

        // M10p2 capability tokens: restore from Tag::Capability (revocation state
        // is serialized inline in each token).
        let capability_registry = Arc::new(crate::governance::CapabilityRegistry::new());
        if let Some(ref sb) = storage {
            let prefix = crate::storage::entity_tag_prefix(
                crate::schema::EntityId::new(0),
                crate::storage::Tag::Capability,
            );
            for (_, value) in sb.items_engine().scan_prefix(&prefix).flatten() {
                if let Some(t) = crate::governance::CapabilityToken::from_bytes(&value) {
                    capability_registry.insert(t);
                }
            }
        }

        let cohort_resolver = Arc::new(crate::cohort::CohortResolver::new(Arc::clone(
            &cohort_registry,
        )));
        let cohort_ledger = Arc::new(crate::cohort::CohortSignalLedger::new(&schema_def));

        // Restore cohort signal state from durable storage (persistent mode only).
        if let Some(ref sb) = storage {
            match cohort_ledger.restore(sb.items_engine()) {
                Ok(Some(meta)) => {
                    tracing::info!(
                        entries = cohort_ledger.entry_count(),
                        wal_sequence = meta.wal_sequence,
                        "cohort signal ledger restored from checkpoint"
                    );
                }
                Ok(None) => {
                    // First boot, no cohort checkpoint yet, OR a corrupt checkpoint
                    // that restore() already escalated at error! and chose to start
                    // empty for (cohort state is a re-accumulating derived aggregate;
                    // see CohortSignalLedger::restore's corruption policy). Nothing
                    // more to do here.
                }
                Err(e) => {
                    // Genuine I/O failure reading the checkpoint (NOT corruption,
                    // which restore() handles internally). Start empty and surface
                    // it loudly so the lost cohort state is observable.
                    tracing::error!(
                        error = %e,
                        "cohort signal ledger restore hit a storage I/O error; \
                         starting from empty cohort state"
                    );
                }
            }
        }

        // ── Text-index open + crash-recovery rebuild + syncer spawn ─────────
        //
        // BLOCKER 7 deadlock fix: the syncer thread holds the Tantivy writer lock
        // for its ENTIRE lifetime (single-writer design). The rebuild-from-store
        // ALSO needs that lock (`rebuild_from` -> `writer_guard()`). If we spawned
        // the syncer first, the rebuild would block forever and open would hang.
        //
        // So open is split into three steps with the writer lock free for the
        // middle one:
        //   1. open the item + creator indexes and create their channels
        //      (`open_text_syncer`), NO thread spawned yet;
        //   2. run `rebuild_text_indexes_at_open` on the freshly-opened indexes
        //      while no thread holds the writer lock — this also folds the
        //      suggestion-index rebuild into the single item scan, and runs the
        //      suggestion rebuild even when there is no item text index;
        //   3. spawn both syncer threads (`.spawn()`); each now grabs the writer
        //      lock for life, which is fine because the rebuild already finished.
        let text_pending = open_text_syncer(
            schema_def.text_fields(),
            &config,
            "text_index",
            "tidaldb-text-syncer",
        );
        let creator_text_pending = open_text_syncer(
            schema_def.creator_text_fields(),
            &config,
            "creator_text_index",
            "tidaldb-creator-text-syncer",
        );

        // The suggestion index is rebuilt in the SAME item scan as the item text
        // index, so it must exist before the rebuild and be moved (not recreated)
        // into the struct below.
        let suggestion_index = Arc::new(crate::query::suggest::SuggestionIndex::new());

        // Step 2: rebuild from the durable store BEFORE any syncer thread takes
        // the writer lock (persistent mode only; ephemeral storage starts empty,
        // so the existing guard inside the rebuild makes this a no-op there).
        // The suggestion-index rebuild runs regardless of whether an item text
        // index exists.
        if let Some(ref sb) = storage {
            crate::db::state_rebuild::rebuild_text_indexes_at_open(
                sb,
                text_pending.index().map(|a| a.as_ref()),
                creator_text_pending.index().map(|a| a.as_ref()),
                &suggestion_index,
            );
        }

        // Step 3: now that the rebuild is done, spawn both syncer threads. The
        // syncer grabbing the writer lock after the rebuild is correct.
        let text_bundle = text_pending.spawn();
        let creator_text_bundle = creator_text_pending.spawn();

        // Wrap the embedding registry early so the checkpoint thread can share it.
        let embedding_registry = Arc::new(RwLock::new(embedding_registry));

        // Rebuild the in-memory ANN index from durable EMB: keys (persistent
        // mode only; ephemeral storage starts empty so this is a no-op).
        //
        // The entity store is the source of truth for full-precision embeddings,
        // but the HNSW/brute-force index is DERIVED state that does not survive a
        // process restart. Without this rebuild, vector SEARCH silently returns
        // nothing after every reopen until each entity's embedding is written
        // again. Rebuild once per entity kind from that kind's own engine,
        // holding the registry write lock for the duration so no concurrent
        // search observes a half-populated index.
        if let Some(ref sb) = storage {
            match embedding_registry.write() {
                Ok(mut registry) => {
                    for (kind, engine) in [
                        (EntityKind::Item, sb.items_engine()),
                        (EntityKind::User, sb.users_engine()),
                        (EntityKind::Creator, sb.creators_engine()),
                    ] {
                        if let Err(e) = registry.rebuild_from_store(kind, engine, &schema_def) {
                            // A failed rebuild leaves ANN search degraded for this
                            // kind, but the durable embeddings are intact and the
                            // next write re-inserts them — surface it, do not abort
                            // the whole open.
                            tracing::error!(
                                entity_kind = %kind,
                                error = %e,
                                "embedding index rebuild failed; vector search for this \
                                 kind is degraded until embeddings are rewritten"
                            );
                        }
                    }
                }
                Err(_) => {
                    tracing::error!(
                        "embedding_registry lock poisoned during open; vector search \
                         starts with an empty in-memory index until embeddings are rewritten"
                    );
                }
            }
        }

        // ── obs-REPL-1: install the control plane BEFORE any metrics clone ──
        // The control plane must be installed into `metrics` while the Arc is
        // still uniquely owned. The checkpoint thread below clones `metrics`
        // (metrics_clone + metrics_supervisor), so installing afterward made
        // `Arc::get_mut` return `None` and the install was silently swallowed —
        // leaving `tidaldb_replication_lag_seqno` and cluster-health readers
        // permanently un-wired in every persistent deployment. Order it here so
        // the install always succeeds.
        //
        // Compute the replication state first (the lag gauge shares it; the
        // receiver advances it as it applies segments, so
        // `ControlPlane::lag_for(SINGLE)` reflects true follower lag rather than
        // staying 0).
        // BLOCKER 6: restore the per-shard applied-seqno high-water-mark from its
        // durable checkpoint so a restarted follower keeps its idempotency
        // boundary instead of resetting to 0 (which would re-apply and
        // double-count every re-shipped segment). On first boot / no checkpoint /
        // ephemeral mode this restores all shards at 0, identical to before.
        // This is the SINGLE Arc the receiver advances, the checkpoint thread
        // persists, and the lag gauge reads (obs-REPL-1: one Arc, never two).
        let replication_state = {
            let mut shards: Vec<crate::replication::ShardId> = vec![config.cluster.shard_id];
            shards.extend_from_slice(&config.cluster.peer_shards);
            let restored = storage.as_ref().map_or_else(
                || crate::replication::state::ReplicationState::new(&shards),
                |sb| state_rebuild::restore_replication_state(sb.items_engine(), &shards),
            );
            Arc::new(restored)
        };
        // The tenant router shares the control plane's topology Arc (see
        // `build_control_plane`) so a topology update is never invisible to the
        // router that decides where a tenant's signals route.
        let (control_plane, tenant_router) =
            Self::build_control_plane(&replication_state, config.cluster.shard_id);
        Self::install_metrics_wiring(&mut metrics, &control_plane, config.cluster.shard_id);

        // Spawn periodic checkpoint thread (persistent mode only).
        let shutdown_checkpoint = Arc::new(AtomicBool::new(false));
        // Shared text-index liveness latch: the checkpoint thread's 10s health
        // poll sets it, and `health_check` reads it. Created here so both the
        // thread and the `TidalDb` field below alias the same `AtomicBool`.
        let text_index_unhealthy = Arc::new(AtomicBool::new(false));
        let checkpoint_thread = {
            let handle = match (storage.as_ref(), ledger.as_ref()) {
                (Some(StorageBox::Fjall(f)), Some(ledger_arc)) => {
                    let items = Box::new(f.backend(EntityKind::Item).clone())
                        as Box<dyn StorageEngine + Send + Sync>;
                    let shutdown = Arc::clone(&shutdown_checkpoint);
                    let ledger_clone = Arc::clone(ledger_arc);
                    let cohort_clone = Arc::clone(&cohort_ledger);
                    let community_clone = Arc::clone(&community_ledger);
                    let co_engagement_clone = Arc::clone(&co_engagement);
                    let preference_clone = Arc::clone(&preference_vectors);
                    // Share the SAME replication_state Arc the receiver advances
                    // and the lag gauge reads, so the persisted high-water-mark is
                    // always this node's real applied seqno (BLOCKER 6 / obs-REPL-1).
                    let replication_state_clone = Arc::clone(&replication_state);
                    let seq_clone = Arc::clone(&last_seq);
                    // Resolve through the single source of truth so an explicit
                    // wal_dir override is honored here exactly as at WAL open.
                    let wal_dir = config.resolved_wal_dir();
                    let metrics_clone = Arc::clone(&metrics);
                    // Separate handle kept by the panic supervisor so it can
                    // flag a dead checkpoint thread even though `metrics_clone`
                    // is moved into `run_checkpoint_thread` (CONCURRENCY-2).
                    let metrics_supervisor = Arc::clone(&metrics);

                    let index_handles = state_rebuild::IndexMetricsHandles {
                        text_index: text_bundle.index.clone(),
                        creator_text_index: creator_text_bundle.index.clone(),
                        text_unhealthy: Arc::clone(&text_index_unhealthy),
                        #[cfg(feature = "metrics")]
                        embedding_registry: Arc::clone(&embedding_registry),
                        #[cfg(feature = "metrics")]
                        bitmap_category: category_index.clone(),
                        #[cfg(feature = "metrics")]
                        bitmap_format: format_index.clone(),
                        #[cfg(feature = "metrics")]
                        bitmap_creator: creator_index.clone(),
                        #[cfg(feature = "metrics")]
                        bitmap_tag: tag_index.clone(),
                    };

                    std::thread::Builder::new()
                        .name("tidaldb-checkpoint".to_string())
                        .spawn(move || {
                            // Supervise the checkpoint loop: if its body panics,
                            // signal durability silently stops. Catch the panic
                            // and record the thread-died flag so /healthz and
                            // tidaldb_health_ok flip to degraded instead of
                            // reporting "ok" against a dead thread (CONCURRENCY-2).
                            let result =
                                std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
                                    run_checkpoint_thread(
                                        shutdown,
                                        ledger_clone,
                                        cohort_clone,
                                        community_clone,
                                        co_engagement_clone,
                                        preference_clone,
                                        replication_state_clone,
                                        items,
                                        seq_clone,
                                        wal_dir,
                                        metrics_clone,
                                        index_handles,
                                    );
                                }));
                            if result.is_err() {
                                metrics_supervisor
                                    .checkpoint_thread_died
                                    .store(true, std::sync::atomic::Ordering::Release);
                                tracing::error!(
                                    "checkpoint thread panicked; signal durability has \
                                     stopped — health now reports degraded"
                                );
                            }
                        })
                        .ok()
                }
                _ => None,
            };
            std::sync::Mutex::new(handle)
        };

        let SharedDefaults {
            sessions,
            next_session_id,
            closed_sessions,
            collection_index,
            notification_tracker,
            load_detector,
            backpressure_config,
            rate_limiter,
            shutdown_sweeper,
            sweeper_thread,
            backup_in_progress,
            receiver_handle,
            shipper_handle,
            session_bridge,
            lock_file,
        } = Self::shared_defaults();

        let db = Self {
            config,
            closed: AtomicBool::new(false),
            metrics,
            #[cfg(feature = "metrics")]
            metrics_handle: None,
            ledger,
            storage,
            wal: std::sync::Mutex::new(wal),
            last_wal_seq: last_seq,
            shutdown_checkpoint,
            checkpoint_thread,
            text_index_unhealthy,
            category_index,
            format_index,
            creator_index,
            tag_index,
            duration_index,
            created_at_index,
            universe: Arc::new(RwLock::new(universe)),
            embedding_registry,
            profile_registry,
            schema_def: Some(schema_def),
            creator_items: Arc::new(creator_items),
            user_state: Arc::new(user_state),
            hard_negatives: Arc::new(hard_negatives),
            interaction_ledger: Arc::new(interaction_ledger),
            preference_vectors,
            text_index: text_bundle.index,
            text_tx: std::sync::Mutex::new(text_bundle.write_tx),
            text_syncer_thread: text_bundle.thread,
            creator_text_index: creator_text_bundle.index,
            creator_text_tx: std::sync::Mutex::new(creator_text_bundle.write_tx),
            creator_text_syncer_thread: creator_text_bundle.thread,
            text_flush_tx: text_bundle.flush_tx,
            creator_text_flush_tx: creator_text_bundle.flush_tx,
            sessions,
            next_session_id,
            closed_sessions,
            co_engagement,
            user_signal_index,
            cohort_registry,
            cohort_resolver,
            cohort_ledger,
            // Empty here; rebuilt from durable storage just below via
            // `rebuild_collections` (the shared default is the same empty index
            // both constructors start from).
            collection_index,
            suggestion_index,
            notification_tracker,
            load_detector,
            backpressure_config,
            rate_limiter,
            shutdown_sweeper,
            sweeper_thread,
            backup_in_progress,
            replication_state,
            receiver_handle,
            shipper_handle,
            session_bridge,
            lock_file,
            tenant_router,
            control_plane,
            membership_registry,
            community_ledger,
            governance_registry,
            capability_registry,
        };

        // M6p4: rebuild collection index from durable storage.
        //
        // BLOCKER 7: the item AND creator Tantivy text indexes (and the
        // suggestion index, folded into the item scan) are rebuilt from the
        // durable entity stores ABOVE — BEFORE the syncer threads are spawned —
        // so the rebuild's writer-lock acquisition does not deadlock against the
        // syncer, which holds that lock for its whole lifetime. Only the
        // collection index remains to rebuild here; it needs no Tantivy writer
        // lock and is independent of the text syncers.
        if let Some(ref sb) = db.storage {
            crate::db::collections::rebuild_collections(&db.collection_index, sb.items_engine());
        }

        // Restore durable tenant-migration routing so a crash mid-migration does
        // not silently revert a finalized tenant to jump-hash routing.
        db.restore_migration_routing();

        if !session_events.is_empty() {
            db.restore_session_wal_events(&session_events);
        }
        db
    }

    /// Returns the bound address of the metrics HTTP server, if running.
    ///
    /// Useful when port 0 was requested to discover the OS-assigned port.
    /// Returns `None` if the `metrics` feature is disabled or if
    /// `enable_metrics` was not called on the builder.
    #[must_use]
    #[allow(clippy::missing_const_for_fn)] // cfg-gated body prevents const
    pub fn metrics_addr(&self) -> Option<std::net::SocketAddr> {
        #[cfg(feature = "metrics")]
        {
            self.metrics_handle.as_ref().map(|h| h.addr)
        }
        #[cfg(not(feature = "metrics"))]
        {
            None
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::replication::ShardId;

    /// Pins the obs-REPL-1 invariant: in BOTH constructors the
    /// `replication_state` field and the control-plane lag gauge must observe ONE
    /// `ReplicationState`, never two divergent Arcs.
    ///
    /// Earlier `from_config` built the lag gauge from one `ReplicationState::single()`
    /// and the field from a SECOND one, so a receiver advancing the field would
    /// leave the lag gauge reading an orphaned Arc — replication lag stuck forever.
    /// We prove the Arcs are shared *behaviorally* rather than poking private
    /// internals: advancing the field's high-water-mark must be observable through
    /// the gauge. `ReplicationLagGauge::applied_seqno()` reads its own `state` Arc,
    /// so the assertion can only pass when both point at the same allocation
    /// (equivalent to `Arc::ptr_eq`, but without widening the gauge's interface).
    fn assert_shared_replication_state(db: &TidalDb) {
        let gauge = db.control_plane().lag_gauge();
        // Fresh DB: both start at 0.
        assert_eq!(gauge.applied_seqno(), 0, "gauge must start at applied 0");
        assert_eq!(
            db.replication_state().applied_seqno(ShardId::SINGLE),
            Some(0),
            "field must start at applied 0"
        );

        // Advance ONLY the field's high-water-mark (what the receiver does).
        db.replication_state().advance(ShardId::SINGLE, 7);

        // The gauge must observe it. If the field and gauge held divergent Arcs
        // this would still read 0 — the exact stuck-lag bug obs-REPL-1 prevents.
        assert_eq!(
            gauge.applied_seqno(),
            7,
            "lag gauge and replication_state field must share ONE Arc; gauge \
             did not observe the field's advance (divergent-Arc regression)"
        );
        assert_eq!(
            db.replication_state().applied_seqno(ShardId::SINGLE),
            Some(7),
            "field must reflect its own advance"
        );
    }

    #[test]
    fn from_config_shares_one_replication_state_arc() {
        // No-schema (M0 compatibility) open routes through `from_config`.
        let db = TidalDb::builder()
            .ephemeral()
            .open()
            .expect("no-schema ephemeral open should succeed");
        assert_shared_replication_state(&db);
    }

    #[test]
    fn from_parts_shares_one_replication_state_arc() {
        use std::time::Duration;

        use crate::schema::{DecaySpec, EntityKind, SchemaBuilder, Window};

        // Persistent-with-schema open routes through `from_parts`.
        let mut builder = SchemaBuilder::new();
        let _ = builder
            .signal(
                "view",
                EntityKind::Item,
                DecaySpec::Exponential {
                    half_life: Duration::from_secs(3600),
                },
            )
            .windows(&[Window::AllTime])
            .velocity(false)
            .add();
        let schema = builder.build().expect("schema must be valid");

        let dir = tempfile::tempdir().expect("tempdir");
        let db = TidalDb::builder()
            .with_data_dir(dir.path())
            .with_schema(schema)
            .open()
            .expect("persistent open with schema should succeed");
        assert_shared_replication_state(&db);
        db.close().expect("close should succeed");
    }
}
