//! `TidalDb::from_parts` — construct a fully-opened database instance from //! all opened storage components. //! //! Split from `db/mod.rs` to satisfy the 600-line file-size limit. use std::sync::{Arc, RwLock}; use crate::entities::{ CoEngagementIndex, CollectionIndex, CommunityMembershipIndex, UserSignalIndex, }; use crate::schema::EntityKind; use crate::storage::StorageEngine; use super::TidalDb; use super::metrics::MetricsState; use super::open::OpenResult; use super::state_rebuild::run_checkpoint_thread; use super::storage_box::StorageBox; use super::text_syncer::spawn_text_syncer; use super::{config::Config, notification_tracker}; #[cfg(feature = "metrics")] use super::http; impl TidalDb { /// Construct a `TidalDb` from all opened components. #[allow( clippy::too_many_arguments, clippy::missing_const_for_fn, clippy::too_many_lines )] pub(crate) fn from_parts( config: Config, metrics: Arc, #[cfg(feature = "metrics")] metrics_handle: Option, result: OpenResult, ) -> Self { let OpenResult { storage, ledger, wal, last_seq, profile_registry, category_index, format_index, creator_index, tag_index, duration_index, created_at_index, universe, embedding_registry, schema_def, creator_items, user_state, hard_negatives, interaction_ledger, preference_vectors, session_events, } = result; let ledger = Some(ledger); let storage = Some(storage); // M6 co-engagement index: restore from durable storage if available. let co_engagement = Arc::new(CoEngagementIndex::new()); if let Some(ref sb) = storage && let Err(e) = co_engagement.restore(sb.items_engine()) { tracing::warn!( error = %e, "co-engagement restore failed; starting from empty state" ); } // M6 user signal index: ephemeral by design; not persisted. let user_signal_index = Arc::new(UserSignalIndex::new()); // M6 cohort engine: initialize from schema and restore from checkpoint. let cohort_registry = Arc::new(crate::cohort::CohortRegistry::new()); // Reload persisted cohort definitions (persistent mode only). if let Some(ref sb) = storage { let prefix = crate::storage::entity_tag_prefix( crate::schema::EntityId::new(0), crate::storage::Tag::CohortDef, ); let iter = sb.items_engine().scan_prefix(&prefix); for (_, value) in iter.flatten() { if let Some(def) = crate::cohort::deserialize_cohort_def(&value) && cohort_registry.get(&def.name).is_none() { let _ = cohort_registry.define(def); } } } let cohort_resolver = Arc::new(crate::cohort::CohortResolver::new(Arc::clone( &cohort_registry, ))); let cohort_ledger = Arc::new(crate::cohort::CohortSignalLedger::new(&schema_def)); let contribution_log = Arc::new(crate::cohort::CohortContributionLog::new()); // Restore cohort signal state from durable storage (persistent mode only). if let Some(ref sb) = storage { match cohort_ledger.restore(sb.items_engine()) { Ok(Some(meta)) => { tracing::info!( entries = cohort_ledger.entry_count(), wal_sequence = meta.wal_sequence, "cohort signal ledger restored from checkpoint" ); } Ok(None) => {} Err(e) => { tracing::warn!( error = %e, "cohort signal ledger restore failed; starting from empty state" ); } } } let text_bundle = spawn_text_syncer( schema_def.text_fields(), &config, "text_index", "tidaldb-text-syncer", ); let creator_text_bundle = spawn_text_syncer( schema_def.creator_text_fields(), &config, "creator_text_index", "tidaldb-creator-text-syncer", ); // Wrap the embedding registry early so the checkpoint thread can share it. let embedding_registry = Arc::new(RwLock::new(embedding_registry)); // Spawn periodic checkpoint thread (persistent mode only). let shutdown_checkpoint = Arc::new(std::sync::atomic::AtomicBool::new(false)); let checkpoint_thread = { let handle = match (storage.as_ref(), ledger.as_ref()) { (Some(StorageBox::Fjall(f)), Some(ledger_arc)) => { let items = Box::new(f.backend(EntityKind::Item).clone()) as Box; let shutdown = Arc::clone(&shutdown_checkpoint); let ledger_clone = Arc::clone(ledger_arc); let cohort_clone = Arc::clone(&cohort_ledger); let seq_clone = Arc::clone(&last_seq); let wal_dir = config.data_dir.as_ref().map(|d| d.join("wal")); let metrics_clone = Arc::clone(&metrics); #[cfg(feature = "metrics")] let index_handles = super::state_rebuild::IndexMetricsHandles { text_index: text_bundle.index.clone(), embedding_registry: Arc::clone(&embedding_registry), bitmap_category: category_index.clone(), bitmap_format: format_index.clone(), bitmap_creator: creator_index.clone(), bitmap_tag: tag_index.clone(), }; #[cfg(not(feature = "metrics"))] let index_handles = super::state_rebuild::IndexMetricsHandles; std::thread::Builder::new() .name("tidaldb-checkpoint".to_string()) .spawn(move || { run_checkpoint_thread( shutdown, ledger_clone, cohort_clone, items, seq_clone, wal_dir, metrics_clone, index_handles, ); }) .ok() } _ => None, }; std::sync::Mutex::new(handle) }; // Compute the replication state before moving config into the struct. let replication_state = { let mut shards: Vec = vec![config.cluster.shard_id]; shards.extend_from_slice(&config.cluster.peer_shards); Arc::new(crate::replication::state::ReplicationState::new(&shards)) }; let db = Self { config, closed: std::sync::atomic::AtomicBool::new(false), metrics, #[cfg(feature = "metrics")] metrics_handle, ledger, storage, wal: std::sync::Mutex::new(wal), last_wal_seq: last_seq, shutdown_checkpoint, checkpoint_thread, category_index, format_index, creator_index, tag_index, duration_index, created_at_index, universe: Arc::new(RwLock::new(universe)), embedding_registry, profile_registry, schema_def: Some(schema_def), creator_items: Arc::new(creator_items), user_state: Arc::new(user_state), hard_negatives: Arc::new(hard_negatives), interaction_ledger: Arc::new(interaction_ledger), preference_vectors: Arc::new(preference_vectors), text_index: text_bundle.index, text_tx: std::sync::Mutex::new(text_bundle.write_tx), text_syncer_thread: text_bundle.thread, creator_text_index: creator_text_bundle.index, creator_text_tx: std::sync::Mutex::new(creator_text_bundle.write_tx), creator_text_syncer_thread: creator_text_bundle.thread, text_flush_tx: text_bundle.flush_tx, creator_text_flush_tx: creator_text_bundle.flush_tx, sessions: dashmap::DashMap::new(), next_session_id: std::sync::atomic::AtomicU64::new(1), closed_sessions: dashmap::DashMap::new(), co_engagement, user_signal_index, cohort_registry, cohort_resolver, contribution_log: Arc::clone(&contribution_log), purge_coordinator: Arc::new(crate::cohort::PurgeCoordinator::new( Arc::clone(&contribution_log), Arc::clone(&cohort_ledger), )), cohort_ledger, purge_job_queue: Arc::new(std::sync::Mutex::new( crate::cohort::rematerialization::PurgeJobQueue::new(), )), rematerialization_metrics: crate::cohort::rematerialization::RematerializationMetrics::new(), rematerialization_handle: std::sync::Mutex::new(None), revocation_index: Arc::new(crate::entities::RevocationIndex::new()), collection_index: Arc::new(CollectionIndex::new()), community_membership: Arc::new(CommunityMembershipIndex::new()), community_leave_status: dashmap::DashMap::new(), suggestion_index: Arc::new(crate::query::suggest::SuggestionIndex::new()), notification_tracker: Arc::new(notification_tracker::NotificationTracker::new()), load_detector: Arc::new(crate::load::LoadDetector::new( crate::load::DegradationThresholds::default(), )), backpressure_config: crate::load::BackpressureConfig::default(), rate_limiter: Arc::new(crate::load::RateLimiter::new( crate::load::RateLimiterConfig::default(), )), shutdown_sweeper: Arc::new(std::sync::atomic::AtomicBool::new(false)), sweeper_thread: std::sync::Mutex::new(None), backup_in_progress: Arc::new(std::sync::atomic::AtomicBool::new(false)), replication_state, receiver_handle: std::sync::Mutex::new(None), shipper_handle: None, session_bridge: Self::make_single_node_session_bridge(), lock_file: None, tenant_router: { let topo = Arc::new(RwLock::new( crate::replication::tenant::ClusterTopology::single(), )); Arc::new(crate::replication::TenantRouter::new(topo)) }, control_plane: { let topo = Arc::new(RwLock::new( crate::replication::tenant::ClusterTopology::single(), )); let router = Arc::new(crate::replication::TenantRouter::new(Arc::clone(&topo))); let rep_state = Arc::new(crate::replication::state::ReplicationState::single()); let lag = Arc::new(crate::replication::ReplicationLagGauge::new( crate::replication::ShardId::SINGLE, rep_state, )); Arc::new(crate::replication::ControlPlane::new(topo, router, lag)) }, }; // M6p4: rebuild collection index from durable storage. // M6p5: rebuild suggestion index title terms from durable item metadata. // M9: rebuild community membership index from durable users storage. // M10: rebuild signal revocation index from durable users storage. if let Some(ref sb) = db.storage { crate::db::collections::rebuild_collections(&db.collection_index, sb.items_engine()); crate::db::state_rebuild::rebuild_suggestion_index(sb, &db.suggestion_index); crate::db::state_rebuild::rebuild_community_memberships(sb, &db.community_membership); crate::db::state_rebuild::rebuild_community_leave_status( sb, &db.community_leave_status, ); crate::db::revocation::rebuild_revocations(&db.revocation_index, sb.users_engine()); } if !session_events.is_empty() { db.restore_session_wal_events(&session_events); } // M9: Start re-materialization background worker (persistent mode only). if let Some(ref data_dir) = db.config.data_dir { let wal_dir = data_dir.join("wal"); let audit_log_path = data_dir.join("purge_audit.log"); if let Ok(queue) = db.purge_job_queue.lock() { queue.requeue_interrupted(); } let engine_config = crate::cohort::rematerialization::RematerializationConfig { cohort_ledger: Arc::clone(&db.cohort_ledger), contribution_log: Arc::clone(&db.contribution_log), schema: db.schema_def.clone().unwrap_or_else(|| { crate::schema::SchemaBuilder::new() .build() .expect("empty schema") }), wal_dir, audit_log_path, job_queue: Arc::clone(&db.purge_job_queue), metrics: Arc::clone(&db.rematerialization_metrics), }; let handle = crate::cohort::rematerialization::start(engine_config); if let Ok(mut guard) = db.rematerialization_handle.lock() { *guard = Some(handle); } } db } }