Adds the M9 purge re-materialization feature: a WAL-replay background
engine that rebuilds community cohort aggregates for a (user, community)
pair after retroactive signal purge, restoring ranking correctness without
modifying the immutable WAL.
Key additions:
- cohort::rematerialization module: PurgeJobQueue, RematerializationEngine,
WAL replay, atomic CohortSignalLedger swap, BLAKE3 audit log, metrics counters
- TidalDb::{submit_purge_job, purge_job_status, rematerialization_metrics}
public API (db/rematerialization.rs)
- Engine auto-starts in persistent mode; clean shutdown before WAL teardown
- 6 integration tests in tests/m9_purge_remat.rs covering ephemeral and
persistent modes, job lifecycle, and multi-job independence
- Split oversized files to comply with 600-line limit: db/mod.rs →
db/from_parts.rs, entities/revocation.rs → revocation/{mod,tests}.rs,
schema/validation/builders.rs → builders/{mod,tests}.rs,
signals/warm.rs → warm/{mod,tests,proptests}.rs
- Fix pre-existing bootstrap errors: export AuditKind from session module,
add overrides_rejected to SessionSnapshot deserialization
337 lines
14 KiB
Rust
337 lines
14 KiB
Rust
//! `TidalDb::from_parts` — construct a fully-opened database instance from
|
|
//! all opened storage components.
|
|
//!
|
|
//! Split from `db/mod.rs` to satisfy the 600-line file-size limit.
|
|
|
|
use std::sync::{Arc, RwLock};
|
|
|
|
use crate::entities::{
|
|
CoEngagementIndex, CollectionIndex, CommunityMembershipIndex, UserSignalIndex,
|
|
};
|
|
use crate::schema::EntityKind;
|
|
use crate::storage::StorageEngine;
|
|
|
|
use super::TidalDb;
|
|
use super::metrics::MetricsState;
|
|
use super::open::OpenResult;
|
|
use super::state_rebuild::run_checkpoint_thread;
|
|
use super::storage_box::StorageBox;
|
|
use super::text_syncer::spawn_text_syncer;
|
|
use super::{config::Config, notification_tracker};
|
|
|
|
#[cfg(feature = "metrics")]
|
|
use super::http;
|
|
|
|
impl TidalDb {
|
|
/// Construct a `TidalDb` from all opened components.
|
|
#[allow(
|
|
clippy::too_many_arguments,
|
|
clippy::missing_const_for_fn,
|
|
clippy::too_many_lines
|
|
)]
|
|
pub(crate) fn from_parts(
|
|
config: Config,
|
|
metrics: Arc<MetricsState>,
|
|
#[cfg(feature = "metrics")] metrics_handle: Option<http::MetricsHandle>,
|
|
result: OpenResult,
|
|
) -> Self {
|
|
let OpenResult {
|
|
storage,
|
|
ledger,
|
|
wal,
|
|
last_seq,
|
|
profile_registry,
|
|
category_index,
|
|
format_index,
|
|
creator_index,
|
|
tag_index,
|
|
duration_index,
|
|
created_at_index,
|
|
universe,
|
|
embedding_registry,
|
|
schema_def,
|
|
creator_items,
|
|
user_state,
|
|
hard_negatives,
|
|
interaction_ledger,
|
|
preference_vectors,
|
|
session_events,
|
|
} = result;
|
|
|
|
let ledger = Some(ledger);
|
|
let storage = Some(storage);
|
|
|
|
// M6 co-engagement index: restore from durable storage if available.
|
|
let co_engagement = Arc::new(CoEngagementIndex::new());
|
|
if let Some(ref sb) = storage
|
|
&& let Err(e) = co_engagement.restore(sb.items_engine())
|
|
{
|
|
tracing::warn!(
|
|
error = %e,
|
|
"co-engagement restore failed; starting from empty state"
|
|
);
|
|
}
|
|
// M6 user signal index: ephemeral by design; not persisted.
|
|
let user_signal_index = Arc::new(UserSignalIndex::new());
|
|
|
|
// M6 cohort engine: initialize from schema and restore from checkpoint.
|
|
let cohort_registry = Arc::new(crate::cohort::CohortRegistry::new());
|
|
|
|
// Reload persisted cohort definitions (persistent mode only).
|
|
if let Some(ref sb) = storage {
|
|
let prefix = crate::storage::entity_tag_prefix(
|
|
crate::schema::EntityId::new(0),
|
|
crate::storage::Tag::CohortDef,
|
|
);
|
|
let iter = sb.items_engine().scan_prefix(&prefix);
|
|
for (_, value) in iter.flatten() {
|
|
if let Some(def) = crate::cohort::deserialize_cohort_def(&value)
|
|
&& cohort_registry.get(&def.name).is_none()
|
|
{
|
|
let _ = cohort_registry.define(def);
|
|
}
|
|
}
|
|
}
|
|
|
|
let cohort_resolver = Arc::new(crate::cohort::CohortResolver::new(Arc::clone(
|
|
&cohort_registry,
|
|
)));
|
|
let cohort_ledger = Arc::new(crate::cohort::CohortSignalLedger::new(&schema_def));
|
|
let contribution_log = Arc::new(crate::cohort::CohortContributionLog::new());
|
|
|
|
// Restore cohort signal state from durable storage (persistent mode only).
|
|
if let Some(ref sb) = storage {
|
|
match cohort_ledger.restore(sb.items_engine()) {
|
|
Ok(Some(meta)) => {
|
|
tracing::info!(
|
|
entries = cohort_ledger.entry_count(),
|
|
wal_sequence = meta.wal_sequence,
|
|
"cohort signal ledger restored from checkpoint"
|
|
);
|
|
}
|
|
Ok(None) => {}
|
|
Err(e) => {
|
|
tracing::warn!(
|
|
error = %e,
|
|
"cohort signal ledger restore failed; starting from empty state"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
let text_bundle = spawn_text_syncer(
|
|
schema_def.text_fields(),
|
|
&config,
|
|
"text_index",
|
|
"tidaldb-text-syncer",
|
|
);
|
|
let creator_text_bundle = spawn_text_syncer(
|
|
schema_def.creator_text_fields(),
|
|
&config,
|
|
"creator_text_index",
|
|
"tidaldb-creator-text-syncer",
|
|
);
|
|
|
|
// Wrap the embedding registry early so the checkpoint thread can share it.
|
|
let embedding_registry = Arc::new(RwLock::new(embedding_registry));
|
|
|
|
// Spawn periodic checkpoint thread (persistent mode only).
|
|
let shutdown_checkpoint = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
|
let checkpoint_thread = {
|
|
let handle = match (storage.as_ref(), ledger.as_ref()) {
|
|
(Some(StorageBox::Fjall(f)), Some(ledger_arc)) => {
|
|
let items = Box::new(f.backend(EntityKind::Item).clone())
|
|
as Box<dyn StorageEngine + Send + Sync>;
|
|
let shutdown = Arc::clone(&shutdown_checkpoint);
|
|
let ledger_clone = Arc::clone(ledger_arc);
|
|
let cohort_clone = Arc::clone(&cohort_ledger);
|
|
let seq_clone = Arc::clone(&last_seq);
|
|
let wal_dir = config.data_dir.as_ref().map(|d| d.join("wal"));
|
|
let metrics_clone = Arc::clone(&metrics);
|
|
|
|
#[cfg(feature = "metrics")]
|
|
let index_handles = super::state_rebuild::IndexMetricsHandles {
|
|
text_index: text_bundle.index.clone(),
|
|
embedding_registry: Arc::clone(&embedding_registry),
|
|
bitmap_category: category_index.clone(),
|
|
bitmap_format: format_index.clone(),
|
|
bitmap_creator: creator_index.clone(),
|
|
bitmap_tag: tag_index.clone(),
|
|
};
|
|
#[cfg(not(feature = "metrics"))]
|
|
let index_handles = super::state_rebuild::IndexMetricsHandles;
|
|
|
|
std::thread::Builder::new()
|
|
.name("tidaldb-checkpoint".to_string())
|
|
.spawn(move || {
|
|
run_checkpoint_thread(
|
|
shutdown,
|
|
ledger_clone,
|
|
cohort_clone,
|
|
items,
|
|
seq_clone,
|
|
wal_dir,
|
|
metrics_clone,
|
|
index_handles,
|
|
);
|
|
})
|
|
.ok()
|
|
}
|
|
_ => None,
|
|
};
|
|
std::sync::Mutex::new(handle)
|
|
};
|
|
|
|
// Compute the replication state before moving config into the struct.
|
|
let replication_state = {
|
|
let mut shards: Vec<crate::replication::ShardId> = vec![config.cluster.shard_id];
|
|
shards.extend_from_slice(&config.cluster.peer_shards);
|
|
Arc::new(crate::replication::state::ReplicationState::new(&shards))
|
|
};
|
|
|
|
let db = Self {
|
|
config,
|
|
closed: std::sync::atomic::AtomicBool::new(false),
|
|
metrics,
|
|
#[cfg(feature = "metrics")]
|
|
metrics_handle,
|
|
ledger,
|
|
storage,
|
|
wal: std::sync::Mutex::new(wal),
|
|
last_wal_seq: last_seq,
|
|
shutdown_checkpoint,
|
|
checkpoint_thread,
|
|
category_index,
|
|
format_index,
|
|
creator_index,
|
|
tag_index,
|
|
duration_index,
|
|
created_at_index,
|
|
universe: Arc::new(RwLock::new(universe)),
|
|
embedding_registry,
|
|
profile_registry,
|
|
schema_def: Some(schema_def),
|
|
creator_items: Arc::new(creator_items),
|
|
user_state: Arc::new(user_state),
|
|
hard_negatives: Arc::new(hard_negatives),
|
|
interaction_ledger: Arc::new(interaction_ledger),
|
|
preference_vectors: Arc::new(preference_vectors),
|
|
text_index: text_bundle.index,
|
|
text_tx: std::sync::Mutex::new(text_bundle.write_tx),
|
|
text_syncer_thread: text_bundle.thread,
|
|
creator_text_index: creator_text_bundle.index,
|
|
creator_text_tx: std::sync::Mutex::new(creator_text_bundle.write_tx),
|
|
creator_text_syncer_thread: creator_text_bundle.thread,
|
|
text_flush_tx: text_bundle.flush_tx,
|
|
creator_text_flush_tx: creator_text_bundle.flush_tx,
|
|
sessions: dashmap::DashMap::new(),
|
|
next_session_id: std::sync::atomic::AtomicU64::new(1),
|
|
closed_sessions: dashmap::DashMap::new(),
|
|
co_engagement,
|
|
user_signal_index,
|
|
cohort_registry,
|
|
cohort_resolver,
|
|
contribution_log: Arc::clone(&contribution_log),
|
|
purge_coordinator: Arc::new(crate::cohort::PurgeCoordinator::new(
|
|
Arc::clone(&contribution_log),
|
|
Arc::clone(&cohort_ledger),
|
|
)),
|
|
cohort_ledger,
|
|
purge_job_queue: Arc::new(std::sync::Mutex::new(
|
|
crate::cohort::rematerialization::PurgeJobQueue::new(),
|
|
)),
|
|
rematerialization_metrics:
|
|
crate::cohort::rematerialization::RematerializationMetrics::new(),
|
|
rematerialization_handle: std::sync::Mutex::new(None),
|
|
revocation_index: Arc::new(crate::entities::RevocationIndex::new()),
|
|
collection_index: Arc::new(CollectionIndex::new()),
|
|
community_membership: Arc::new(CommunityMembershipIndex::new()),
|
|
community_leave_status: dashmap::DashMap::new(),
|
|
suggestion_index: Arc::new(crate::query::suggest::SuggestionIndex::new()),
|
|
notification_tracker: Arc::new(notification_tracker::NotificationTracker::new()),
|
|
load_detector: Arc::new(crate::load::LoadDetector::new(
|
|
crate::load::DegradationThresholds::default(),
|
|
)),
|
|
backpressure_config: crate::load::BackpressureConfig::default(),
|
|
rate_limiter: Arc::new(crate::load::RateLimiter::new(
|
|
crate::load::RateLimiterConfig::default(),
|
|
)),
|
|
shutdown_sweeper: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
|
sweeper_thread: std::sync::Mutex::new(None),
|
|
backup_in_progress: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
|
replication_state,
|
|
receiver_handle: std::sync::Mutex::new(None),
|
|
shipper_handle: None,
|
|
session_bridge: Self::make_single_node_session_bridge(),
|
|
lock_file: None,
|
|
tenant_router: {
|
|
let topo = Arc::new(RwLock::new(
|
|
crate::replication::tenant::ClusterTopology::single(),
|
|
));
|
|
Arc::new(crate::replication::TenantRouter::new(topo))
|
|
},
|
|
control_plane: {
|
|
let topo = Arc::new(RwLock::new(
|
|
crate::replication::tenant::ClusterTopology::single(),
|
|
));
|
|
let router = Arc::new(crate::replication::TenantRouter::new(Arc::clone(&topo)));
|
|
let rep_state = Arc::new(crate::replication::state::ReplicationState::single());
|
|
let lag = Arc::new(crate::replication::ReplicationLagGauge::new(
|
|
crate::replication::ShardId::SINGLE,
|
|
rep_state,
|
|
));
|
|
Arc::new(crate::replication::ControlPlane::new(topo, router, lag))
|
|
},
|
|
};
|
|
|
|
// M6p4: rebuild collection index from durable storage.
|
|
// M6p5: rebuild suggestion index title terms from durable item metadata.
|
|
// M9: rebuild community membership index from durable users storage.
|
|
// M10: rebuild signal revocation index from durable users storage.
|
|
if let Some(ref sb) = db.storage {
|
|
crate::db::collections::rebuild_collections(&db.collection_index, sb.items_engine());
|
|
crate::db::state_rebuild::rebuild_suggestion_index(sb, &db.suggestion_index);
|
|
crate::db::state_rebuild::rebuild_community_memberships(sb, &db.community_membership);
|
|
crate::db::state_rebuild::rebuild_community_leave_status(
|
|
sb,
|
|
&db.community_leave_status,
|
|
);
|
|
crate::db::revocation::rebuild_revocations(&db.revocation_index, sb.users_engine());
|
|
}
|
|
|
|
if !session_events.is_empty() {
|
|
db.restore_session_wal_events(&session_events);
|
|
}
|
|
|
|
// M9: Start re-materialization background worker (persistent mode only).
|
|
if let Some(ref data_dir) = db.config.data_dir {
|
|
let wal_dir = data_dir.join("wal");
|
|
let audit_log_path = data_dir.join("purge_audit.log");
|
|
|
|
if let Ok(queue) = db.purge_job_queue.lock() {
|
|
queue.requeue_interrupted();
|
|
}
|
|
|
|
let engine_config = crate::cohort::rematerialization::RematerializationConfig {
|
|
cohort_ledger: Arc::clone(&db.cohort_ledger),
|
|
contribution_log: Arc::clone(&db.contribution_log),
|
|
schema: db.schema_def.clone().unwrap_or_else(|| {
|
|
crate::schema::SchemaBuilder::new()
|
|
.build()
|
|
.expect("empty schema")
|
|
}),
|
|
wal_dir,
|
|
audit_log_path,
|
|
job_queue: Arc::clone(&db.purge_job_queue),
|
|
metrics: Arc::clone(&db.rematerialization_metrics),
|
|
};
|
|
let handle = crate::cohort::rematerialization::start(engine_config);
|
|
if let Ok(mut guard) = db.rematerialization_handle.lock() {
|
|
*guard = Some(handle);
|
|
}
|
|
}
|
|
|
|
db
|
|
}
|
|
}
|