tidaldb/tidal/src/db/from_parts.rs
jordan 6f26d03c77 feat(m9): implement purge re-materialization engine
Adds the M9 purge re-materialization feature: a WAL-replay background
engine that rebuilds community cohort aggregates for a (user, community)
pair after retroactive signal purge, restoring ranking correctness without
modifying the immutable WAL.

Key additions:
- cohort::rematerialization module: PurgeJobQueue, RematerializationEngine,
  WAL replay, atomic CohortSignalLedger swap, BLAKE3 audit log, metrics counters
- TidalDb::{submit_purge_job, purge_job_status, rematerialization_metrics}
  public API (db/rematerialization.rs)
- Engine auto-starts in persistent mode; clean shutdown before WAL teardown
- 6 integration tests in tests/m9_purge_remat.rs covering ephemeral and
  persistent modes, job lifecycle, and multi-job independence
- Split oversized files to comply with 600-line limit: db/mod.rs →
  db/from_parts.rs, entities/revocation.rs → revocation/{mod,tests}.rs,
  schema/validation/builders.rs → builders/{mod,tests}.rs,
  signals/warm.rs → warm/{mod,tests,proptests}.rs
- Fix pre-existing bootstrap errors: export AuditKind from session module,
  add overrides_rejected to SessionSnapshot deserialization
2026-03-03 19:18:16 -07:00

337 lines
14 KiB
Rust

//! `TidalDb::from_parts` — construct a fully-opened database instance from
//! all opened storage components.
//!
//! Split from `db/mod.rs` to satisfy the 600-line file-size limit.
use std::sync::{Arc, RwLock};
use crate::entities::{
CoEngagementIndex, CollectionIndex, CommunityMembershipIndex, UserSignalIndex,
};
use crate::schema::EntityKind;
use crate::storage::StorageEngine;
use super::TidalDb;
use super::metrics::MetricsState;
use super::open::OpenResult;
use super::state_rebuild::run_checkpoint_thread;
use super::storage_box::StorageBox;
use super::text_syncer::spawn_text_syncer;
use super::{config::Config, notification_tracker};
#[cfg(feature = "metrics")]
use super::http;
impl TidalDb {
/// Construct a `TidalDb` from all opened components.
#[allow(
clippy::too_many_arguments,
clippy::missing_const_for_fn,
clippy::too_many_lines
)]
pub(crate) fn from_parts(
config: Config,
metrics: Arc<MetricsState>,
#[cfg(feature = "metrics")] metrics_handle: Option<http::MetricsHandle>,
result: OpenResult,
) -> Self {
let OpenResult {
storage,
ledger,
wal,
last_seq,
profile_registry,
category_index,
format_index,
creator_index,
tag_index,
duration_index,
created_at_index,
universe,
embedding_registry,
schema_def,
creator_items,
user_state,
hard_negatives,
interaction_ledger,
preference_vectors,
session_events,
} = result;
let ledger = Some(ledger);
let storage = Some(storage);
// M6 co-engagement index: restore from durable storage if available.
let co_engagement = Arc::new(CoEngagementIndex::new());
if let Some(ref sb) = storage
&& let Err(e) = co_engagement.restore(sb.items_engine())
{
tracing::warn!(
error = %e,
"co-engagement restore failed; starting from empty state"
);
}
// M6 user signal index: ephemeral by design; not persisted.
let user_signal_index = Arc::new(UserSignalIndex::new());
// M6 cohort engine: initialize from schema and restore from checkpoint.
let cohort_registry = Arc::new(crate::cohort::CohortRegistry::new());
// Reload persisted cohort definitions (persistent mode only).
if let Some(ref sb) = storage {
let prefix = crate::storage::entity_tag_prefix(
crate::schema::EntityId::new(0),
crate::storage::Tag::CohortDef,
);
let iter = sb.items_engine().scan_prefix(&prefix);
for (_, value) in iter.flatten() {
if let Some(def) = crate::cohort::deserialize_cohort_def(&value)
&& cohort_registry.get(&def.name).is_none()
{
let _ = cohort_registry.define(def);
}
}
}
let cohort_resolver = Arc::new(crate::cohort::CohortResolver::new(Arc::clone(
&cohort_registry,
)));
let cohort_ledger = Arc::new(crate::cohort::CohortSignalLedger::new(&schema_def));
let contribution_log = Arc::new(crate::cohort::CohortContributionLog::new());
// Restore cohort signal state from durable storage (persistent mode only).
if let Some(ref sb) = storage {
match cohort_ledger.restore(sb.items_engine()) {
Ok(Some(meta)) => {
tracing::info!(
entries = cohort_ledger.entry_count(),
wal_sequence = meta.wal_sequence,
"cohort signal ledger restored from checkpoint"
);
}
Ok(None) => {}
Err(e) => {
tracing::warn!(
error = %e,
"cohort signal ledger restore failed; starting from empty state"
);
}
}
}
let text_bundle = spawn_text_syncer(
schema_def.text_fields(),
&config,
"text_index",
"tidaldb-text-syncer",
);
let creator_text_bundle = spawn_text_syncer(
schema_def.creator_text_fields(),
&config,
"creator_text_index",
"tidaldb-creator-text-syncer",
);
// Wrap the embedding registry early so the checkpoint thread can share it.
let embedding_registry = Arc::new(RwLock::new(embedding_registry));
// Spawn periodic checkpoint thread (persistent mode only).
let shutdown_checkpoint = Arc::new(std::sync::atomic::AtomicBool::new(false));
let checkpoint_thread = {
let handle = match (storage.as_ref(), ledger.as_ref()) {
(Some(StorageBox::Fjall(f)), Some(ledger_arc)) => {
let items = Box::new(f.backend(EntityKind::Item).clone())
as Box<dyn StorageEngine + Send + Sync>;
let shutdown = Arc::clone(&shutdown_checkpoint);
let ledger_clone = Arc::clone(ledger_arc);
let cohort_clone = Arc::clone(&cohort_ledger);
let seq_clone = Arc::clone(&last_seq);
let wal_dir = config.data_dir.as_ref().map(|d| d.join("wal"));
let metrics_clone = Arc::clone(&metrics);
#[cfg(feature = "metrics")]
let index_handles = super::state_rebuild::IndexMetricsHandles {
text_index: text_bundle.index.clone(),
embedding_registry: Arc::clone(&embedding_registry),
bitmap_category: category_index.clone(),
bitmap_format: format_index.clone(),
bitmap_creator: creator_index.clone(),
bitmap_tag: tag_index.clone(),
};
#[cfg(not(feature = "metrics"))]
let index_handles = super::state_rebuild::IndexMetricsHandles;
std::thread::Builder::new()
.name("tidaldb-checkpoint".to_string())
.spawn(move || {
run_checkpoint_thread(
shutdown,
ledger_clone,
cohort_clone,
items,
seq_clone,
wal_dir,
metrics_clone,
index_handles,
);
})
.ok()
}
_ => None,
};
std::sync::Mutex::new(handle)
};
// Compute the replication state before moving config into the struct.
let replication_state = {
let mut shards: Vec<crate::replication::ShardId> = vec![config.cluster.shard_id];
shards.extend_from_slice(&config.cluster.peer_shards);
Arc::new(crate::replication::state::ReplicationState::new(&shards))
};
let db = Self {
config,
closed: std::sync::atomic::AtomicBool::new(false),
metrics,
#[cfg(feature = "metrics")]
metrics_handle,
ledger,
storage,
wal: std::sync::Mutex::new(wal),
last_wal_seq: last_seq,
shutdown_checkpoint,
checkpoint_thread,
category_index,
format_index,
creator_index,
tag_index,
duration_index,
created_at_index,
universe: Arc::new(RwLock::new(universe)),
embedding_registry,
profile_registry,
schema_def: Some(schema_def),
creator_items: Arc::new(creator_items),
user_state: Arc::new(user_state),
hard_negatives: Arc::new(hard_negatives),
interaction_ledger: Arc::new(interaction_ledger),
preference_vectors: Arc::new(preference_vectors),
text_index: text_bundle.index,
text_tx: std::sync::Mutex::new(text_bundle.write_tx),
text_syncer_thread: text_bundle.thread,
creator_text_index: creator_text_bundle.index,
creator_text_tx: std::sync::Mutex::new(creator_text_bundle.write_tx),
creator_text_syncer_thread: creator_text_bundle.thread,
text_flush_tx: text_bundle.flush_tx,
creator_text_flush_tx: creator_text_bundle.flush_tx,
sessions: dashmap::DashMap::new(),
next_session_id: std::sync::atomic::AtomicU64::new(1),
closed_sessions: dashmap::DashMap::new(),
co_engagement,
user_signal_index,
cohort_registry,
cohort_resolver,
contribution_log: Arc::clone(&contribution_log),
purge_coordinator: Arc::new(crate::cohort::PurgeCoordinator::new(
Arc::clone(&contribution_log),
Arc::clone(&cohort_ledger),
)),
cohort_ledger,
purge_job_queue: Arc::new(std::sync::Mutex::new(
crate::cohort::rematerialization::PurgeJobQueue::new(),
)),
rematerialization_metrics:
crate::cohort::rematerialization::RematerializationMetrics::new(),
rematerialization_handle: std::sync::Mutex::new(None),
revocation_index: Arc::new(crate::entities::RevocationIndex::new()),
collection_index: Arc::new(CollectionIndex::new()),
community_membership: Arc::new(CommunityMembershipIndex::new()),
community_leave_status: dashmap::DashMap::new(),
suggestion_index: Arc::new(crate::query::suggest::SuggestionIndex::new()),
notification_tracker: Arc::new(notification_tracker::NotificationTracker::new()),
load_detector: Arc::new(crate::load::LoadDetector::new(
crate::load::DegradationThresholds::default(),
)),
backpressure_config: crate::load::BackpressureConfig::default(),
rate_limiter: Arc::new(crate::load::RateLimiter::new(
crate::load::RateLimiterConfig::default(),
)),
shutdown_sweeper: Arc::new(std::sync::atomic::AtomicBool::new(false)),
sweeper_thread: std::sync::Mutex::new(None),
backup_in_progress: Arc::new(std::sync::atomic::AtomicBool::new(false)),
replication_state,
receiver_handle: std::sync::Mutex::new(None),
shipper_handle: None,
session_bridge: Self::make_single_node_session_bridge(),
lock_file: None,
tenant_router: {
let topo = Arc::new(RwLock::new(
crate::replication::tenant::ClusterTopology::single(),
));
Arc::new(crate::replication::TenantRouter::new(topo))
},
control_plane: {
let topo = Arc::new(RwLock::new(
crate::replication::tenant::ClusterTopology::single(),
));
let router = Arc::new(crate::replication::TenantRouter::new(Arc::clone(&topo)));
let rep_state = Arc::new(crate::replication::state::ReplicationState::single());
let lag = Arc::new(crate::replication::ReplicationLagGauge::new(
crate::replication::ShardId::SINGLE,
rep_state,
));
Arc::new(crate::replication::ControlPlane::new(topo, router, lag))
},
};
// M6p4: rebuild collection index from durable storage.
// M6p5: rebuild suggestion index title terms from durable item metadata.
// M9: rebuild community membership index from durable users storage.
// M10: rebuild signal revocation index from durable users storage.
if let Some(ref sb) = db.storage {
crate::db::collections::rebuild_collections(&db.collection_index, sb.items_engine());
crate::db::state_rebuild::rebuild_suggestion_index(sb, &db.suggestion_index);
crate::db::state_rebuild::rebuild_community_memberships(sb, &db.community_membership);
crate::db::state_rebuild::rebuild_community_leave_status(
sb,
&db.community_leave_status,
);
crate::db::revocation::rebuild_revocations(&db.revocation_index, sb.users_engine());
}
if !session_events.is_empty() {
db.restore_session_wal_events(&session_events);
}
// M9: Start re-materialization background worker (persistent mode only).
if let Some(ref data_dir) = db.config.data_dir {
let wal_dir = data_dir.join("wal");
let audit_log_path = data_dir.join("purge_audit.log");
if let Ok(queue) = db.purge_job_queue.lock() {
queue.requeue_interrupted();
}
let engine_config = crate::cohort::rematerialization::RematerializationConfig {
cohort_ledger: Arc::clone(&db.cohort_ledger),
contribution_log: Arc::clone(&db.contribution_log),
schema: db.schema_def.clone().unwrap_or_else(|| {
crate::schema::SchemaBuilder::new()
.build()
.expect("empty schema")
}),
wal_dir,
audit_log_path,
job_queue: Arc::clone(&db.purge_job_queue),
metrics: Arc::clone(&db.rematerialization_metrics),
};
let handle = crate::cohort::rematerialization::start(engine_config);
if let Ok(mut guard) = db.rematerialization_handle.lock() {
*guard = Some(handle);
}
}
db
}
}