tidaldb/tidal/src/signals/ledger/core.rs
jordan 6f26d03c77 feat(m9): implement purge re-materialization engine
Adds the M9 purge re-materialization feature: a WAL-replay background
engine that rebuilds community cohort aggregates for a (user, community)
pair after retroactive signal purge, restoring ranking correctness without
modifying the immutable WAL.

Key additions:
- cohort::rematerialization module: PurgeJobQueue, RematerializationEngine,
  WAL replay, atomic CohortSignalLedger swap, BLAKE3 audit log, metrics counters
- TidalDb::{submit_purge_job, purge_job_status, rematerialization_metrics}
  public API (db/rematerialization.rs)
- Engine auto-starts in persistent mode; clean shutdown before WAL teardown
- 6 integration tests in tests/m9_purge_remat.rs covering ephemeral and
  persistent modes, job lifecycle, and multi-job independence
- Split oversized files to comply with 600-line limit: db/mod.rs →
  db/from_parts.rs, entities/revocation.rs → revocation/{mod,tests}.rs,
  schema/validation/builders.rs → builders/{mod,tests}.rs,
  signals/warm.rs → warm/{mod,tests,proptests}.rs
- Fix pre-existing bootstrap errors: export AuditKind from session module,
  add overrides_rejected to SessionSnapshot deserialization
2026-03-03 19:18:16 -07:00

374 lines
14 KiB
Rust

//! The signal ledger: coordinates hot and warm tiers for all active entities.
//!
//! `SignalLedger` is the single entry point for signal state management.
//! It owns a `DashMap<(EntityId, SignalTypeId), EntitySignalEntry>` that
//! provides concurrent access to per-entity signal state.
use std::collections::HashMap;
use std::fmt;
use dashmap::DashMap;
use crate::schema::{DecayModel, EntityId, Schema, SchemaError, TidalError, Timestamp, Window};
use super::super::SignalTypeId;
use super::super::hot::HotSignalState;
use super::super::warm::BucketedCounter;
use super::types::{EntitySignalEntry, WalWriter};
// ── Ledger ────────────────────────────────────────────────────────────────────
/// The signal ledger: coordinates hot and warm tiers for all active entities.
///
/// This is the single entry point for signal state management. `TidalDB`
/// holds a `SignalLedger` and delegates all signal operations to it.
pub struct SignalLedger {
/// Per-(entity, `signal_type`) state. Sharded for concurrent access.
pub(crate) entries: DashMap<(EntityId, SignalTypeId), EntitySignalEntry>,
/// WAL writer for durability.
wal: Box<dyn WalWriter>,
/// Schema for signal type lookup and lambda retrieval.
schema: Schema,
/// Signal name -> `SignalTypeId` mapping (built at construction, immutable).
signal_name_to_id: HashMap<String, SignalTypeId>,
/// `SignalTypeId` -> lambda array (cached from schema, immutable after construction).
signal_lambdas: HashMap<SignalTypeId, Vec<f64>>,
}
impl SignalLedger {
/// Construct a new ledger from a validated schema and WAL writer.
///
/// Signal types are enumerated in alphabetical order and assigned
/// sequential `SignalTypeId` values (0, 1, 2, ...).
#[must_use]
pub fn new(schema: Schema, wal: Box<dyn WalWriter>) -> Self {
let mut signal_list: Vec<&crate::schema::SignalTypeDef> = schema.signals().collect();
signal_list.sort_by_key(|s| s.name());
let mut signal_name_to_id = HashMap::with_capacity(signal_list.len());
let mut signal_lambdas = HashMap::with_capacity(signal_list.len());
for (idx, sig) in signal_list.iter().enumerate() {
let type_id = SignalTypeId::new(idx as u16);
signal_name_to_id.insert(sig.name().to_owned(), type_id);
let lambdas = match sig.decay() {
DecayModel::Exponential { lambda, .. } => vec![*lambda],
DecayModel::Linear { .. } | DecayModel::Permanent => vec![],
};
signal_lambdas.insert(type_id, lambdas);
}
Self {
entries: DashMap::with_shard_amount(16),
wal,
schema,
signal_name_to_id,
signal_lambdas,
}
}
/// Record a signal event for an entity.
///
/// Steps:
/// 1. Resolve signal type name -> `SignalTypeId`
/// 2. Append event to WAL (WAL-first)
/// 3. Get or create `EntitySignalEntry` in `DashMap`
/// 4. Update hot-tier decay score
/// 5. Update warm-tier windowed counters
///
/// # Errors
///
/// - `TidalError::Schema` if `signal_type_name` is not defined
/// - `TidalError::Durability` if WAL write fails
///
/// # Durability contract
///
/// The WAL is written **before** in-memory state is updated. If
/// `wal.append_signal()` returns `Err`, the in-memory state is NOT updated
/// and the error is returned to the caller. Implementors of `WalWriter`
/// MUST guarantee persistence before returning `Ok(())`.
pub fn record_signal(
&self,
signal_type_name: &str,
entity_id: EntityId,
weight: f64,
timestamp: Timestamp,
) -> crate::Result<()> {
let type_id = self.resolve_signal_type(signal_type_name)?;
// WAL-first: durability before in-memory update.
self.wal
.append_signal(type_id, entity_id, weight, timestamp)?;
#[cfg(any(test, feature = "test-utils"))]
crate::testing::crash_injector::check_crash_point(
crate::testing::CrashPoint::WalPreAggregate,
);
let lambdas = self
.signal_lambdas
.get(&type_id)
.map_or_else(<&[f64]>::default, Vec::as_slice);
let ts_ns = timestamp.as_nanos();
let entry = self
.entries
.entry((entity_id, type_id))
.or_insert_with(|| EntitySignalEntry {
hot: HotSignalState::new(entity_id.as_u64(), type_id.as_u16()),
warm: BucketedCounter::with_start_time(ts_ns),
});
entry.hot.on_signal(weight, ts_ns, lambdas);
entry.warm.increment(ts_ns);
#[cfg(any(test, feature = "test-utils"))]
crate::testing::crash_injector::check_crash_point(
crate::testing::CrashPoint::WalPostAggregate,
);
// Explicitly drop the DashMap entry ref to release the shard lock before
// returning (satisfies clippy::significant_drop_tightening).
drop(entry);
tracing::trace!(
signal = signal_type_name,
entity_id = entity_id.as_u64(),
weight,
"signal recorded"
);
Ok(())
}
/// Read the current decay score for an entity-signal pair.
///
/// Applies lazy decay from the stored timestamp to the current wall-clock time.
/// Returns `None` if no signals have been recorded for this entity-signal pair.
///
/// # Errors
///
/// - `TidalError::Schema` if `signal_type_name` is not defined
pub fn read_decay_score(
&self,
entity_id: EntityId,
signal_type_name: &str,
decay_rate_idx: usize,
) -> crate::Result<Option<f64>> {
let type_id = self.resolve_signal_type(signal_type_name)?;
let now_ns = Timestamp::now().as_nanos();
let lambda = self
.signal_lambdas
.get(&type_id)
.and_then(|v| v.get(decay_rate_idx))
.copied()
.unwrap_or(0.0);
match self.entries.get(&(entity_id, type_id)) {
None => Ok(None),
Some(entry) => Ok(Some(entry.hot.current_score(
decay_rate_idx,
now_ns,
lambda,
))),
}
}
/// Read the windowed event count for an entity-signal pair.
///
/// Returns 0 if no signals have been recorded for this entity-signal pair.
///
/// # Errors
///
/// - `TidalError::Schema` if `signal_type_name` is not defined
pub fn read_windowed_count(
&self,
entity_id: EntityId,
signal_type_name: &str,
window: Window,
) -> crate::Result<u64> {
let type_id = self.resolve_signal_type(signal_type_name)?;
match self.entries.get(&(entity_id, type_id)) {
None => Ok(0),
Some(entry) => Ok(entry.warm.windowed_count(window)),
}
}
/// Read the velocity (events per second) for an entity-signal-window.
///
/// Velocity = `windowed_count / window_duration_seconds`.
/// `AllTime` returns 0.0 (velocity is undefined for unbounded windows).
/// Returns 0.0 if no signals have been recorded for this entity-signal pair.
///
/// # Errors
///
/// - `TidalError::Schema` if `signal_type_name` is not defined
pub fn read_velocity(
&self,
entity_id: EntityId,
signal_type_name: &str,
window: Window,
) -> crate::Result<f64> {
let count = self.read_windowed_count(entity_id, signal_type_name, window)?;
let duration_secs = window.duration_secs_f64();
if duration_secs.is_infinite() {
// AllTime window -- velocity is undefined.
return Ok(0.0);
}
#[allow(clippy::cast_precision_loss)]
Ok(count as f64 / duration_secs)
}
/// Resolve a signal type name to its `SignalTypeId`.
///
/// # Errors
///
/// - `TidalError::Schema` if the name is not defined in this ledger's schema
pub fn resolve_signal_type(&self, name: &str) -> crate::Result<SignalTypeId> {
self.signal_name_to_id
.get(name)
.copied()
.ok_or_else(|| TidalError::Schema(SchemaError::UnknownSignalType(name.to_owned())))
}
/// Apply a WAL event directly to in-memory state, bypassing the WAL write.
///
/// Used during WAL replay on startup. The event has already been durably
/// stored on disk; we only need to rebuild in-memory state from it.
///
/// Unlike `record_signal`, this method:
/// - Does **not** call `wal.append_signal()`.
/// - Accepts a `SignalTypeId` directly (already resolved from the WAL record).
/// - Silently ignores unknown `signal_type_id` values (may occur when the
/// schema changes between restarts; missing lambdas default to 0.0).
pub(crate) fn apply_wal_event(
&self,
signal_type_id: SignalTypeId,
entity_id: EntityId,
weight: f64,
timestamp: Timestamp,
) {
let lambdas = self
.signal_lambdas
.get(&signal_type_id)
.map_or_else(<&[f64]>::default, Vec::as_slice);
let ts_ns = timestamp.as_nanos();
let entry = self
.entries
.entry((entity_id, signal_type_id))
.or_insert_with(|| EntitySignalEntry {
hot: HotSignalState::new(entity_id.as_u64(), signal_type_id.as_u16()),
warm: BucketedCounter::with_start_time(ts_ns),
});
entry.hot.on_signal(weight, ts_ns, lambdas);
entry.warm.increment(ts_ns);
drop(entry);
}
/// Apply a merged CRDT signal state to the ledger entry.
///
/// Called by `ReconciliationEngine::apply()` after a merge plan is computed.
/// Computes the combined decay score at the current time and force-sets it
/// on the hot tier, bypassing the normal WAL-first path.
///
/// This is intentionally NOT WAL-backed: reconciliation replaces diverged
/// in-memory state from CRDT-merged snapshots. The WAL remains the
/// authoritative event log; reconciliation is a state-level correction
/// that aligns the hot tier with the merged truth.
///
/// # Errors
///
/// Returns `TidalError::Internal` if the signal type has no lambda entry
/// in this ledger (unknown signal type).
pub(crate) fn apply_crdt_state(
&self,
entity_id: EntityId,
signal_type_id: SignalTypeId,
state: &crate::replication::crdt::CrdtSignalState,
) -> crate::Result<()> {
let now_ns = Timestamp::now().as_nanos();
// Verify the signal type is known. This also gives us the lambda
// array so we know how many decay rate slots to fill.
let lambdas = self.signal_lambdas.get(&signal_type_id).ok_or_else(|| {
TidalError::internal(
"apply_crdt_state",
format!("unknown signal type id: {}", signal_type_id.as_u16()),
)
})?;
// Compute the merged decay score at the current time.
let score = state.decay_score(now_ns);
// Get or create the entry (same pattern as apply_wal_event).
let entry = self
.entries
.entry((entity_id, signal_type_id))
.or_insert_with(|| EntitySignalEntry {
hot: HotSignalState::new(entity_id.as_u64(), signal_type_id.as_u16()),
warm: BucketedCounter::with_start_time(now_ns),
});
// Force-set the score for each lambda index. CrdtSignalState uses a
// single lambda per signal type, so all indices receive the same merged
// score (each index corresponds to a different decay rate for the same
// signal, but the CRDT only tracks the primary rate).
for i in 0..lambdas.len().min(crate::signals::hot::MAX_DECAY_RATES) {
entry.hot.force_set_score(i, score);
}
// If no lambdas are configured (permanent signal), set index 0 as fallback.
if lambdas.is_empty() {
entry.hot.force_set_score(0, score);
}
entry.hot.force_set_last_update_ns(now_ns);
drop(entry);
Ok(())
}
/// Return the `last_update_ns` timestamp for an `(entity_id, signal_type_id)` pair.
///
/// Returns `0` if no signals have been recorded for this pair.
///
/// Used by the ranking executor to determine whether a signal falls within
/// a revocation's time-bounded scope.
#[must_use]
pub fn entry_last_update_ns(&self, entity_id: EntityId, type_id: SignalTypeId) -> u64 {
self.entries
.get(&(entity_id, type_id))
.map_or(0, |e| e.hot.last_update_ns())
}
/// Reference to the `DashMap` for checkpoint iteration and benchmarking.
#[must_use]
pub const fn entries(&self) -> &DashMap<(EntityId, SignalTypeId), EntitySignalEntry> {
&self.entries
}
/// The schema this ledger was constructed with.
#[must_use]
pub const fn schema(&self) -> &Schema {
&self.schema
}
}
impl fmt::Debug for SignalLedger {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("SignalLedger")
.field("entry_count", &self.entries.len())
.field("signal_count", &self.schema.signal_count())
.finish_non_exhaustive()
}
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::cast_precision_loss)]
#[path = "core_tests.rs"]
mod tests;