Adds the M9 purge re-materialization feature: a WAL-replay background
engine that rebuilds community cohort aggregates for a (user, community)
pair after retroactive signal purge, restoring ranking correctness without
modifying the immutable WAL.
Key additions:
- cohort::rematerialization module: PurgeJobQueue, RematerializationEngine,
WAL replay, atomic CohortSignalLedger swap, BLAKE3 audit log, metrics counters
- TidalDb::{submit_purge_job, purge_job_status, rematerialization_metrics}
public API (db/rematerialization.rs)
- Engine auto-starts in persistent mode; clean shutdown before WAL teardown
- 6 integration tests in tests/m9_purge_remat.rs covering ephemeral and
persistent modes, job lifecycle, and multi-job independence
- Split oversized files to comply with 600-line limit: db/mod.rs →
db/from_parts.rs, entities/revocation.rs → revocation/{mod,tests}.rs,
schema/validation/builders.rs → builders/{mod,tests}.rs,
signals/warm.rs → warm/{mod,tests,proptests}.rs
- Fix pre-existing bootstrap errors: export AuditKind from session module,
add overrides_rejected to SessionSnapshot deserialization
374 lines
14 KiB
Rust
374 lines
14 KiB
Rust
//! The signal ledger: coordinates hot and warm tiers for all active entities.
|
|
//!
|
|
//! `SignalLedger` is the single entry point for signal state management.
|
|
//! It owns a `DashMap<(EntityId, SignalTypeId), EntitySignalEntry>` that
|
|
//! provides concurrent access to per-entity signal state.
|
|
|
|
use std::collections::HashMap;
|
|
use std::fmt;
|
|
|
|
use dashmap::DashMap;
|
|
|
|
use crate::schema::{DecayModel, EntityId, Schema, SchemaError, TidalError, Timestamp, Window};
|
|
|
|
use super::super::SignalTypeId;
|
|
use super::super::hot::HotSignalState;
|
|
use super::super::warm::BucketedCounter;
|
|
use super::types::{EntitySignalEntry, WalWriter};
|
|
|
|
// ── Ledger ────────────────────────────────────────────────────────────────────
|
|
|
|
/// The signal ledger: coordinates hot and warm tiers for all active entities.
|
|
///
|
|
/// This is the single entry point for signal state management. `TidalDB`
|
|
/// holds a `SignalLedger` and delegates all signal operations to it.
|
|
pub struct SignalLedger {
|
|
/// Per-(entity, `signal_type`) state. Sharded for concurrent access.
|
|
pub(crate) entries: DashMap<(EntityId, SignalTypeId), EntitySignalEntry>,
|
|
/// WAL writer for durability.
|
|
wal: Box<dyn WalWriter>,
|
|
/// Schema for signal type lookup and lambda retrieval.
|
|
schema: Schema,
|
|
/// Signal name -> `SignalTypeId` mapping (built at construction, immutable).
|
|
signal_name_to_id: HashMap<String, SignalTypeId>,
|
|
/// `SignalTypeId` -> lambda array (cached from schema, immutable after construction).
|
|
signal_lambdas: HashMap<SignalTypeId, Vec<f64>>,
|
|
}
|
|
|
|
impl SignalLedger {
|
|
/// Construct a new ledger from a validated schema and WAL writer.
|
|
///
|
|
/// Signal types are enumerated in alphabetical order and assigned
|
|
/// sequential `SignalTypeId` values (0, 1, 2, ...).
|
|
#[must_use]
|
|
pub fn new(schema: Schema, wal: Box<dyn WalWriter>) -> Self {
|
|
let mut signal_list: Vec<&crate::schema::SignalTypeDef> = schema.signals().collect();
|
|
signal_list.sort_by_key(|s| s.name());
|
|
|
|
let mut signal_name_to_id = HashMap::with_capacity(signal_list.len());
|
|
let mut signal_lambdas = HashMap::with_capacity(signal_list.len());
|
|
|
|
for (idx, sig) in signal_list.iter().enumerate() {
|
|
let type_id = SignalTypeId::new(idx as u16);
|
|
signal_name_to_id.insert(sig.name().to_owned(), type_id);
|
|
|
|
let lambdas = match sig.decay() {
|
|
DecayModel::Exponential { lambda, .. } => vec![*lambda],
|
|
DecayModel::Linear { .. } | DecayModel::Permanent => vec![],
|
|
};
|
|
signal_lambdas.insert(type_id, lambdas);
|
|
}
|
|
|
|
Self {
|
|
entries: DashMap::with_shard_amount(16),
|
|
wal,
|
|
schema,
|
|
signal_name_to_id,
|
|
signal_lambdas,
|
|
}
|
|
}
|
|
|
|
/// Record a signal event for an entity.
|
|
///
|
|
/// Steps:
|
|
/// 1. Resolve signal type name -> `SignalTypeId`
|
|
/// 2. Append event to WAL (WAL-first)
|
|
/// 3. Get or create `EntitySignalEntry` in `DashMap`
|
|
/// 4. Update hot-tier decay score
|
|
/// 5. Update warm-tier windowed counters
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// - `TidalError::Schema` if `signal_type_name` is not defined
|
|
/// - `TidalError::Durability` if WAL write fails
|
|
///
|
|
/// # Durability contract
|
|
///
|
|
/// The WAL is written **before** in-memory state is updated. If
|
|
/// `wal.append_signal()` returns `Err`, the in-memory state is NOT updated
|
|
/// and the error is returned to the caller. Implementors of `WalWriter`
|
|
/// MUST guarantee persistence before returning `Ok(())`.
|
|
pub fn record_signal(
|
|
&self,
|
|
signal_type_name: &str,
|
|
entity_id: EntityId,
|
|
weight: f64,
|
|
timestamp: Timestamp,
|
|
) -> crate::Result<()> {
|
|
let type_id = self.resolve_signal_type(signal_type_name)?;
|
|
|
|
// WAL-first: durability before in-memory update.
|
|
self.wal
|
|
.append_signal(type_id, entity_id, weight, timestamp)?;
|
|
|
|
#[cfg(any(test, feature = "test-utils"))]
|
|
crate::testing::crash_injector::check_crash_point(
|
|
crate::testing::CrashPoint::WalPreAggregate,
|
|
);
|
|
|
|
let lambdas = self
|
|
.signal_lambdas
|
|
.get(&type_id)
|
|
.map_or_else(<&[f64]>::default, Vec::as_slice);
|
|
|
|
let ts_ns = timestamp.as_nanos();
|
|
|
|
let entry = self
|
|
.entries
|
|
.entry((entity_id, type_id))
|
|
.or_insert_with(|| EntitySignalEntry {
|
|
hot: HotSignalState::new(entity_id.as_u64(), type_id.as_u16()),
|
|
warm: BucketedCounter::with_start_time(ts_ns),
|
|
});
|
|
|
|
entry.hot.on_signal(weight, ts_ns, lambdas);
|
|
entry.warm.increment(ts_ns);
|
|
|
|
#[cfg(any(test, feature = "test-utils"))]
|
|
crate::testing::crash_injector::check_crash_point(
|
|
crate::testing::CrashPoint::WalPostAggregate,
|
|
);
|
|
|
|
// Explicitly drop the DashMap entry ref to release the shard lock before
|
|
// returning (satisfies clippy::significant_drop_tightening).
|
|
drop(entry);
|
|
|
|
tracing::trace!(
|
|
signal = signal_type_name,
|
|
entity_id = entity_id.as_u64(),
|
|
weight,
|
|
"signal recorded"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Read the current decay score for an entity-signal pair.
|
|
///
|
|
/// Applies lazy decay from the stored timestamp to the current wall-clock time.
|
|
/// Returns `None` if no signals have been recorded for this entity-signal pair.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// - `TidalError::Schema` if `signal_type_name` is not defined
|
|
pub fn read_decay_score(
|
|
&self,
|
|
entity_id: EntityId,
|
|
signal_type_name: &str,
|
|
decay_rate_idx: usize,
|
|
) -> crate::Result<Option<f64>> {
|
|
let type_id = self.resolve_signal_type(signal_type_name)?;
|
|
let now_ns = Timestamp::now().as_nanos();
|
|
|
|
let lambda = self
|
|
.signal_lambdas
|
|
.get(&type_id)
|
|
.and_then(|v| v.get(decay_rate_idx))
|
|
.copied()
|
|
.unwrap_or(0.0);
|
|
|
|
match self.entries.get(&(entity_id, type_id)) {
|
|
None => Ok(None),
|
|
Some(entry) => Ok(Some(entry.hot.current_score(
|
|
decay_rate_idx,
|
|
now_ns,
|
|
lambda,
|
|
))),
|
|
}
|
|
}
|
|
|
|
/// Read the windowed event count for an entity-signal pair.
|
|
///
|
|
/// Returns 0 if no signals have been recorded for this entity-signal pair.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// - `TidalError::Schema` if `signal_type_name` is not defined
|
|
pub fn read_windowed_count(
|
|
&self,
|
|
entity_id: EntityId,
|
|
signal_type_name: &str,
|
|
window: Window,
|
|
) -> crate::Result<u64> {
|
|
let type_id = self.resolve_signal_type(signal_type_name)?;
|
|
match self.entries.get(&(entity_id, type_id)) {
|
|
None => Ok(0),
|
|
Some(entry) => Ok(entry.warm.windowed_count(window)),
|
|
}
|
|
}
|
|
|
|
/// Read the velocity (events per second) for an entity-signal-window.
|
|
///
|
|
/// Velocity = `windowed_count / window_duration_seconds`.
|
|
/// `AllTime` returns 0.0 (velocity is undefined for unbounded windows).
|
|
/// Returns 0.0 if no signals have been recorded for this entity-signal pair.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// - `TidalError::Schema` if `signal_type_name` is not defined
|
|
pub fn read_velocity(
|
|
&self,
|
|
entity_id: EntityId,
|
|
signal_type_name: &str,
|
|
window: Window,
|
|
) -> crate::Result<f64> {
|
|
let count = self.read_windowed_count(entity_id, signal_type_name, window)?;
|
|
let duration_secs = window.duration_secs_f64();
|
|
if duration_secs.is_infinite() {
|
|
// AllTime window -- velocity is undefined.
|
|
return Ok(0.0);
|
|
}
|
|
#[allow(clippy::cast_precision_loss)]
|
|
Ok(count as f64 / duration_secs)
|
|
}
|
|
|
|
/// Resolve a signal type name to its `SignalTypeId`.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// - `TidalError::Schema` if the name is not defined in this ledger's schema
|
|
pub fn resolve_signal_type(&self, name: &str) -> crate::Result<SignalTypeId> {
|
|
self.signal_name_to_id
|
|
.get(name)
|
|
.copied()
|
|
.ok_or_else(|| TidalError::Schema(SchemaError::UnknownSignalType(name.to_owned())))
|
|
}
|
|
|
|
/// Apply a WAL event directly to in-memory state, bypassing the WAL write.
|
|
///
|
|
/// Used during WAL replay on startup. The event has already been durably
|
|
/// stored on disk; we only need to rebuild in-memory state from it.
|
|
///
|
|
/// Unlike `record_signal`, this method:
|
|
/// - Does **not** call `wal.append_signal()`.
|
|
/// - Accepts a `SignalTypeId` directly (already resolved from the WAL record).
|
|
/// - Silently ignores unknown `signal_type_id` values (may occur when the
|
|
/// schema changes between restarts; missing lambdas default to 0.0).
|
|
pub(crate) fn apply_wal_event(
|
|
&self,
|
|
signal_type_id: SignalTypeId,
|
|
entity_id: EntityId,
|
|
weight: f64,
|
|
timestamp: Timestamp,
|
|
) {
|
|
let lambdas = self
|
|
.signal_lambdas
|
|
.get(&signal_type_id)
|
|
.map_or_else(<&[f64]>::default, Vec::as_slice);
|
|
|
|
let ts_ns = timestamp.as_nanos();
|
|
|
|
let entry = self
|
|
.entries
|
|
.entry((entity_id, signal_type_id))
|
|
.or_insert_with(|| EntitySignalEntry {
|
|
hot: HotSignalState::new(entity_id.as_u64(), signal_type_id.as_u16()),
|
|
warm: BucketedCounter::with_start_time(ts_ns),
|
|
});
|
|
|
|
entry.hot.on_signal(weight, ts_ns, lambdas);
|
|
entry.warm.increment(ts_ns);
|
|
drop(entry);
|
|
}
|
|
|
|
/// Apply a merged CRDT signal state to the ledger entry.
|
|
///
|
|
/// Called by `ReconciliationEngine::apply()` after a merge plan is computed.
|
|
/// Computes the combined decay score at the current time and force-sets it
|
|
/// on the hot tier, bypassing the normal WAL-first path.
|
|
///
|
|
/// This is intentionally NOT WAL-backed: reconciliation replaces diverged
|
|
/// in-memory state from CRDT-merged snapshots. The WAL remains the
|
|
/// authoritative event log; reconciliation is a state-level correction
|
|
/// that aligns the hot tier with the merged truth.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns `TidalError::Internal` if the signal type has no lambda entry
|
|
/// in this ledger (unknown signal type).
|
|
pub(crate) fn apply_crdt_state(
|
|
&self,
|
|
entity_id: EntityId,
|
|
signal_type_id: SignalTypeId,
|
|
state: &crate::replication::crdt::CrdtSignalState,
|
|
) -> crate::Result<()> {
|
|
let now_ns = Timestamp::now().as_nanos();
|
|
|
|
// Verify the signal type is known. This also gives us the lambda
|
|
// array so we know how many decay rate slots to fill.
|
|
let lambdas = self.signal_lambdas.get(&signal_type_id).ok_or_else(|| {
|
|
TidalError::internal(
|
|
"apply_crdt_state",
|
|
format!("unknown signal type id: {}", signal_type_id.as_u16()),
|
|
)
|
|
})?;
|
|
|
|
// Compute the merged decay score at the current time.
|
|
let score = state.decay_score(now_ns);
|
|
|
|
// Get or create the entry (same pattern as apply_wal_event).
|
|
let entry = self
|
|
.entries
|
|
.entry((entity_id, signal_type_id))
|
|
.or_insert_with(|| EntitySignalEntry {
|
|
hot: HotSignalState::new(entity_id.as_u64(), signal_type_id.as_u16()),
|
|
warm: BucketedCounter::with_start_time(now_ns),
|
|
});
|
|
|
|
// Force-set the score for each lambda index. CrdtSignalState uses a
|
|
// single lambda per signal type, so all indices receive the same merged
|
|
// score (each index corresponds to a different decay rate for the same
|
|
// signal, but the CRDT only tracks the primary rate).
|
|
for i in 0..lambdas.len().min(crate::signals::hot::MAX_DECAY_RATES) {
|
|
entry.hot.force_set_score(i, score);
|
|
}
|
|
// If no lambdas are configured (permanent signal), set index 0 as fallback.
|
|
if lambdas.is_empty() {
|
|
entry.hot.force_set_score(0, score);
|
|
}
|
|
entry.hot.force_set_last_update_ns(now_ns);
|
|
|
|
drop(entry);
|
|
Ok(())
|
|
}
|
|
|
|
/// Return the `last_update_ns` timestamp for an `(entity_id, signal_type_id)` pair.
|
|
///
|
|
/// Returns `0` if no signals have been recorded for this pair.
|
|
///
|
|
/// Used by the ranking executor to determine whether a signal falls within
|
|
/// a revocation's time-bounded scope.
|
|
#[must_use]
|
|
pub fn entry_last_update_ns(&self, entity_id: EntityId, type_id: SignalTypeId) -> u64 {
|
|
self.entries
|
|
.get(&(entity_id, type_id))
|
|
.map_or(0, |e| e.hot.last_update_ns())
|
|
}
|
|
|
|
/// Reference to the `DashMap` for checkpoint iteration and benchmarking.
|
|
#[must_use]
|
|
pub const fn entries(&self) -> &DashMap<(EntityId, SignalTypeId), EntitySignalEntry> {
|
|
&self.entries
|
|
}
|
|
|
|
/// The schema this ledger was constructed with.
|
|
#[must_use]
|
|
pub const fn schema(&self) -> &Schema {
|
|
&self.schema
|
|
}
|
|
}
|
|
|
|
impl fmt::Debug for SignalLedger {
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
f.debug_struct("SignalLedger")
|
|
.field("entry_count", &self.entries.len())
|
|
.field("signal_count", &self.schema.signal_count())
|
|
.finish_non_exhaustive()
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
#[allow(clippy::unwrap_used, clippy::cast_precision_loss)]
|
|
#[path = "core_tests.rs"]
|
|
mod tests;
|