tidaldb/tidal/src/cohort/rematerialization/replay.rs
jordan 6f26d03c77 feat(m9): implement purge re-materialization engine
Adds the M9 purge re-materialization feature: a WAL-replay background
engine that rebuilds community cohort aggregates for a (user, community)
pair after retroactive signal purge, restoring ranking correctness without
modifying the immutable WAL.

Key additions:
- cohort::rematerialization module: PurgeJobQueue, RematerializationEngine,
  WAL replay, atomic CohortSignalLedger swap, BLAKE3 audit log, metrics counters
- TidalDb::{submit_purge_job, purge_job_status, rematerialization_metrics}
  public API (db/rematerialization.rs)
- Engine auto-starts in persistent mode; clean shutdown before WAL teardown
- 6 integration tests in tests/m9_purge_remat.rs covering ephemeral and
  persistent modes, job lifecycle, and multi-job independence
- Split oversized files to comply with 600-line limit: db/mod.rs →
  db/from_parts.rs, entities/revocation.rs → revocation/{mod,tests}.rs,
  schema/validation/builders.rs → builders/{mod,tests}.rs,
  signals/warm.rs → warm/{mod,tests,proptests}.rs
- Fix pre-existing bootstrap errors: export AuditKind from session module,
  add overrides_rejected to SessionSnapshot deserialization
2026-03-03 19:18:16 -07:00

278 lines
9.5 KiB
Rust

//! WAL replay filter for community signal re-materialization.
//!
//! Reads all events from the WAL, partitions them into retained and excluded
//! sets based on the purge job's contribution records, then replays the retained
//! events into a fresh `CohortSignalLedger`.
#![allow(clippy::must_use_candidate, dead_code)]
use std::collections::HashSet;
use std::path::Path;
use crate::cohort::CohortSignalLedger;
use crate::cohort::ContributionRecord;
use crate::schema::{Schema, TidalError};
use crate::signals::SignalTypeId;
use crate::wal::error::WalError;
/// Statistics from a WAL replay pass.
#[derive(Debug, Clone, Default)]
pub struct ReplayStats {
/// Number of events replayed into the scratch ledger.
pub events_replayed: u64,
/// Number of events excluded (purged).
pub events_excluded: u64,
/// Number of ledger entries created or updated during replay.
pub entries_updated: u64,
}
/// Key used for precise event exclusion.
///
/// Matches on all four fields of an `EventRecord` to avoid accidentally
/// excluding signals from other users that happen to share the same entity.
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
struct ExclusionKey {
entity_id: u64,
signal_type: u8,
/// Weight stored as bits to allow hashing.
weight_bits: u32,
timestamp_nanos: u64,
}
/// Build an exclusion set from a list of contribution records.
///
/// The exclusion set uses 4-tuples `(entity_id, signal_type, weight, ts_ns)`
/// for precise matching, preventing accidental exclusion of signals from other
/// users on the same entity.
fn build_exclusion_set(
contributions: &[ContributionRecord],
signal_type_filter: Option<&[String]>,
schema: &Schema,
) -> HashSet<ExclusionKey> {
let mut set = HashSet::with_capacity(contributions.len());
for rec in contributions {
// Apply signal type filter if specified.
if let Some(filter) = signal_type_filter {
// Check whether this record's signal_type_id matches any name in the filter.
let matches = filter.iter().any(|name| {
// Resolve name to id via schema signal list (sorted alphabetically).
let mut signal_list: Vec<&crate::schema::SignalTypeDef> =
schema.signals().collect();
signal_list.sort_by_key(|s| s.name());
signal_list.iter().enumerate().any(|(idx, sig)| {
sig.name() == name
&& SignalTypeId::new(idx as u16).as_u16() == rec.signal_type_id.as_u16()
})
});
if !matches {
continue;
}
}
set.insert(ExclusionKey {
entity_id: rec.entity_id.as_u64(),
signal_type: rec.signal_type_id.as_u16() as u8,
weight_bits: rec.weight.to_bits(),
timestamp_nanos: rec.timestamp_ns,
});
}
set
}
/// Result of a filtered WAL replay.
pub struct ReplayResult {
/// Scratch ledger containing the post-purge aggregate state.
pub ledger: CohortSignalLedger,
/// Replay statistics.
pub stats: ReplayStats,
}
/// Perform a filtered WAL replay for a purge job.
///
/// Reads all signal events from `wal_dir`, excludes events matching the
/// contribution records, and replays the retained events into a fresh
/// `CohortSignalLedger` under `community_id`.
///
/// Events are sorted by `timestamp_nanos` before replay to ensure correct
/// exponential decay accumulation.
///
/// # Errors
///
/// Returns a `TidalError` if the WAL directory cannot be read.
pub fn filtered_replay(
wal_dir: &Path,
schema: &Schema,
community_id: &str,
contributions: &[ContributionRecord],
signal_type_filter: Option<&[String]>,
) -> crate::Result<ReplayResult> {
// Build exclusion set from contribution records.
let exclusion_set = build_exclusion_set(contributions, signal_type_filter, schema);
// Read all WAL events.
let all_events = crate::wal::reader::read_all_events(wal_dir).map_err(map_wal_error)?;
// Sort by timestamp for correct decay accumulation.
let mut sorted_events = all_events;
sorted_events.sort_unstable_by_key(|e| e.timestamp_nanos);
// Partition into retained and excluded.
let mut stats = ReplayStats::default();
let scratch = CohortSignalLedger::new(schema);
// Build a set of signal type IDs from schema (sorted alphabetically, same as ledger).
let mut signal_list: Vec<&crate::schema::SignalTypeDef> = schema.signals().collect();
signal_list.sort_by_key(|s| s.name());
for event in &sorted_events {
let key = ExclusionKey {
entity_id: event.entity_id,
signal_type: event.signal_type,
weight_bits: event.weight.to_bits(),
timestamp_nanos: event.timestamp_nanos,
};
if exclusion_set.contains(&key) {
stats.events_excluded += 1;
} else {
// Only replay events whose signal_type maps to a known signal in schema.
if signal_list.get(event.signal_type as usize).is_some() {
let type_id = SignalTypeId::new(u16::from(event.signal_type));
scratch.record(
community_id,
crate::schema::EntityId::new(event.entity_id),
type_id,
f64::from(event.weight),
event.timestamp_nanos,
);
}
stats.events_replayed += 1;
}
}
// Count entries actually created in scratch ledger.
stats.entries_updated = scratch.entry_count() as u64;
Ok(ReplayResult {
ledger: scratch,
stats,
})
}
fn map_wal_error(e: WalError) -> TidalError {
TidalError::Durability(crate::schema::DurabilityError {
message: format!("WAL read failed during filtered_replay: {e}"),
})
}
// ── Tests ─────────────────────────────────────────────────────────────────────
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::cast_possible_truncation)]
mod tests {
use std::time::Duration;
use super::*;
use crate::schema::{DecaySpec, EntityId, EntityKind, SchemaBuilder, Window};
fn test_schema() -> Schema {
let mut builder = SchemaBuilder::new();
let _ = builder
.signal(
"view",
EntityKind::Item,
DecaySpec::Exponential {
half_life: Duration::from_secs(7 * 24 * 3600),
},
)
.windows(&[Window::AllTime])
.velocity(false)
.add();
builder.build().unwrap()
}
fn make_contribution(
entity_id: u64,
signal_type_id: u16,
weight: f32,
ts_ns: u64,
) -> ContributionRecord {
ContributionRecord {
user_id: 1,
cohort: "tech".to_owned(),
entity_id: EntityId::new(entity_id),
signal_type_id: SignalTypeId::new(signal_type_id),
weight,
timestamp_ns: ts_ns,
}
}
#[test]
fn empty_contributions_returns_zero_excluded() {
// With no contributions to exclude, all events are retained.
let schema = test_schema();
let contributions: Vec<ContributionRecord> = vec![];
let exclusion_set = build_exclusion_set(&contributions, None, &schema);
assert!(exclusion_set.is_empty());
}
#[test]
fn exclusion_set_built_correctly() {
let schema = test_schema();
let contributions = vec![
make_contribution(42, 0, 1.0, 1_000_000),
make_contribution(43, 0, 0.5, 2_000_000),
];
let exclusion_set = build_exclusion_set(&contributions, None, &schema);
assert_eq!(exclusion_set.len(), 2);
let key1 = ExclusionKey {
entity_id: 42,
signal_type: 0,
weight_bits: 1.0f32.to_bits(),
timestamp_nanos: 1_000_000,
};
assert!(exclusion_set.contains(&key1));
}
#[test]
fn signal_type_filter_excludes_non_matching() {
let schema = test_schema();
// Contribution is for signal type 0 ("view").
let contributions = vec![make_contribution(42, 0, 1.0, 1_000_000)];
// Filter for a non-existent type: nothing gets excluded.
let exclusion_set =
build_exclusion_set(&contributions, Some(&["like".to_owned()]), &schema);
assert!(exclusion_set.is_empty());
}
#[test]
fn signal_type_filter_includes_matching() {
let schema = test_schema();
let contributions = vec![make_contribution(42, 0, 1.0, 1_000_000)];
// Filter for "view" (which maps to type_id 0 in our test schema).
let exclusion_set =
build_exclusion_set(&contributions, Some(&["view".to_owned()]), &schema);
assert_eq!(exclusion_set.len(), 1);
}
#[test]
fn exclusion_key_differentiates_by_weight() {
let schema = test_schema();
// Same entity, signal, ts — different weight.
let contributions = vec![make_contribution(42, 0, 1.0, 1_000_000)];
let set = build_exclusion_set(&contributions, None, &schema);
// A key with different weight should NOT be in the set.
let different_weight = ExclusionKey {
entity_id: 42,
signal_type: 0,
weight_bits: 0.5f32.to_bits(),
timestamp_nanos: 1_000_000,
};
assert!(!set.contains(&different_weight));
}
}