//! WAL replay filter for community signal re-materialization. //! //! Reads all events from the WAL, partitions them into retained and excluded //! sets based on the purge job's contribution records, then replays the retained //! events into a fresh `CohortSignalLedger`. #![allow(clippy::must_use_candidate, dead_code)] use std::collections::HashSet; use std::path::Path; use crate::cohort::CohortSignalLedger; use crate::cohort::ContributionRecord; use crate::schema::{Schema, TidalError}; use crate::signals::SignalTypeId; use crate::wal::error::WalError; /// Statistics from a WAL replay pass. #[derive(Debug, Clone, Default)] pub struct ReplayStats { /// Number of events replayed into the scratch ledger. pub events_replayed: u64, /// Number of events excluded (purged). pub events_excluded: u64, /// Number of ledger entries created or updated during replay. pub entries_updated: u64, } /// Key used for precise event exclusion. /// /// Matches on all four fields of an `EventRecord` to avoid accidentally /// excluding signals from other users that happen to share the same entity. #[derive(Debug, Clone, PartialEq, Eq, Hash)] struct ExclusionKey { entity_id: u64, signal_type: u8, /// Weight stored as bits to allow hashing. weight_bits: u32, timestamp_nanos: u64, } /// Build an exclusion set from a list of contribution records. /// /// The exclusion set uses 4-tuples `(entity_id, signal_type, weight, ts_ns)` /// for precise matching, preventing accidental exclusion of signals from other /// users on the same entity. fn build_exclusion_set( contributions: &[ContributionRecord], signal_type_filter: Option<&[String]>, schema: &Schema, ) -> HashSet { let mut set = HashSet::with_capacity(contributions.len()); for rec in contributions { // Apply signal type filter if specified. if let Some(filter) = signal_type_filter { // Check whether this record's signal_type_id matches any name in the filter. let matches = filter.iter().any(|name| { // Resolve name to id via schema signal list (sorted alphabetically). let mut signal_list: Vec<&crate::schema::SignalTypeDef> = schema.signals().collect(); signal_list.sort_by_key(|s| s.name()); signal_list.iter().enumerate().any(|(idx, sig)| { sig.name() == name && SignalTypeId::new(idx as u16).as_u16() == rec.signal_type_id.as_u16() }) }); if !matches { continue; } } set.insert(ExclusionKey { entity_id: rec.entity_id.as_u64(), signal_type: rec.signal_type_id.as_u16() as u8, weight_bits: rec.weight.to_bits(), timestamp_nanos: rec.timestamp_ns, }); } set } /// Result of a filtered WAL replay. pub struct ReplayResult { /// Scratch ledger containing the post-purge aggregate state. pub ledger: CohortSignalLedger, /// Replay statistics. pub stats: ReplayStats, } /// Perform a filtered WAL replay for a purge job. /// /// Reads all signal events from `wal_dir`, excludes events matching the /// contribution records, and replays the retained events into a fresh /// `CohortSignalLedger` under `community_id`. /// /// Events are sorted by `timestamp_nanos` before replay to ensure correct /// exponential decay accumulation. /// /// # Errors /// /// Returns a `TidalError` if the WAL directory cannot be read. pub fn filtered_replay( wal_dir: &Path, schema: &Schema, community_id: &str, contributions: &[ContributionRecord], signal_type_filter: Option<&[String]>, ) -> crate::Result { // Build exclusion set from contribution records. let exclusion_set = build_exclusion_set(contributions, signal_type_filter, schema); // Read all WAL events. let all_events = crate::wal::reader::read_all_events(wal_dir).map_err(map_wal_error)?; // Sort by timestamp for correct decay accumulation. let mut sorted_events = all_events; sorted_events.sort_unstable_by_key(|e| e.timestamp_nanos); // Partition into retained and excluded. let mut stats = ReplayStats::default(); let scratch = CohortSignalLedger::new(schema); // Build a set of signal type IDs from schema (sorted alphabetically, same as ledger). let mut signal_list: Vec<&crate::schema::SignalTypeDef> = schema.signals().collect(); signal_list.sort_by_key(|s| s.name()); for event in &sorted_events { let key = ExclusionKey { entity_id: event.entity_id, signal_type: event.signal_type, weight_bits: event.weight.to_bits(), timestamp_nanos: event.timestamp_nanos, }; if exclusion_set.contains(&key) { stats.events_excluded += 1; } else { // Only replay events whose signal_type maps to a known signal in schema. if signal_list.get(event.signal_type as usize).is_some() { let type_id = SignalTypeId::new(u16::from(event.signal_type)); scratch.record( community_id, crate::schema::EntityId::new(event.entity_id), type_id, f64::from(event.weight), event.timestamp_nanos, ); } stats.events_replayed += 1; } } // Count entries actually created in scratch ledger. stats.entries_updated = scratch.entry_count() as u64; Ok(ReplayResult { ledger: scratch, stats, }) } fn map_wal_error(e: WalError) -> TidalError { TidalError::Durability(crate::schema::DurabilityError { message: format!("WAL read failed during filtered_replay: {e}"), }) } // ── Tests ───────────────────────────────────────────────────────────────────── #[cfg(test)] #[allow(clippy::unwrap_used, clippy::cast_possible_truncation)] mod tests { use std::time::Duration; use super::*; use crate::schema::{DecaySpec, EntityId, EntityKind, SchemaBuilder, Window}; fn test_schema() -> Schema { let mut builder = SchemaBuilder::new(); let _ = builder .signal( "view", EntityKind::Item, DecaySpec::Exponential { half_life: Duration::from_secs(7 * 24 * 3600), }, ) .windows(&[Window::AllTime]) .velocity(false) .add(); builder.build().unwrap() } fn make_contribution( entity_id: u64, signal_type_id: u16, weight: f32, ts_ns: u64, ) -> ContributionRecord { ContributionRecord { user_id: 1, cohort: "tech".to_owned(), entity_id: EntityId::new(entity_id), signal_type_id: SignalTypeId::new(signal_type_id), weight, timestamp_ns: ts_ns, } } #[test] fn empty_contributions_returns_zero_excluded() { // With no contributions to exclude, all events are retained. let schema = test_schema(); let contributions: Vec = vec![]; let exclusion_set = build_exclusion_set(&contributions, None, &schema); assert!(exclusion_set.is_empty()); } #[test] fn exclusion_set_built_correctly() { let schema = test_schema(); let contributions = vec![ make_contribution(42, 0, 1.0, 1_000_000), make_contribution(43, 0, 0.5, 2_000_000), ]; let exclusion_set = build_exclusion_set(&contributions, None, &schema); assert_eq!(exclusion_set.len(), 2); let key1 = ExclusionKey { entity_id: 42, signal_type: 0, weight_bits: 1.0f32.to_bits(), timestamp_nanos: 1_000_000, }; assert!(exclusion_set.contains(&key1)); } #[test] fn signal_type_filter_excludes_non_matching() { let schema = test_schema(); // Contribution is for signal type 0 ("view"). let contributions = vec![make_contribution(42, 0, 1.0, 1_000_000)]; // Filter for a non-existent type: nothing gets excluded. let exclusion_set = build_exclusion_set(&contributions, Some(&["like".to_owned()]), &schema); assert!(exclusion_set.is_empty()); } #[test] fn signal_type_filter_includes_matching() { let schema = test_schema(); let contributions = vec![make_contribution(42, 0, 1.0, 1_000_000)]; // Filter for "view" (which maps to type_id 0 in our test schema). let exclusion_set = build_exclusion_set(&contributions, Some(&["view".to_owned()]), &schema); assert_eq!(exclusion_set.len(), 1); } #[test] fn exclusion_key_differentiates_by_weight() { let schema = test_schema(); // Same entity, signal, ts — different weight. let contributions = vec![make_contribution(42, 0, 1.0, 1_000_000)]; let set = build_exclusion_set(&contributions, None, &schema); // A key with different weight should NOT be in the set. let different_weight = ExclusionKey { entity_id: 42, signal_type: 0, weight_bits: 0.5f32.to_bits(), timestamp_nanos: 1_000_000, }; assert!(!set.contains(&different_weight)); } }