Adds the M9 purge re-materialization feature: a WAL-replay background
engine that rebuilds community cohort aggregates for a (user, community)
pair after retroactive signal purge, restoring ranking correctness without
modifying the immutable WAL.
Key additions:
- cohort::rematerialization module: PurgeJobQueue, RematerializationEngine,
WAL replay, atomic CohortSignalLedger swap, BLAKE3 audit log, metrics counters
- TidalDb::{submit_purge_job, purge_job_status, rematerialization_metrics}
public API (db/rematerialization.rs)
- Engine auto-starts in persistent mode; clean shutdown before WAL teardown
- 6 integration tests in tests/m9_purge_remat.rs covering ephemeral and
persistent modes, job lifecycle, and multi-job independence
- Split oversized files to comply with 600-line limit: db/mod.rs →
db/from_parts.rs, entities/revocation.rs → revocation/{mod,tests}.rs,
schema/validation/builders.rs → builders/{mod,tests}.rs,
signals/warm.rs → warm/{mod,tests,proptests}.rs
- Fix pre-existing bootstrap errors: export AuditKind from session module,
add overrides_rejected to SessionSnapshot deserialization
278 lines
9.5 KiB
Rust
278 lines
9.5 KiB
Rust
//! WAL replay filter for community signal re-materialization.
|
|
//!
|
|
//! Reads all events from the WAL, partitions them into retained and excluded
|
|
//! sets based on the purge job's contribution records, then replays the retained
|
|
//! events into a fresh `CohortSignalLedger`.
|
|
|
|
#![allow(clippy::must_use_candidate, dead_code)]
|
|
|
|
use std::collections::HashSet;
|
|
use std::path::Path;
|
|
|
|
use crate::cohort::CohortSignalLedger;
|
|
use crate::cohort::ContributionRecord;
|
|
use crate::schema::{Schema, TidalError};
|
|
use crate::signals::SignalTypeId;
|
|
use crate::wal::error::WalError;
|
|
|
|
/// Statistics from a WAL replay pass.
|
|
#[derive(Debug, Clone, Default)]
|
|
pub struct ReplayStats {
|
|
/// Number of events replayed into the scratch ledger.
|
|
pub events_replayed: u64,
|
|
/// Number of events excluded (purged).
|
|
pub events_excluded: u64,
|
|
/// Number of ledger entries created or updated during replay.
|
|
pub entries_updated: u64,
|
|
}
|
|
|
|
/// Key used for precise event exclusion.
|
|
///
|
|
/// Matches on all four fields of an `EventRecord` to avoid accidentally
|
|
/// excluding signals from other users that happen to share the same entity.
|
|
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
|
struct ExclusionKey {
|
|
entity_id: u64,
|
|
signal_type: u8,
|
|
/// Weight stored as bits to allow hashing.
|
|
weight_bits: u32,
|
|
timestamp_nanos: u64,
|
|
}
|
|
|
|
/// Build an exclusion set from a list of contribution records.
|
|
///
|
|
/// The exclusion set uses 4-tuples `(entity_id, signal_type, weight, ts_ns)`
|
|
/// for precise matching, preventing accidental exclusion of signals from other
|
|
/// users on the same entity.
|
|
fn build_exclusion_set(
|
|
contributions: &[ContributionRecord],
|
|
signal_type_filter: Option<&[String]>,
|
|
schema: &Schema,
|
|
) -> HashSet<ExclusionKey> {
|
|
let mut set = HashSet::with_capacity(contributions.len());
|
|
|
|
for rec in contributions {
|
|
// Apply signal type filter if specified.
|
|
if let Some(filter) = signal_type_filter {
|
|
// Check whether this record's signal_type_id matches any name in the filter.
|
|
let matches = filter.iter().any(|name| {
|
|
// Resolve name to id via schema signal list (sorted alphabetically).
|
|
let mut signal_list: Vec<&crate::schema::SignalTypeDef> =
|
|
schema.signals().collect();
|
|
signal_list.sort_by_key(|s| s.name());
|
|
signal_list.iter().enumerate().any(|(idx, sig)| {
|
|
sig.name() == name
|
|
&& SignalTypeId::new(idx as u16).as_u16() == rec.signal_type_id.as_u16()
|
|
})
|
|
});
|
|
if !matches {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
set.insert(ExclusionKey {
|
|
entity_id: rec.entity_id.as_u64(),
|
|
signal_type: rec.signal_type_id.as_u16() as u8,
|
|
weight_bits: rec.weight.to_bits(),
|
|
timestamp_nanos: rec.timestamp_ns,
|
|
});
|
|
}
|
|
|
|
set
|
|
}
|
|
|
|
/// Result of a filtered WAL replay.
|
|
pub struct ReplayResult {
|
|
/// Scratch ledger containing the post-purge aggregate state.
|
|
pub ledger: CohortSignalLedger,
|
|
/// Replay statistics.
|
|
pub stats: ReplayStats,
|
|
}
|
|
|
|
/// Perform a filtered WAL replay for a purge job.
|
|
///
|
|
/// Reads all signal events from `wal_dir`, excludes events matching the
|
|
/// contribution records, and replays the retained events into a fresh
|
|
/// `CohortSignalLedger` under `community_id`.
|
|
///
|
|
/// Events are sorted by `timestamp_nanos` before replay to ensure correct
|
|
/// exponential decay accumulation.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns a `TidalError` if the WAL directory cannot be read.
|
|
pub fn filtered_replay(
|
|
wal_dir: &Path,
|
|
schema: &Schema,
|
|
community_id: &str,
|
|
contributions: &[ContributionRecord],
|
|
signal_type_filter: Option<&[String]>,
|
|
) -> crate::Result<ReplayResult> {
|
|
// Build exclusion set from contribution records.
|
|
let exclusion_set = build_exclusion_set(contributions, signal_type_filter, schema);
|
|
|
|
// Read all WAL events.
|
|
let all_events = crate::wal::reader::read_all_events(wal_dir).map_err(map_wal_error)?;
|
|
|
|
// Sort by timestamp for correct decay accumulation.
|
|
let mut sorted_events = all_events;
|
|
sorted_events.sort_unstable_by_key(|e| e.timestamp_nanos);
|
|
|
|
// Partition into retained and excluded.
|
|
let mut stats = ReplayStats::default();
|
|
let scratch = CohortSignalLedger::new(schema);
|
|
|
|
// Build a set of signal type IDs from schema (sorted alphabetically, same as ledger).
|
|
let mut signal_list: Vec<&crate::schema::SignalTypeDef> = schema.signals().collect();
|
|
signal_list.sort_by_key(|s| s.name());
|
|
|
|
for event in &sorted_events {
|
|
let key = ExclusionKey {
|
|
entity_id: event.entity_id,
|
|
signal_type: event.signal_type,
|
|
weight_bits: event.weight.to_bits(),
|
|
timestamp_nanos: event.timestamp_nanos,
|
|
};
|
|
|
|
if exclusion_set.contains(&key) {
|
|
stats.events_excluded += 1;
|
|
} else {
|
|
// Only replay events whose signal_type maps to a known signal in schema.
|
|
if signal_list.get(event.signal_type as usize).is_some() {
|
|
let type_id = SignalTypeId::new(u16::from(event.signal_type));
|
|
scratch.record(
|
|
community_id,
|
|
crate::schema::EntityId::new(event.entity_id),
|
|
type_id,
|
|
f64::from(event.weight),
|
|
event.timestamp_nanos,
|
|
);
|
|
}
|
|
stats.events_replayed += 1;
|
|
}
|
|
}
|
|
|
|
// Count entries actually created in scratch ledger.
|
|
stats.entries_updated = scratch.entry_count() as u64;
|
|
|
|
Ok(ReplayResult {
|
|
ledger: scratch,
|
|
stats,
|
|
})
|
|
}
|
|
|
|
fn map_wal_error(e: WalError) -> TidalError {
|
|
TidalError::Durability(crate::schema::DurabilityError {
|
|
message: format!("WAL read failed during filtered_replay: {e}"),
|
|
})
|
|
}
|
|
|
|
// ── Tests ─────────────────────────────────────────────────────────────────────
|
|
|
|
#[cfg(test)]
|
|
#[allow(clippy::unwrap_used, clippy::cast_possible_truncation)]
|
|
mod tests {
|
|
use std::time::Duration;
|
|
|
|
use super::*;
|
|
use crate::schema::{DecaySpec, EntityId, EntityKind, SchemaBuilder, Window};
|
|
|
|
fn test_schema() -> Schema {
|
|
let mut builder = SchemaBuilder::new();
|
|
let _ = builder
|
|
.signal(
|
|
"view",
|
|
EntityKind::Item,
|
|
DecaySpec::Exponential {
|
|
half_life: Duration::from_secs(7 * 24 * 3600),
|
|
},
|
|
)
|
|
.windows(&[Window::AllTime])
|
|
.velocity(false)
|
|
.add();
|
|
builder.build().unwrap()
|
|
}
|
|
|
|
fn make_contribution(
|
|
entity_id: u64,
|
|
signal_type_id: u16,
|
|
weight: f32,
|
|
ts_ns: u64,
|
|
) -> ContributionRecord {
|
|
ContributionRecord {
|
|
user_id: 1,
|
|
cohort: "tech".to_owned(),
|
|
entity_id: EntityId::new(entity_id),
|
|
signal_type_id: SignalTypeId::new(signal_type_id),
|
|
weight,
|
|
timestamp_ns: ts_ns,
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn empty_contributions_returns_zero_excluded() {
|
|
// With no contributions to exclude, all events are retained.
|
|
let schema = test_schema();
|
|
let contributions: Vec<ContributionRecord> = vec![];
|
|
let exclusion_set = build_exclusion_set(&contributions, None, &schema);
|
|
assert!(exclusion_set.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn exclusion_set_built_correctly() {
|
|
let schema = test_schema();
|
|
let contributions = vec![
|
|
make_contribution(42, 0, 1.0, 1_000_000),
|
|
make_contribution(43, 0, 0.5, 2_000_000),
|
|
];
|
|
let exclusion_set = build_exclusion_set(&contributions, None, &schema);
|
|
assert_eq!(exclusion_set.len(), 2);
|
|
|
|
let key1 = ExclusionKey {
|
|
entity_id: 42,
|
|
signal_type: 0,
|
|
weight_bits: 1.0f32.to_bits(),
|
|
timestamp_nanos: 1_000_000,
|
|
};
|
|
assert!(exclusion_set.contains(&key1));
|
|
}
|
|
|
|
#[test]
|
|
fn signal_type_filter_excludes_non_matching() {
|
|
let schema = test_schema();
|
|
// Contribution is for signal type 0 ("view").
|
|
let contributions = vec![make_contribution(42, 0, 1.0, 1_000_000)];
|
|
// Filter for a non-existent type: nothing gets excluded.
|
|
let exclusion_set =
|
|
build_exclusion_set(&contributions, Some(&["like".to_owned()]), &schema);
|
|
assert!(exclusion_set.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn signal_type_filter_includes_matching() {
|
|
let schema = test_schema();
|
|
let contributions = vec![make_contribution(42, 0, 1.0, 1_000_000)];
|
|
// Filter for "view" (which maps to type_id 0 in our test schema).
|
|
let exclusion_set =
|
|
build_exclusion_set(&contributions, Some(&["view".to_owned()]), &schema);
|
|
assert_eq!(exclusion_set.len(), 1);
|
|
}
|
|
|
|
#[test]
|
|
fn exclusion_key_differentiates_by_weight() {
|
|
let schema = test_schema();
|
|
// Same entity, signal, ts — different weight.
|
|
let contributions = vec![make_contribution(42, 0, 1.0, 1_000_000)];
|
|
let set = build_exclusion_set(&contributions, None, &schema);
|
|
|
|
// A key with different weight should NOT be in the set.
|
|
let different_weight = ExclusionKey {
|
|
entity_id: 42,
|
|
signal_type: 0,
|
|
weight_bits: 0.5f32.to_bits(),
|
|
timestamp_nanos: 1_000_000,
|
|
};
|
|
assert!(!set.contains(&different_weight));
|
|
}
|
|
}
|