//! Cache-line-aligned, lock-free per-entity signal state for the hot path. //! //! `HotSignalState` is the single hottest struct in `TidalDB`'s ranking pipeline. //! Every ranking query touches it for every candidate entity. The design is //! driven by three constraints: //! //! 1. **Cache-line alignment** -- one entity's signal state never shares a cache //! line with another, eliminating false sharing under concurrent reads. //! 2. **Lock-free updates** -- signal ingestion uses CAS loops on individual //! decay scores, so readers are never blocked by writers. //! 3. **O(1) running decay** -- scores are maintained incrementally via the //! identity `S(t) = S(prev) * exp(-lambda * dt) + weight`, avoiding //! re-summation of the full event history. //! //! # Memory ordering rationale //! //! - `last_update_ns` loads use `Acquire` to establish happens-before with the //! writer's `Release` store, ensuring all prior score CAS operations are visible. //! - `last_update_ns` CAS success uses `Release` to make all prior score writes //! visible to readers who subsequently `Acquire` the timestamp. //! - `last_update_ns` CAS failure uses `Relaxed` because we discard the result //! on failure (a concurrent writer already advanced the timestamp). //! - `decay_scores[i]` loads use `Acquire` to see the latest CAS'd value. //! - `decay_scores[i]` CAS success uses `AcqRel` -- `Release` makes the new //! score visible, `Acquire` loads the freshest competing value. //! - `decay_scores[i]` CAS failure uses `Acquire` to load the freshest //! competing write for the next retry iteration. use std::fmt; use std::sync::atomic::{AtomicU64, Ordering}; /// Maximum number of concurrent decay rates tracked per entity-signal pair. pub const MAX_DECAY_RATES: usize = 3; /// Bit 0 of `flags`: velocity tracking is enabled for this signal. const FLAG_VELOCITY_ENABLED: u16 = 0x0001; /// Per-entity, per-signal-type hot state for the ranking pipeline. /// /// Fits exactly one cache line (64 bytes). All mutable fields are atomic, /// enabling lock-free concurrent reads and writes. Immutable fields /// (`entity_id`, `signal_type_id`, `flags`) are set at construction and /// never modified. #[repr(C, align(64))] pub struct HotSignalState { /// Immutable after construction. Identifies the entity this state belongs to. entity_id: u64, /// Nanosecond timestamp of the most recent in-order signal event processed. /// Updated only when a new event's timestamp exceeds the current value. last_update_ns: AtomicU64, /// Immutable after construction. Identifies the signal type. signal_type_id: u16, /// Immutable after construction. Bit flags (see `FLAG_VELOCITY_ENABLED`). flags: u16, /// Padding to maintain field alignment. _pad0: [u8; 4], /// Running exponentially-decayed scores, one per decay rate. /// Stored as `f64::to_bits()` for atomic CAS. decay_scores: [AtomicU64; 3], /// Padding to fill the cache line to exactly 64 bytes. _pad1: [u8; 16], } // Compile-time assertions: struct must be exactly one cache line. const _SIZE: () = assert!(std::mem::size_of::() == 64); const _ALIGN: () = assert!(std::mem::align_of::() == 64); impl HotSignalState { /// Creates a new zeroed state with velocity tracking disabled. #[must_use] pub const fn new(entity_id: u64, signal_type_id: u16) -> Self { Self::with_flags(entity_id, signal_type_id, false) } /// Creates a new zeroed state with explicit velocity flag. #[must_use] pub const fn with_flags(entity_id: u64, signal_type_id: u16, velocity_enabled: bool) -> Self { let flags = if velocity_enabled { FLAG_VELOCITY_ENABLED } else { 0 }; Self { entity_id, last_update_ns: AtomicU64::new(0), signal_type_id, flags, _pad0: [0; 4], decay_scores: [ AtomicU64::new(0_f64.to_bits()), AtomicU64::new(0_f64.to_bits()), AtomicU64::new(0_f64.to_bits()), ], _pad1: [0; 16], } } /// Returns the entity ID this state belongs to. Immutable after construction. #[must_use] pub const fn entity_id(&self) -> u64 { self.entity_id } /// Returns the signal type ID. Immutable after construction. #[must_use] pub const fn signal_type_id(&self) -> u16 { self.signal_type_id } /// Returns whether velocity tracking is enabled for this signal. #[must_use] pub const fn velocity_enabled(&self) -> bool { self.flags & FLAG_VELOCITY_ENABLED != 0 } /// Records a signal event, updating all decay scores atomically. /// /// Handles both in-order and out-of-order events: /// - **In-order** (`event_time_ns >= last_update_ns`): decays existing scores /// by `dt` then adds `weight`. Advances the timestamp. /// - **Out-of-order** (`event_time_ns < last_update_ns`): pre-decays the /// weight by the event's age, then adds the reduced weight. Does NOT /// regress the timestamp. /// /// Each decay score is updated via an independent CAS loop, so concurrent /// writers on different decay rates do not contend. #[allow(clippy::cast_precision_loss)] pub fn on_signal(&self, weight: f64, event_time_ns: u64, lambdas: &[f64]) { let last_ns = self.last_update_ns.load(Ordering::Acquire); if event_time_ns >= last_ns { // In-order path: decay existing scores forward, then add weight. let dt_secs = (event_time_ns - last_ns) as f64 / 1e9; for (i, &lambda) in lambdas.iter().take(MAX_DECAY_RATES).enumerate() { let decay_factor = (-lambda * dt_secs).exp(); loop { let old_bits = self.decay_scores[i].load(Ordering::Acquire); let old_score = f64::from_bits(old_bits); let new_score = old_score.mul_add(decay_factor, weight); debug_assert!(new_score >= 0.0); if self.decay_scores[i] .compare_exchange_weak( old_bits, new_score.to_bits(), Ordering::AcqRel, Ordering::Acquire, ) .is_ok() { break; } } } #[cfg(any(test, feature = "test-utils"))] crate::testing::crash_injector::check_crash_point( crate::testing::CrashPoint::SignalAggregationUpdate, ); // Advance timestamp. CAS failure is acceptable: a concurrent writer // already pushed the timestamp further forward. let _ = self.last_update_ns.compare_exchange( last_ns, event_time_ns, Ordering::Release, Ordering::Relaxed, ); } else { // Out-of-order path: pre-decay the weight by the event's age. let age_secs = (last_ns - event_time_ns) as f64 / 1e9; for (i, &lambda) in lambdas.iter().take(MAX_DECAY_RATES).enumerate() { let effective_weight = weight * (-lambda * age_secs).exp(); loop { let old_bits = self.decay_scores[i].load(Ordering::Acquire); let old_score = f64::from_bits(old_bits); let new_score = old_score + effective_weight; debug_assert!(new_score >= 0.0); if self.decay_scores[i] .compare_exchange_weak( old_bits, new_score.to_bits(), Ordering::AcqRel, Ordering::Acquire, ) .is_ok() { break; } } } // Do NOT update last_update_ns -- the timestamp must not regress. } } /// Returns the current decayed score for a given decay rate index and query time. /// /// The stored score is decayed forward from `last_update_ns` to `query_time_ns`. /// If `query_time_ns` is in the past relative to the last update, no additional /// decay is applied (dt clamped to zero). /// /// Out-of-bounds `decay_rate_idx` is saturated to `MAX_DECAY_RATES - 1` to /// avoid panicking on the hot path. /// /// # Concurrency /// /// Reads `last_update_ns` and the score without a lock. The score may /// reflect a slightly different timestamp than `query_time_ns` if a concurrent /// `on_signal` is in flight. This is intentional — approximate reads are /// acceptable for ranking. Callers must not rely on the score being exactly /// consistent with `query_time_ns`. #[must_use] #[allow(clippy::cast_precision_loss)] pub fn current_score(&self, decay_rate_idx: usize, query_time_ns: u64, lambda: f64) -> f64 { debug_assert!( decay_rate_idx < MAX_DECAY_RATES, "decay_rate_idx {decay_rate_idx} out of bounds (max {MAX_DECAY_RATES})" ); let idx = decay_rate_idx.min(MAX_DECAY_RATES - 1); let last_ns = self.last_update_ns.load(Ordering::Acquire); let stored = f64::from_bits(self.decay_scores[idx].load(Ordering::Acquire)); let dt_secs = if query_time_ns >= last_ns { (query_time_ns - last_ns) as f64 / 1e9 } else { 0.0 }; let score = stored * (-lambda * dt_secs).exp(); score.max(0.0) } /// Returns the raw stored score for a given decay rate index, without /// applying any additional time-based decay. /// /// Out-of-bounds `decay_rate_idx` is saturated to `MAX_DECAY_RATES - 1`. #[must_use] pub fn stored_score(&self, decay_rate_idx: usize) -> f64 { debug_assert!( decay_rate_idx < MAX_DECAY_RATES, "decay_rate_idx {decay_rate_idx} out of bounds (max {MAX_DECAY_RATES})" ); let idx = decay_rate_idx.min(MAX_DECAY_RATES - 1); f64::from_bits(self.decay_scores[idx].load(Ordering::Acquire)) } /// Returns the nanosecond timestamp of the most recent in-order signal event. #[must_use] pub fn last_update_ns(&self) -> u64 { self.last_update_ns.load(Ordering::Acquire) } /// Force-set the decay score at the given lambda index. /// /// Used by reconciliation to apply merged CRDT state. Bypasses the /// CAS-loop increment pattern for a direct write. /// /// # Ordering /// /// Uses `Release` so that any subsequent `Acquire` load on the same /// index sees the updated score. pub(crate) fn force_set_score(&self, lambda_idx: usize, score: f64) { if lambda_idx < MAX_DECAY_RATES { self.decay_scores[lambda_idx].store(score.to_bits(), Ordering::Release); } } /// Force-set the last-update timestamp. /// /// Used by reconciliation to align the timestamp with merged CRDT state. /// /// # Ordering /// /// Uses `Release` so that any subsequent `Acquire` load sees the /// updated timestamp along with any previously force-set scores. pub(crate) fn force_set_last_update_ns(&self, ts_ns: u64) { self.last_update_ns.store(ts_ns, Ordering::Release); } /// Restores state from durable storage during crash recovery or cold start. /// /// Scores are stored first, then the timestamp is stored last with `Release` /// ordering. This ensures any reader who sees the new timestamp via `Acquire` /// will also see all the restored scores. pub fn restore(&self, last_update_ns: u64, scores: &[f64]) { for (i, &score) in scores.iter().take(MAX_DECAY_RATES).enumerate() { self.decay_scores[i].store(score.to_bits(), Ordering::Release); } // Timestamp stored LAST so readers see scores before timestamp. self.last_update_ns.store(last_update_ns, Ordering::Release); } } #[allow(clippy::missing_fields_in_debug)] impl fmt::Debug for HotSignalState { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("HotSignalState") .field("entity_id", &self.entity_id) .field("signal_type_id", &self.signal_type_id) .field("velocity_enabled", &self.velocity_enabled()) .field("last_update_ns", &self.last_update_ns()) .field("score[0]", &self.stored_score(0)) .field("score[1]", &self.stored_score(1)) .field("score[2]", &self.stored_score(2)) .finish() } } #[cfg(test)] #[allow( clippy::unwrap_used, clippy::float_cmp, clippy::cast_sign_loss, clippy::cast_precision_loss )] #[path = "hot_tests.rs"] mod tests;