//! Cache-line-aligned, lock-free per-entity signal state for the hot path. //! //! `HotSignalState` is the single hottest struct in `TidalDB`'s ranking pipeline. //! Every ranking query touches it for every candidate entity. The design is //! driven by three constraints: //! //! 1. **Cache-line alignment** -- one entity's signal state never shares a cache //! line with another, eliminating false sharing under concurrent reads. //! 2. **Lock-free updates** -- signal ingestion uses CAS loops on individual //! decay scores, so readers are never blocked by writers. //! 3. **O(1) running decay** -- scores are maintained incrementally via the //! identity `S(t) = S(prev) * exp(-lambda * dt) + weight`, avoiding //! re-summation of the full event history. //! //! # Memory ordering rationale //! //! - `last_update_ns` loads use `Acquire` to establish happens-before with the //! writer's `Release` store, ensuring all prior score CAS operations are visible. //! - `last_update_ns` CAS success uses `Release` to make all prior score writes //! visible to readers who subsequently `Acquire` the timestamp. //! - `last_update_ns` CAS failure uses `Relaxed` because we discard the result //! on failure (a concurrent writer already advanced the timestamp). //! - `decay_scores[i]` loads use `Acquire` to see the latest CAS'd value. //! - `decay_scores[i]` CAS success uses `AcqRel` -- `Release` makes the new //! score visible, `Acquire` loads the freshest competing value. //! - `decay_scores[i]` CAS failure uses `Acquire` to load the freshest //! competing write for the next retry iteration. use std::{ fmt, sync::atomic::{AtomicU64, Ordering}, }; use crate::signals::decay::{NANOS_PER_SEC, forward_decay_step}; /// Maximum number of concurrent decay rates tracked per entity-signal pair. pub const MAX_DECAY_RATES: usize = 3; /// Bit 0 of `flags`: velocity tracking is enabled for this signal. const FLAG_VELOCITY_ENABLED: u16 = 0x0001; /// Per-entity, per-signal-type hot state for the ranking pipeline. /// /// Fits exactly one cache line (64 bytes). All mutable fields are atomic, /// enabling lock-free concurrent reads and writes. Immutable fields /// (`entity_id`, `signal_type_id`, `flags`) are set at construction and /// never modified. #[repr(C, align(64))] pub struct HotSignalState { /// Immutable after construction. Identifies the entity this state belongs to. entity_id: u64, /// Nanosecond timestamp of the most recent in-order signal event processed. /// Updated only when a new event's timestamp exceeds the current value. last_update_ns: AtomicU64, /// Immutable after construction. Identifies the signal type. signal_type_id: u16, /// Immutable after construction. Bit flags (see `FLAG_VELOCITY_ENABLED`). flags: u16, /// Padding to maintain field alignment. _pad0: [u8; 4], /// Running exponentially-decayed scores, one per decay rate. /// Stored as `f64::to_bits()` for atomic CAS. decay_scores: [AtomicU64; 3], /// Padding to fill the cache line to exactly 64 bytes. _pad1: [u8; 16], } // Compile-time assertions: struct must be exactly one cache line. const _SIZE: () = assert!(std::mem::size_of::() == 64); const _ALIGN: () = assert!(std::mem::align_of::() == 64); impl HotSignalState { /// Creates a new zeroed state with velocity tracking disabled. #[must_use] pub const fn new(entity_id: u64, signal_type_id: u16) -> Self { Self::with_flags(entity_id, signal_type_id, false) } /// Creates a new zeroed state with explicit velocity flag. #[must_use] pub const fn with_flags(entity_id: u64, signal_type_id: u16, velocity_enabled: bool) -> Self { let flags = if velocity_enabled { FLAG_VELOCITY_ENABLED } else { 0 }; Self { entity_id, last_update_ns: AtomicU64::new(0), signal_type_id, flags, _pad0: [0; 4], decay_scores: [ AtomicU64::new(0_f64.to_bits()), AtomicU64::new(0_f64.to_bits()), AtomicU64::new(0_f64.to_bits()), ], _pad1: [0; 16], } } /// Returns the entity ID this state belongs to. Immutable after construction. #[must_use] pub const fn entity_id(&self) -> u64 { self.entity_id } /// Returns the signal type ID. Immutable after construction. #[must_use] pub const fn signal_type_id(&self) -> u16 { self.signal_type_id } /// Returns whether velocity tracking is enabled for this signal. #[must_use] pub const fn velocity_enabled(&self) -> bool { self.flags & FLAG_VELOCITY_ENABLED != 0 } /// Records a signal event, updating all decay scores atomically. /// /// Both temporal orderings are handled by the canonical decay kernel /// [`forward_decay_step`](crate::signals::decay::forward_decay_step), the /// single source of truth for forward-decay math (`CODING_GUIDELINES` §3): /// - **In-order** (`event_time_ns >= last_update_ns`): decays existing scores /// by `dt` then adds `weight`. Advances the timestamp. /// - **Out-of-order** (`event_time_ns < last_update_ns`): pre-decays the /// weight by the event's age, then adds the reduced weight. Does NOT /// regress the timestamp. /// /// Each decay score is updated via an independent CAS loop, so concurrent /// writers on different decay rates do not contend. The kernel is a pure, /// allocation-free function, so calling it inside the retry loop is free. /// /// # CAS-retry consistency /// /// `last_update_ns` is re-read **inside** each score's retry loop. A CAS /// failure on `decay_scores[i]` means a concurrent `on_signal` won the race /// and folded its event against the timestamp it observed — and it may also /// have advanced `last_update_ns`. Reusing the stale entry-time snapshot for /// the `dt`/`age` of the retry would decay the freshly-CAS'd score from the /// wrong reference time, double- or under-counting the interval. Re-reading /// the timestamp on each attempt keeps the `(old_score, last_update_ns)` pair /// the kernel consumes mutually consistent. The kernel is pure and /// allocation-free, so the extra `Acquire` load on a (rare) contended retry /// is cheap. pub fn on_signal(&self, weight: f64, event_time_ns: u64, lambdas: &[f64]) { // Snapshot the entry-time timestamp for the in-order advance decision and // the final timestamp CAS (both are conditioned on the value we observed // *on entry*). The per-score retry loop re-reads the timestamp itself so a // contended retry decays the freshest score from the right reference time. let entry_last_ns = self.last_update_ns.load(Ordering::Acquire); // The advance decision is identical across lambdas (it depends only on // event_time_ns vs the entry-time timestamp), so we capture it once to // drive the single timestamp CAS below. let advance_timestamp = event_time_ns >= entry_last_ns; for (i, &lambda) in lambdas.iter().take(MAX_DECAY_RATES).enumerate() { loop { // Re-read both the timestamp and the score each attempt so the // pair the kernel folds against stays consistent under a losing // CAS (see "CAS-retry consistency" above). let last_ns = self.last_update_ns.load(Ordering::Acquire); let old_bits = self.decay_scores[i].load(Ordering::Acquire); let old_score = f64::from_bits(old_bits); let step = forward_decay_step(old_score, last_ns, event_time_ns, lambda, weight); debug_assert!(step.new_score >= 0.0); if self.decay_scores[i] .compare_exchange_weak( old_bits, step.new_score.to_bits(), Ordering::AcqRel, Ordering::Acquire, ) .is_ok() { break; } } } if advance_timestamp { // Crash point lives on the in-order path only, between the score // updates and the timestamp advance -- exactly where it was before // the kernel refactor, so crash-recovery semantics are unchanged. #[cfg(any(test, feature = "test-utils"))] crate::testing::crash_injector::check_crash_point( crate::testing::CrashPoint::SignalAggregationUpdate, ); // In-order: advance the timestamp. The CAS is conditioned on the // entry-time snapshot; failure is acceptable -- a concurrent writer // already pushed the timestamp to (or past) event_time_ns. let _ = self.last_update_ns.compare_exchange( entry_last_ns, event_time_ns, Ordering::Release, Ordering::Relaxed, ); } // Out-of-order: do NOT update last_update_ns -- the timestamp must not // regress (CODING_GUIDELINES §3). } /// Returns the current decayed score for a given decay rate index and query time. /// /// The stored score is decayed forward from `last_update_ns` to `query_time_ns`. /// If `query_time_ns` is in the past relative to the last update, no additional /// decay is applied (dt clamped to zero). /// /// Out-of-bounds `decay_rate_idx` is saturated to `MAX_DECAY_RATES - 1` to /// avoid panicking on the hot path. The saturation is unconditional (prod /// and test agree); there is deliberately no `debug_assert!` that would /// panic in debug builds on an index the doc promises to clamp. /// /// # Concurrency /// /// Reads `last_update_ns` and the score without a lock. The score may /// reflect a slightly different timestamp than `query_time_ns` if a concurrent /// `on_signal` is in flight. This is intentional — approximate reads are /// acceptable for ranking. Callers must not rely on the score being exactly /// consistent with `query_time_ns`. #[must_use] #[allow(clippy::cast_precision_loss)] pub fn current_score(&self, decay_rate_idx: usize, query_time_ns: u64, lambda: f64) -> f64 { let idx = decay_rate_idx.min(MAX_DECAY_RATES - 1); let last_ns = self.last_update_ns.load(Ordering::Acquire); let stored = f64::from_bits(self.decay_scores[idx].load(Ordering::Acquire)); let dt_secs = if query_time_ns >= last_ns { (query_time_ns - last_ns) as f64 / NANOS_PER_SEC } else { 0.0 }; let score = stored * (-lambda * dt_secs).exp(); score.max(0.0) } /// Returns the raw stored score for a given decay rate index, without /// applying any additional time-based decay. /// /// Out-of-bounds `decay_rate_idx` is saturated to `MAX_DECAY_RATES - 1` /// unconditionally (prod and test agree); there is no `debug_assert!` that /// would panic on an index the doc promises to clamp. #[must_use] pub fn stored_score(&self, decay_rate_idx: usize) -> f64 { let idx = decay_rate_idx.min(MAX_DECAY_RATES - 1); f64::from_bits(self.decay_scores[idx].load(Ordering::Acquire)) } /// Returns the nanosecond timestamp of the most recent in-order signal event. #[must_use] pub fn last_update_ns(&self) -> u64 { self.last_update_ns.load(Ordering::Acquire) } /// Force-set the decay score at the given lambda index. /// /// Used by reconciliation to apply merged CRDT state. Bypasses the /// CAS-loop increment pattern for a direct write. /// /// # Ordering /// /// Uses `Release` so that any subsequent `Acquire` load on the same /// index sees the updated score. pub(crate) fn force_set_score(&self, lambda_idx: usize, score: f64) { if lambda_idx < MAX_DECAY_RATES { self.decay_scores[lambda_idx].store(score.to_bits(), Ordering::Release); } } /// Force-set the last-update timestamp. /// /// Used by reconciliation to align the timestamp with merged CRDT state. /// /// # Ordering /// /// Uses `Release` so that any subsequent `Acquire` load sees the /// updated timestamp along with any previously force-set scores. pub(crate) fn force_set_last_update_ns(&self, ts_ns: u64) { self.last_update_ns.store(ts_ns, Ordering::Release); } /// Restores state from durable storage during crash recovery or cold start. /// /// Scores are stored first, then the timestamp is stored last with `Release` /// ordering. This ensures any reader who sees the new timestamp via `Acquire` /// will also see all the restored scores. pub fn restore(&self, last_update_ns: u64, scores: &[f64]) { for (i, &score) in scores.iter().take(MAX_DECAY_RATES).enumerate() { self.decay_scores[i].store(score.to_bits(), Ordering::Release); } // Timestamp stored LAST so readers see scores before timestamp. self.last_update_ns.store(last_update_ns, Ordering::Release); } } #[allow(clippy::missing_fields_in_debug)] impl fmt::Debug for HotSignalState { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("HotSignalState") .field("entity_id", &self.entity_id) .field("signal_type_id", &self.signal_type_id) .field("velocity_enabled", &self.velocity_enabled()) .field("last_update_ns", &self.last_update_ns()) .field("score[0]", &self.stored_score(0)) .field("score[1]", &self.stored_score(1)) .field("score[2]", &self.stored_score(2)) .finish() } } #[cfg(test)] #[allow( clippy::unwrap_used, clippy::float_cmp, clippy::cast_sign_loss, clippy::cast_precision_loss )] #[path = "hot_tests.rs"] mod tests;