tidaldb/tidal/src/signals/hot.rs

//! Cache-line-aligned, lock-free per-entity signal state for the hot path.
//!
//! `HotSignalState` is the single hottest struct in `TidalDB`'s ranking pipeline.
//! Every ranking query touches it for every candidate entity. The design is
//! driven by three constraints:
//!
//! 1. **Cache-line alignment** -- one entity's signal state never shares a cache
//!    line with another, eliminating false sharing under concurrent reads.
//! 2. **Lock-free updates** -- signal ingestion uses CAS loops on individual
//!    decay scores, so readers are never blocked by writers.
//! 3. **O(1) running decay** -- scores are maintained incrementally via the
//!    identity `S(t) = S(prev) * exp(-lambda * dt) + weight`, avoiding
//!    re-summation of the full event history.
//!
//! # Memory ordering rationale
//!
//! - `last_update_ns` loads use `Acquire` to establish happens-before with the
//!   writer's `Release` store, ensuring all prior score CAS operations are visible.
//! - `last_update_ns` CAS success uses `Release` to make all prior score writes
//!   visible to readers who subsequently `Acquire` the timestamp.
//! - `last_update_ns` CAS failure uses `Relaxed` because we discard the result
//!   on failure (a concurrent writer already advanced the timestamp).
//! - `decay_scores[i]` loads use `Acquire` to see the latest CAS'd value.
//! - `decay_scores[i]` CAS success uses `AcqRel` -- `Release` makes the new
//!   score visible, `Acquire` loads the freshest competing value.
//! - `decay_scores[i]` CAS failure uses `Acquire` to load the freshest
//!   competing write for the next retry iteration.

use std::{
    fmt,
    sync::atomic::{AtomicU64, Ordering},
};

use crate::signals::decay::{NANOS_PER_SEC, forward_decay_step};

/// Maximum number of concurrent decay rates tracked per entity-signal pair.
pub const MAX_DECAY_RATES: usize = 3;

/// Bit 0 of `flags`: velocity tracking is enabled for this signal.
const FLAG_VELOCITY_ENABLED: u16 = 0x0001;

/// Per-entity, per-signal-type hot state for the ranking pipeline.
///
/// Fits exactly one cache line (64 bytes). All mutable fields are atomic,
/// enabling lock-free concurrent reads and writes. Immutable fields
/// (`entity_id`, `signal_type_id`, `flags`) are set at construction and
/// never modified.
#[repr(C, align(64))]
pub struct HotSignalState {
    /// Immutable after construction. Identifies the entity this state belongs to.
    entity_id: u64,
    /// Nanosecond timestamp of the most recent in-order signal event processed.
    /// Updated only when a new event's timestamp exceeds the current value.
    last_update_ns: AtomicU64,
    /// Immutable after construction. Identifies the signal type.
    signal_type_id: u16,
    /// Immutable after construction. Bit flags (see `FLAG_VELOCITY_ENABLED`).
    flags: u16,
    /// Padding to maintain field alignment.
    _pad0: [u8; 4],
    /// Running exponentially-decayed scores, one per decay rate.
    /// Stored as `f64::to_bits()` for atomic CAS.
    decay_scores: [AtomicU64; 3],
    /// Padding to fill the cache line to exactly 64 bytes.
    _pad1: [u8; 16],
}

// Compile-time assertions: struct must be exactly one cache line.
const _SIZE: () = assert!(std::mem::size_of::<HotSignalState>() == 64);
const _ALIGN: () = assert!(std::mem::align_of::<HotSignalState>() == 64);

impl HotSignalState {
    /// Creates a new zeroed state with velocity tracking disabled.
    #[must_use]
    pub const fn new(entity_id: u64, signal_type_id: u16) -> Self {
        Self::with_flags(entity_id, signal_type_id, false)
    }

    /// Creates a new zeroed state with explicit velocity flag.
    #[must_use]
    pub const fn with_flags(entity_id: u64, signal_type_id: u16, velocity_enabled: bool) -> Self {
        let flags = if velocity_enabled {
            FLAG_VELOCITY_ENABLED
        } else {
            0
        };
        Self {
            entity_id,
            last_update_ns: AtomicU64::new(0),
            signal_type_id,
            flags,
            _pad0: [0; 4],
            decay_scores: [
                AtomicU64::new(0_f64.to_bits()),
                AtomicU64::new(0_f64.to_bits()),
                AtomicU64::new(0_f64.to_bits()),
            ],
            _pad1: [0; 16],
        }
    }

    /// Returns the entity ID this state belongs to. Immutable after construction.
    #[must_use]
    pub const fn entity_id(&self) -> u64 {
        self.entity_id
    }

    /// Returns the signal type ID. Immutable after construction.
    #[must_use]
    pub const fn signal_type_id(&self) -> u16 {
        self.signal_type_id
    }

    /// Returns whether velocity tracking is enabled for this signal.
    #[must_use]
    pub const fn velocity_enabled(&self) -> bool {
        self.flags & FLAG_VELOCITY_ENABLED != 0
    }

    /// Records a signal event, updating all decay scores atomically.
    ///
    /// Both temporal orderings are handled by the canonical decay kernel
    /// [`forward_decay_step`](crate::signals::decay::forward_decay_step), the
    /// single source of truth for forward-decay math (`CODING_GUIDELINES` §3):
    /// - **In-order** (`event_time_ns >= last_update_ns`): decays existing scores
    ///   by `dt` then adds `weight`. Advances the timestamp.
    /// - **Out-of-order** (`event_time_ns < last_update_ns`): pre-decays the
    ///   weight by the event's age, then adds the reduced weight. Does NOT
    ///   regress the timestamp.
    ///
    /// Each decay score is updated via an independent CAS loop, so concurrent
    /// writers on different decay rates do not contend. The kernel is a pure,
    /// allocation-free function, so calling it inside the retry loop is free.
    ///
    /// # CAS-retry consistency
    ///
    /// `last_update_ns` is re-read **inside** each score's retry loop. A CAS
    /// failure on `decay_scores[i]` means a concurrent `on_signal` won the race
    /// and folded its event against the timestamp it observed — and it may also
    /// have advanced `last_update_ns`. Reusing the stale entry-time snapshot for
    /// the `dt`/`age` of the retry would decay the freshly-CAS'd score from the
    /// wrong reference time, double- or under-counting the interval. Re-reading
    /// the timestamp on each attempt keeps the `(old_score, last_update_ns)` pair
    /// the kernel consumes mutually consistent. The kernel is pure and
    /// allocation-free, so the extra `Acquire` load on a (rare) contended retry
    /// is cheap.
    pub fn on_signal(&self, weight: f64, event_time_ns: u64, lambdas: &[f64]) {
        // Snapshot the entry-time timestamp for the in-order advance decision and
        // the final timestamp CAS (both are conditioned on the value we observed
        // *on entry*). The per-score retry loop re-reads the timestamp itself so a
        // contended retry decays the freshest score from the right reference time.
        let entry_last_ns = self.last_update_ns.load(Ordering::Acquire);

        // The advance decision is identical across lambdas (it depends only on
        // event_time_ns vs the entry-time timestamp), so we capture it once to
        // drive the single timestamp CAS below.
        let advance_timestamp = event_time_ns >= entry_last_ns;

        for (i, &lambda) in lambdas.iter().take(MAX_DECAY_RATES).enumerate() {
            loop {
                // Re-read both the timestamp and the score each attempt so the
                // pair the kernel folds against stays consistent under a losing
                // CAS (see "CAS-retry consistency" above).
                let last_ns = self.last_update_ns.load(Ordering::Acquire);
                let old_bits = self.decay_scores[i].load(Ordering::Acquire);
                let old_score = f64::from_bits(old_bits);
                let step = forward_decay_step(old_score, last_ns, event_time_ns, lambda, weight);
                debug_assert!(step.new_score >= 0.0);
                if self.decay_scores[i]
                    .compare_exchange_weak(
                        old_bits,
                        step.new_score.to_bits(),
                        Ordering::AcqRel,
                        Ordering::Acquire,
                    )
                    .is_ok()
                {
                    break;
                }
            }
        }

        if advance_timestamp {
            // Crash point lives on the in-order path only, between the score
            // updates and the timestamp advance -- exactly where it was before
            // the kernel refactor, so crash-recovery semantics are unchanged.
            #[cfg(any(test, feature = "test-utils"))]
            crate::testing::crash_injector::check_crash_point(
                crate::testing::CrashPoint::SignalAggregationUpdate,
            );

            // In-order: advance the timestamp. The CAS is conditioned on the
            // entry-time snapshot; failure is acceptable -- a concurrent writer
            // already pushed the timestamp to (or past) event_time_ns.
            let _ = self.last_update_ns.compare_exchange(
                entry_last_ns,
                event_time_ns,
                Ordering::Release,
                Ordering::Relaxed,
            );
        }
        // Out-of-order: do NOT update last_update_ns -- the timestamp must not
        // regress (CODING_GUIDELINES §3).
    }

    /// Returns the current decayed score for a given decay rate index and query time.
    ///
    /// The stored score is decayed forward from `last_update_ns` to `query_time_ns`.
    /// If `query_time_ns` is in the past relative to the last update, no additional
    /// decay is applied (dt clamped to zero).
    ///
    /// Out-of-bounds `decay_rate_idx` is saturated to `MAX_DECAY_RATES - 1` to
    /// avoid panicking on the hot path. The saturation is unconditional (prod
    /// and test agree); there is deliberately no `debug_assert!` that would
    /// panic in debug builds on an index the doc promises to clamp.
    ///
    /// # Concurrency
    ///
    /// Reads `last_update_ns` and the score without a lock. The score may
    /// reflect a slightly different timestamp than `query_time_ns` if a concurrent
    /// `on_signal` is in flight. This is intentional — approximate reads are
    /// acceptable for ranking. Callers must not rely on the score being exactly
    /// consistent with `query_time_ns`.
    #[must_use]
    #[allow(clippy::cast_precision_loss)]
    pub fn current_score(&self, decay_rate_idx: usize, query_time_ns: u64, lambda: f64) -> f64 {
        let idx = decay_rate_idx.min(MAX_DECAY_RATES - 1);

        let last_ns = self.last_update_ns.load(Ordering::Acquire);
        let stored = f64::from_bits(self.decay_scores[idx].load(Ordering::Acquire));
        let dt_secs = if query_time_ns >= last_ns {
            (query_time_ns - last_ns) as f64 / NANOS_PER_SEC
        } else {
            0.0
        };
        let score = stored * (-lambda * dt_secs).exp();
        score.max(0.0)
    }

    /// Returns the raw stored score for a given decay rate index, without
    /// applying any additional time-based decay.
    ///
    /// Out-of-bounds `decay_rate_idx` is saturated to `MAX_DECAY_RATES - 1`
    /// unconditionally (prod and test agree); there is no `debug_assert!` that
    /// would panic on an index the doc promises to clamp.
    #[must_use]
    pub fn stored_score(&self, decay_rate_idx: usize) -> f64 {
        let idx = decay_rate_idx.min(MAX_DECAY_RATES - 1);
        f64::from_bits(self.decay_scores[idx].load(Ordering::Acquire))
    }

    /// Returns the nanosecond timestamp of the most recent in-order signal event.
    #[must_use]
    pub fn last_update_ns(&self) -> u64 {
        self.last_update_ns.load(Ordering::Acquire)
    }

    /// Force-set the decay score at the given lambda index.
    ///
    /// Used by reconciliation to apply merged CRDT state. Bypasses the
    /// CAS-loop increment pattern for a direct write.
    ///
    /// # Ordering
    ///
    /// Uses `Release` so that any subsequent `Acquire` load on the same
    /// index sees the updated score.
    pub(crate) fn force_set_score(&self, lambda_idx: usize, score: f64) {
        if lambda_idx < MAX_DECAY_RATES {
            self.decay_scores[lambda_idx].store(score.to_bits(), Ordering::Release);
        }
    }

    /// Force-set the last-update timestamp.
    ///
    /// Used by reconciliation to align the timestamp with merged CRDT state.
    ///
    /// # Ordering
    ///
    /// Uses `Release` so that any subsequent `Acquire` load sees the
    /// updated timestamp along with any previously force-set scores.
    pub(crate) fn force_set_last_update_ns(&self, ts_ns: u64) {
        self.last_update_ns.store(ts_ns, Ordering::Release);
    }

    /// Restores state from durable storage during crash recovery or cold start.
    ///
    /// Scores are stored first, then the timestamp is stored last with `Release`
    /// ordering. This ensures any reader who sees the new timestamp via `Acquire`
    /// will also see all the restored scores.
    pub fn restore(&self, last_update_ns: u64, scores: &[f64]) {
        for (i, &score) in scores.iter().take(MAX_DECAY_RATES).enumerate() {
            self.decay_scores[i].store(score.to_bits(), Ordering::Release);
        }
        // Timestamp stored LAST so readers see scores before timestamp.
        self.last_update_ns.store(last_update_ns, Ordering::Release);
    }
}

#[allow(clippy::missing_fields_in_debug)]
impl fmt::Debug for HotSignalState {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("HotSignalState")
            .field("entity_id", &self.entity_id)
            .field("signal_type_id", &self.signal_type_id)
            .field("velocity_enabled", &self.velocity_enabled())
            .field("last_update_ns", &self.last_update_ns())
            .field("score[0]", &self.stored_score(0))
            .field("score[1]", &self.stored_score(1))
            .field("score[2]", &self.stored_score(2))
            .finish()
    }
}

#[cfg(test)]
#[allow(
    clippy::unwrap_used,
    clippy::float_cmp,
    clippy::cast_sign_loss,
    clippy::cast_precision_loss
)]
#[path = "hot_tests.rs"]
mod tests;