tidaldb/tidal/src/db/signals.rs

//! Signal write/read operations on `TidalDb`.

use std::collections::HashMap;

use super::{
    TidalDb,
    metadata::{
        UserStateSignal, classify_user_state_signal, is_positive_engagement_signal,
        serialize_metadata,
    },
};
use crate::{
    entities::{HardNegIndex, RelationshipType},
    governance::SignalScope,
    schema::{EntityId, EntityKind, TidalError, Timestamp, Window},
};

impl TidalDb {
    /// Write (or overwrite) item metadata.
    ///
    /// Routes through [`write_item_with_metadata`](Self::write_item_with_metadata)
    /// so it applies the SAME metadata size validation and in-memory index
    /// population (bitmap/range/universe/creator-items, text, suggestions) as the
    /// primary write path. A previous direct-to-storage `write_item` bypassed
    /// both: items written through it were durable but invisible to RETRIEVE and
    /// uncapped in metadata size — a latent inconsistency between two "write an
    /// item" entry points. There is now only one indexing path.
    ///
    /// # Errors
    ///
    /// - `TidalError::Internal` if no storage backend is wired (use `with_schema()`).
    /// - `TidalError::invalid_input` if the metadata exceeds the size limits.
    /// - `TidalError::Storage` on storage engine failure.
    pub fn write_item(
        &self,
        id: EntityId,
        metadata: &HashMap<String, String>,
    ) -> crate::Result<()> {
        self.write_item_with_metadata(id, metadata)
    }

    /// Persist item metadata directly to the items storage backend with no
    /// validation or index population.
    ///
    /// Internal-only: this is the raw durable-write step that
    /// [`write_item_with_metadata`](Self::write_item_with_metadata) calls AFTER
    /// validating sizes and materializing defaults. No public path may bypass the
    /// indexing wrapper via this method.
    pub(super) fn persist_item_metadata(
        &self,
        id: EntityId,
        metadata: &HashMap<String, String>,
    ) -> crate::Result<()> {
        use crate::storage::{Tag, encode_key};
        self.require_writeable("write_item")?;

        let storage = self.storage()?;
        let key = encode_key(id, Tag::Meta, b"");
        let value = serialize_metadata(metadata);
        storage
            .items_engine()
            .put(&key, &value)
            .map_err(TidalError::from)
    }

    /// Record a signal event for an entity.
    ///
    /// Atomically:
    /// 1. Appends the event to the WAL (WAL-first durability).
    /// 2. Updates the in-memory decay score (hot tier).
    /// 3. Updates the in-memory windowed counter (warm tier).
    ///
    /// # Errors
    ///
    /// - `TidalError::Internal` if no ledger is wired (use `with_schema()`).
    /// - `TidalError::Schema` if `signal_type` is not defined in the schema.
    /// - `TidalError::Durability` if the WAL write fails.
    #[tracing::instrument(skip_all, fields(signal_type, entity_id = entity_id.as_u64(), weight))]
    pub fn signal(
        &self,
        signal_type: &str,
        entity_id: EntityId,
        weight: f64,
        timestamp: Timestamp,
    ) -> crate::Result<()> {
        self.require_writeable("signal")?;
        if !weight.is_finite() {
            return Err(TidalError::invalid_input(
                "signal weight must be finite (NaN and Inf are not allowed)",
            ));
        }

        // Backup-in-progress guard: reject writes while the data directory
        // is being copied. Acquire ordering ensures we see the flag set by
        // create_backup's Release store.
        if self
            .backup_in_progress
            .load(std::sync::atomic::Ordering::Acquire)
        {
            return Err(TidalError::Backpressure {
                retry_after_ms: 500,
            });
        }

        #[cfg(feature = "metrics")]
        let write_start = std::time::Instant::now();

        // Backpressure check: inspect WAL channel depth before enqueuing.
        // This is O(1) — crossbeam::channel::bounded::len() is atomic.
        if let Ok(guard) = self.wal.lock()
            && let Some(wal) = guard.as_ref()
        {
            let queue_depth = wal.channel_len();
            if queue_depth >= self.backpressure_config.queue_depth_threshold {
                tracing::warn!(
                    queue_depth,
                    threshold = self.backpressure_config.queue_depth_threshold,
                    "WAL backpressure: rejecting signal write"
                );
                return Err(TidalError::Backpressure {
                    retry_after_ms: self.backpressure_config.retry_after_ms,
                });
            }
        }

        let result = self
            .ledger()?
            .record_signal(signal_type, entity_id, weight, timestamp);

        #[cfg(feature = "metrics")]
        {
            use std::sync::atomic::Ordering;
            let elapsed_us = write_start.elapsed().as_micros() as u64;
            self.metrics
                .signal_writes_total
                .fetch_add(1, Ordering::Relaxed);
            self.metrics.signal_write_latency.observe(elapsed_us);
        }

        result
    }

    /// Record a signal event with an explicit [`SignalScope`].
    ///
    /// [`signal`](Self::signal) writes [`SignalScope::Local`]; this is the
    /// opt-in path for community/session/agent-scoped writes. The event is
    /// always durably logged with its v3 governance envelope, but only
    /// local-scope events affect the local profile — community-scoped events are
    /// logged for replication and community aggregation without perturbing local
    /// ranking state (the "local profile remains intact" invariant).
    ///
    /// `share_policy_version` is recorded in the event's provenance; pass `0`
    /// when no share policy governs the write.
    ///
    /// # Errors
    ///
    /// - `TidalError::Internal` if no ledger is wired (use `with_schema()`).
    /// - `TidalError::Schema` if `signal_type` is not defined in the schema.
    /// - `TidalError::Durability` if the WAL write fails.
    /// - `TidalError::Backpressure` if the WAL queue is saturated or a backup is in progress.
    pub fn signal_scoped(
        &self,
        signal_type: &str,
        entity_id: EntityId,
        weight: f64,
        timestamp: Timestamp,
        scope: SignalScope,
        share_policy_version: u16,
    ) -> crate::Result<()> {
        self.require_writeable("signal_scoped")?;
        if !weight.is_finite() {
            return Err(TidalError::invalid_input(
                "signal weight must be finite (NaN and Inf are not allowed)",
            ));
        }

        // Backup-in-progress guard (mirrors `signal`).
        if self
            .backup_in_progress
            .load(std::sync::atomic::Ordering::Acquire)
        {
            return Err(TidalError::Backpressure {
                retry_after_ms: 500,
            });
        }

        // Backpressure check on WAL channel depth (mirrors `signal`).
        if let Ok(guard) = self.wal.lock()
            && let Some(wal) = guard.as_ref()
        {
            let queue_depth = wal.channel_len();
            if queue_depth >= self.backpressure_config.queue_depth_threshold {
                return Err(TidalError::Backpressure {
                    retry_after_ms: self.backpressure_config.retry_after_ms,
                });
            }
        }

        self.ledger()?.record_signal_scoped(
            signal_type,
            entity_id,
            weight,
            timestamp,
            scope,
            share_policy_version,
            0, // membership_epoch: community-membership writes go via signal_for_community
        )
    }

    /// Read the current decay score for an entity-signal pair.
    ///
    /// Applies lazy decay from the stored timestamp to the current wall-clock
    /// time. Returns `None` if no signals have been recorded.
    ///
    /// `decay_rate_idx` selects the lambda index from the signal definition.
    /// For exponential signals with one rate, use `0`.
    ///
    /// # Errors
    ///
    /// - `TidalError::Internal` if no ledger is wired.
    /// - `TidalError::Schema` if `signal_type` is not defined.
    pub fn read_decay_score(
        &self,
        entity_id: EntityId,
        signal_type: &str,
        decay_rate_idx: usize,
    ) -> crate::Result<Option<f64>> {
        self.ledger()?
            .read_decay_score(entity_id, signal_type, decay_rate_idx)
    }

    /// Read the windowed event count for an entity-signal pair.
    ///
    /// Returns `0` if no signals have been recorded.
    ///
    /// # Errors
    ///
    /// - `TidalError::Internal` if no ledger is wired.
    /// - `TidalError::Schema` if `signal_type` is not defined.
    pub fn read_windowed_count(
        &self,
        entity_id: EntityId,
        signal_type: &str,
        window: Window,
    ) -> crate::Result<u64> {
        self.ledger()?
            .read_windowed_count(entity_id, signal_type, window)
    }

    /// Read the velocity (events per second) for an entity-signal-window.
    ///
    /// Velocity = `windowed_count / window_duration_seconds`.
    /// Returns `0.0` for `AllTime` windows or if no signals recorded.
    ///
    /// # Errors
    ///
    /// - `TidalError::Internal` if no ledger is wired.
    /// - `TidalError::Schema` if `signal_type` is not defined.
    pub fn read_velocity(
        &self,
        entity_id: EntityId,
        signal_type: &str,
        window: Window,
    ) -> crate::Result<f64> {
        self.ledger()?.read_velocity(entity_id, signal_type, window)
    }

    /// Records a signal with user context, updating the interaction ledger, seen state,
    /// preference vectors, and cohort signal state in-memory.
    ///
    /// In addition to updating the signal ledger, this method:
    /// 1. Hard negatives: if the signal is skip/hide/dislike/block, records
    ///    a hard negative for the (user, item) pair.
    /// 2. Interaction weight: if `for_user` is provided, updates the
    ///    (user, creator) interaction weight.
    /// 3. Seen tracking: if `for_user` is provided, marks the item as seen.
    /// 4. Preference vector: for positive engagement signals (like, share,
    ///    completion), looks up the item's embedding from durable storage and
    ///    blends it into the user's preference vector via EMA.
    /// 5. Cohort attribution (M6): resolves the user's cohort memberships
    ///    and records the signal into each matching cohort's signal state.
    ///
    /// # Preference vector updates
    ///
    /// The update triggers when all three conditions are met:
    /// - The signal type is a positive engagement signal ("like", "share", "completion").
    /// - `for_user` is `Some` (the acting user is known).
    /// - The item has a stored embedding in the entity store (written via
    ///   `insert_embedding` or `update_embedding` during item ingestion).
    ///
    /// The embedding is read from the first Item embedding slot declared in the
    /// schema. If no schema or no embedding slot is declared, falls back to the
    /// slot name "content". If the lookup fails (no embedding stored, storage
    /// error), the preference update is silently skipped -- the base signal is
    /// still recorded.
    ///
    /// # Durability
    ///
    /// The base signal (entity, type, weight, timestamp) is WAL-backed and survives crashes.
    /// The per-user side effects do NOT live in the WAL (the signal record carries no
    /// `for_user` identity), so each one is made crash-recoverable on its own path:
    ///
    /// - **Hard negatives** (skip/hide/dislike/block) are written write-through as durable
    ///   `Tag::HardNeg` rows (zero loss window) and restored on open via
    ///   [`HardNegIndex::restore`](crate::entities::HardNegIndex::restore).
    /// - **Interaction weights** are persisted write-through as a merged
    ///   `RelationshipType::InteractionWeight` edge carrying the decayed running score and
    ///   last-update timestamp, so `rebuild_entity_state` reproduces the ledger on open.
    /// - **Completion / save / like** state is written write-through as durable
    ///   `Tag::UserState` rows and restored on open, so the `InProgress` filter and
    ///   `DateSaved` sort survive a crash.
    /// - **Preference vectors** are periodically checkpointed (bounded post-crash loss
    ///   window) and restored on open.
    ///
    /// Seen state from regular views/likes is intentionally ephemeral -- users should see
    /// content again after a restart. Only explicit hides (via `write_relationship` with
    /// `RelationshipType::Hide`) survive restarts as "seen".
    ///
    /// # Errors
    ///
    /// Returns errors from the underlying `signal()` method.
    pub fn signal_with_context(
        &self,
        signal_type: &str,
        entity_id: EntityId,
        weight: f64,
        timestamp: Timestamp,
        for_user: Option<u64>,
        creator_id: Option<u64>,
    ) -> crate::Result<()> {
        self.require_writeable("signal_with_context")?;
        // Record the base signal.
        self.signal(signal_type, entity_id, weight, timestamp)?;

        // Signal dispatch: side effects based on signal type and context.
        if let Some(user_id) = for_user {
            let item_u32 = entity_id.as_u64() as u32;
            // Durable-write target for the write-through side effects below.
            // `None` in ephemeral mode (no persistence; in-memory state is the
            // whole story and there is nothing to crash-recover).
            let storage = self.storage.as_ref();

            // 1. Hard negatives. Write-through: a durable Tag::HardNeg row is
            //    persisted with the SAME durability class as the base signal so a
            //    skip/dislike survives a hard crash with zero loss window, then
            //    restored into the in-memory bitmap on open.
            if HardNegIndex::is_hard_neg_signal(signal_type) {
                self.hard_negatives.add(user_id, item_u32);
                if let Some(sb) = storage
                    && let Err(e) =
                        self.hard_negatives
                            .persist(sb.items_engine(), user_id, item_u32)
                {
                    tracing::error!(
                        user_id,
                        item = item_u32,
                        error = %e,
                        "failed to persist hard-negative durable row; \
                         it would not survive a crash"
                    );
                }
            }

            // 2. Seen tracking.
            self.user_state.mark_seen(user_id, item_u32);

            // 3. Interaction weight: if creator is known, update the
            //    (user, creator) interaction strength, then persist the decayed
            //    running score + last-update as a merged InteractionWeight edge so
            //    `rebuild_entity_state` reproduces the ledger on open.
            if let Some(cid) = creator_id {
                self.interaction_ledger
                    .record(user_id, cid, weight, timestamp.as_nanos());
                self.persist_interaction_edge(user_id, cid);
            }

            // 3b. Completion / save / like: drive the UserStateIndex maps the
            //     InProgress filter and DateSaved sort read. Write-through so they
            //     survive a crash. Signals do not carry a completion ratio, so the
            //     signal weight IS the completion ratio (documented choice).
            if let Some(kind) = classify_user_state_signal(signal_type) {
                self.persist_user_state_signal(
                    kind, user_id, entity_id, item_u32, weight, timestamp,
                );
            }

            // 4. Preference vector + co-engagement: for positive engagement signals,
            //    look up the item's embedding and blend into the user's taste vector,
            //    and record co-occurrence with user's recent positively-engaged items.
            if self.is_positive_engagement(signal_type) {
                self.try_update_preference_vector(user_id, entity_id);
                self.co_engagement.record_positive(user_id, entity_id);
            }

            // 5. Per-user signal tracking for social-graph-scoped trending.
            if let Ok(ledger) = self.ledger()
                && let Ok(type_id) = ledger.resolve_signal_type(signal_type)
            {
                self.user_signal_index
                    .record(user_id, entity_id, type_id, timestamp.as_nanos());
            }

            // 6. Cohort attribution (M6): read user metadata, resolve cohorts,
            //    record signal into each matching cohort's ledger.
            self.try_cohort_attribution(signal_type, entity_id, weight, timestamp, user_id);
        }

        Ok(())
    }

    /// Persist the current `(user, creator)` interaction weight as a durable
    /// `RelationshipType::InteractionWeight` edge so `rebuild_entity_state`
    /// reproduces the ledger on open.
    ///
    /// Reads back the raw running score + last-update timestamp (post-`record`)
    /// and overwrites the edge with those exact values. Because the rebuild path
    /// replays a stored edge through `InteractionLedger::record` into a fresh
    /// ledger — folding the persisted score as one in-order event at
    /// `last_update_ns` — the rebuilt entry equals this one within decay
    /// tolerance. Best-effort: a storage error is logged, not propagated, so the
    /// base signal still succeeds.
    fn persist_interaction_edge(&self, user_id: u64, creator_id: u64) {
        use crate::entities::relationship::{
            encode_relationship_key, serialize_relationship_value,
        };
        let Some((score, last_update_ns)) = self.interaction_ledger.raw_entry(user_id, creator_id)
        else {
            return;
        };
        let Some(storage) = self.storage.as_ref() else {
            return;
        };
        let key = encode_relationship_key(
            EntityId::new(user_id),
            RelationshipType::InteractionWeight,
            EntityId::new(creator_id),
        );
        let value = serialize_relationship_value(score, Timestamp::from_nanos(last_update_ns));
        if let Err(e) = storage.users_engine().put(&key, &value) {
            tracing::error!(
                user_id,
                creator_id,
                error = %e,
                "failed to persist interaction-weight edge; \
                 it would reset to zero on restart"
            );
        }
    }

    /// Drive (and durably persist) the `UserStateIndex` map for a
    /// completion / save / like signal.
    ///
    /// The signal weight is used as the completion ratio (signals carry no
    /// explicit ratio). Write-through so the `InProgress` filter / `DateSaved`
    /// sort survive a crash; best-effort persistence (logged, not propagated).
    fn persist_user_state_signal(
        &self,
        kind: UserStateSignal,
        user_id: u64,
        entity_id: EntityId,
        item_u32: u32,
        weight: f64,
        timestamp: Timestamp,
    ) {
        let Some(storage) = self.storage.as_ref() else {
            // Ephemeral mode: update in-memory state only (nothing to recover).
            match kind {
                UserStateSignal::Completion => {
                    self.user_state
                        .record_completion(user_id, entity_id.as_u64(), weight);
                }
                UserStateSignal::Save => {
                    self.user_state
                        .add_save_timestamped(user_id, item_u32, timestamp.as_nanos());
                }
                UserStateSignal::Like => self.user_state.add_like(user_id, item_u32),
            }
            return;
        };
        let engine = storage.items_engine();
        let result = match kind {
            UserStateSignal::Completion => self.user_state.record_completion_durable(
                engine,
                user_id,
                entity_id.as_u64(),
                weight,
            ),
            UserStateSignal::Save => {
                self.user_state
                    .add_save_durable(engine, user_id, item_u32, timestamp.as_nanos())
            }
            UserStateSignal::Like => self.user_state.add_like_durable(engine, user_id, item_u32),
        };
        if let Err(e) = result {
            tracing::error!(
                user_id,
                item = item_u32,
                error = %e,
                "failed to persist user-state durable row; \
                 InProgress/DateSaved would not survive a crash"
            );
        }
    }

    /// Best-effort cohort attribution: resolves user cohorts and records the signal
    /// into each matching cohort's signal ledger.
    ///
    /// Failures are silently ignored -- cohort attribution must not fail the
    /// primary signal write path.
    fn try_cohort_attribution(
        &self,
        signal_type: &str,
        entity_id: EntityId,
        weight: f64,
        timestamp: Timestamp,
        user_id: u64,
    ) {
        // Skip if no cohorts are defined.
        if self.cohort_registry.is_empty() {
            return;
        }

        // Resolve signal type to ID. If unknown, skip (should not happen if
        // the base signal succeeded, but defensive).
        let Ok(type_id) = self.cohort_ledger.resolve_signal_type(signal_type) else {
            return;
        };

        // Read user metadata from users_engine.
        let Ok(Some(metadata)) = self.get_user_metadata(EntityId::new(user_id)) else {
            return; // No metadata or storage error -- skip cohort attribution.
        };

        // Resolve cohort memberships.
        let cohorts = self.cohort_resolver.resolve(user_id, &metadata);

        // Record the signal into each matching cohort's ledger.
        let ts_ns = timestamp.as_nanos();
        for cohort_name in &cohorts {
            self.cohort_ledger
                .record(cohort_name, entity_id, type_id, weight, ts_ns);
        }
    }

    /// Whether `signal_type` is a positive-engagement signal — one that should
    /// fold the signalled item's embedding into the acting user's preference
    /// vector.
    ///
    /// Schema-driven with a back-compat fallback: when the schema declares any
    /// positive-engagement signal (`Schema::positive_engagement_enabled`), the
    /// declarations are authoritative — a signal is positive iff its
    /// `SignalTypeDef::is_positive_engagement()` is set. When the schema declares
    /// none (or there is no schema), the engine falls back to the built-in
    /// default set so apps predating the flag keep their prior behavior.
    pub(super) fn is_positive_engagement(&self, signal_type: &str) -> bool {
        match self.schema_def.as_ref() {
            Some(schema) if schema.positive_engagement_enabled() => schema
                .signal(signal_type)
                .is_some_and(crate::schema::SignalTypeDef::is_positive_engagement),
            _ => is_positive_engagement_signal(signal_type),
        }
    }

    /// The embedding slot name used for item content embeddings.
    ///
    /// Single source of truth for every path that reads or writes an item
    /// embedding (`write_item_embedding`, `read_item_embedding`,
    /// `try_update_preference_vector`, and the retrieve-time preference boost):
    /// the schema's first Item embedding slot, falling back to "content" when no
    /// schema declares one. Keeping all paths on this resolver is what makes the
    /// preference-vector chain coherent regardless of how an app names its slot.
    pub(super) fn item_embedding_slot(&self) -> &str {
        self.schema_def
            .as_ref()
            .and_then(|s| {
                s.embedding_slots()
                    .iter()
                    .find(|slot| slot.entity_kind == EntityKind::Item)
                    .map(|slot| slot.name.as_str())
            })
            .unwrap_or("content")
    }

    /// Attempt to update a user's preference vector from the item's stored embedding.
    ///
    /// Reads the item's embedding from durable storage (entity store) and blends it
    /// into the user's preference vector via `PreferenceVectors::update()`. This is
    /// a best-effort operation: if the item has no embedding, no storage is wired, or
    /// the embedding cannot be deserialized, the update is silently skipped.
    ///
    /// The slot name is resolved by [`Self::item_embedding_slot`] (the schema's
    /// first Item embedding slot, falling back to "content").
    pub(super) fn try_update_preference_vector(&self, user_id: u64, entity_id: EntityId) {
        // Determine which embedding slot to read.
        let slot_name = self.item_embedding_slot();

        // Read the item's embedding from durable storage.
        let Some(storage) = self.storage.as_ref() else {
            return;
        };
        let key = crate::storage::vector::embedding_store_key(entity_id, slot_name);
        let embedding_bytes = match storage.items_engine().get(&key) {
            Ok(Some(bytes)) => bytes,
            Ok(None) => {
                tracing::debug!(
                    entity_id = entity_id.as_u64(),
                    slot = slot_name,
                    "preference vector update skipped: item has no stored embedding"
                );
                return;
            }
            Err(e) => {
                tracing::debug!(
                    entity_id = entity_id.as_u64(),
                    error = %e,
                    "preference vector update skipped: storage read failed"
                );
                return;
            }
        };

        // Deserialize the embedding.
        let embedding = match crate::storage::vector::deserialize_embedding(&embedding_bytes) {
            Ok(v) => v,
            Err(e) => {
                tracing::debug!(
                    entity_id = entity_id.as_u64(),
                    error = %e,
                    "preference vector update skipped: embedding deserialization failed"
                );
                return;
            }
        };

        // Blend into the user's preference vector.
        if !self.preference_vectors.update(user_id, &embedding) {
            tracing::debug!(
                user_id,
                entity_id = entity_id.as_u64(),
                embedding_dim = embedding.len(),
                "preference vector update skipped: dimension mismatch"
            );
        }
    }
}