//! Signal write/read operations on `TidalDb`. use std::collections::HashMap; use super::{ TidalDb, metadata::{is_positive_engagement_signal, serialize_metadata}, }; use crate::{ entities::HardNegIndex, governance::SignalScope, schema::{EntityId, EntityKind, TidalError, Timestamp, Window}, }; impl TidalDb { /// Write (or overwrite) item metadata. /// /// Routes through [`write_item_with_metadata`](Self::write_item_with_metadata) /// so it applies the SAME metadata size validation and in-memory index /// population (bitmap/range/universe/creator-items, text, suggestions) as the /// primary write path. A previous direct-to-storage `write_item` bypassed /// both: items written through it were durable but invisible to RETRIEVE and /// uncapped in metadata size — a latent inconsistency between two "write an /// item" entry points. There is now only one indexing path. /// /// # Errors /// /// - `TidalError::Internal` if no storage backend is wired (use `with_schema()`). /// - `TidalError::invalid_input` if the metadata exceeds the size limits. /// - `TidalError::Storage` on storage engine failure. pub fn write_item( &self, id: EntityId, metadata: &HashMap, ) -> crate::Result<()> { self.write_item_with_metadata(id, metadata) } /// Persist item metadata directly to the items storage backend with no /// validation or index population. /// /// Internal-only: this is the raw durable-write step that /// [`write_item_with_metadata`](Self::write_item_with_metadata) calls AFTER /// validating sizes and materializing defaults. No public path may bypass the /// indexing wrapper via this method. pub(super) fn persist_item_metadata( &self, id: EntityId, metadata: &HashMap, ) -> crate::Result<()> { use crate::storage::{Tag, encode_key}; self.require_writeable("write_item")?; let storage = self.storage()?; let key = encode_key(id, Tag::Meta, b""); let value = serialize_metadata(metadata); storage .items_engine() .put(&key, &value) .map_err(TidalError::from) } /// Record a signal event for an entity. /// /// Atomically: /// 1. Appends the event to the WAL (WAL-first durability). /// 2. Updates the in-memory decay score (hot tier). /// 3. Updates the in-memory windowed counter (warm tier). /// /// # Errors /// /// - `TidalError::Internal` if no ledger is wired (use `with_schema()`). /// - `TidalError::Schema` if `signal_type` is not defined in the schema. /// - `TidalError::Durability` if the WAL write fails. #[tracing::instrument(skip_all, fields(signal_type, entity_id = entity_id.as_u64(), weight))] pub fn signal( &self, signal_type: &str, entity_id: EntityId, weight: f64, timestamp: Timestamp, ) -> crate::Result<()> { self.require_writeable("signal")?; if !weight.is_finite() { return Err(TidalError::invalid_input( "signal weight must be finite (NaN and Inf are not allowed)", )); } // Backup-in-progress guard: reject writes while the data directory // is being copied. Acquire ordering ensures we see the flag set by // create_backup's Release store. if self .backup_in_progress .load(std::sync::atomic::Ordering::Acquire) { return Err(TidalError::Backpressure { retry_after_ms: 500, }); } #[cfg(feature = "metrics")] let write_start = std::time::Instant::now(); // Backpressure check: inspect WAL channel depth before enqueuing. // This is O(1) — crossbeam::channel::bounded::len() is atomic. if let Ok(guard) = self.wal.lock() && let Some(wal) = guard.as_ref() { let queue_depth = wal.channel_len(); if queue_depth >= self.backpressure_config.queue_depth_threshold { tracing::warn!( queue_depth, threshold = self.backpressure_config.queue_depth_threshold, "WAL backpressure: rejecting signal write" ); return Err(TidalError::Backpressure { retry_after_ms: self.backpressure_config.retry_after_ms, }); } } let result = self .ledger()? .record_signal(signal_type, entity_id, weight, timestamp); #[cfg(feature = "metrics")] { use std::sync::atomic::Ordering; let elapsed_us = write_start.elapsed().as_micros() as u64; self.metrics .signal_writes_total .fetch_add(1, Ordering::Relaxed); self.metrics.signal_write_latency.observe(elapsed_us); } result } /// Record a signal event with an explicit [`SignalScope`]. /// /// [`signal`](Self::signal) writes [`SignalScope::Local`]; this is the /// opt-in path for community/session/agent-scoped writes. The event is /// always durably logged with its v3 governance envelope, but only /// local-scope events affect the local profile — community-scoped events are /// logged for replication and community aggregation without perturbing local /// ranking state (the "local profile remains intact" invariant). /// /// `share_policy_version` is recorded in the event's provenance; pass `0` /// when no share policy governs the write. /// /// # Errors /// /// - `TidalError::Internal` if no ledger is wired (use `with_schema()`). /// - `TidalError::Schema` if `signal_type` is not defined in the schema. /// - `TidalError::Durability` if the WAL write fails. /// - `TidalError::Backpressure` if the WAL queue is saturated or a backup is in progress. pub fn signal_scoped( &self, signal_type: &str, entity_id: EntityId, weight: f64, timestamp: Timestamp, scope: SignalScope, share_policy_version: u16, ) -> crate::Result<()> { self.require_writeable("signal_scoped")?; if !weight.is_finite() { return Err(TidalError::invalid_input( "signal weight must be finite (NaN and Inf are not allowed)", )); } // Backup-in-progress guard (mirrors `signal`). if self .backup_in_progress .load(std::sync::atomic::Ordering::Acquire) { return Err(TidalError::Backpressure { retry_after_ms: 500, }); } // Backpressure check on WAL channel depth (mirrors `signal`). if let Ok(guard) = self.wal.lock() && let Some(wal) = guard.as_ref() { let queue_depth = wal.channel_len(); if queue_depth >= self.backpressure_config.queue_depth_threshold { return Err(TidalError::Backpressure { retry_after_ms: self.backpressure_config.retry_after_ms, }); } } self.ledger()?.record_signal_scoped( signal_type, entity_id, weight, timestamp, scope, share_policy_version, 0, // membership_epoch: community-membership writes go via signal_for_community ) } /// Read the current decay score for an entity-signal pair. /// /// Applies lazy decay from the stored timestamp to the current wall-clock /// time. Returns `None` if no signals have been recorded. /// /// `decay_rate_idx` selects the lambda index from the signal definition. /// For exponential signals with one rate, use `0`. /// /// # Errors /// /// - `TidalError::Internal` if no ledger is wired. /// - `TidalError::Schema` if `signal_type` is not defined. pub fn read_decay_score( &self, entity_id: EntityId, signal_type: &str, decay_rate_idx: usize, ) -> crate::Result> { self.ledger()? .read_decay_score(entity_id, signal_type, decay_rate_idx) } /// Read the windowed event count for an entity-signal pair. /// /// Returns `0` if no signals have been recorded. /// /// # Errors /// /// - `TidalError::Internal` if no ledger is wired. /// - `TidalError::Schema` if `signal_type` is not defined. pub fn read_windowed_count( &self, entity_id: EntityId, signal_type: &str, window: Window, ) -> crate::Result { self.ledger()? .read_windowed_count(entity_id, signal_type, window) } /// Read the velocity (events per second) for an entity-signal-window. /// /// Velocity = `windowed_count / window_duration_seconds`. /// Returns `0.0` for `AllTime` windows or if no signals recorded. /// /// # Errors /// /// - `TidalError::Internal` if no ledger is wired. /// - `TidalError::Schema` if `signal_type` is not defined. pub fn read_velocity( &self, entity_id: EntityId, signal_type: &str, window: Window, ) -> crate::Result { self.ledger()?.read_velocity(entity_id, signal_type, window) } /// Records a signal with user context, updating the interaction ledger, seen state, /// preference vectors, and cohort signal state in-memory. /// /// In addition to updating the signal ledger, this method: /// 1. Hard negatives: if the signal is skip/hide/dislike/block, records /// a hard negative for the (user, item) pair. /// 2. Interaction weight: if `for_user` is provided, updates the /// (user, creator) interaction weight. /// 3. Seen tracking: if `for_user` is provided, marks the item as seen. /// 4. Preference vector: for positive engagement signals (like, share, /// completion), looks up the item's embedding from durable storage and /// blends it into the user's preference vector via EMA. /// 5. Cohort attribution (M6): resolves the user's cohort memberships /// and records the signal into each matching cohort's signal state. /// /// # Preference vector updates /// /// The update triggers when all three conditions are met: /// - The signal type is a positive engagement signal ("like", "share", "completion"). /// - `for_user` is `Some` (the acting user is known). /// - The item has a stored embedding in the entity store (written via /// `insert_embedding` or `update_embedding` during item ingestion). /// /// The embedding is read from the first Item embedding slot declared in the /// schema. If no schema or no embedding slot is declared, falls back to the /// slot name "content". If the lookup fails (no embedding stored, storage /// error), the preference update is silently skipped -- the base signal is /// still recorded. /// /// # Durability /// /// The base signal (entity, type, weight, timestamp) is WAL-backed and survives crashes. /// User-context side effects (seen state, interaction weights, preference vector updates) /// are reconstructed from durable storage on restart via `rebuild_entity_state`. /// Hard negatives (hide/block) are durably written via `write_relationship()`. /// /// Seen state from regular views/likes is intentionally ephemeral -- users should see /// content again after a restart. Only explicit hides (via `write_relationship` with /// `RelationshipType::Hide`) survive restarts as "seen". /// /// # Errors /// /// Returns errors from the underlying `signal()` method. pub fn signal_with_context( &self, signal_type: &str, entity_id: EntityId, weight: f64, timestamp: Timestamp, for_user: Option, creator_id: Option, ) -> crate::Result<()> { self.require_writeable("signal_with_context")?; // Record the base signal. self.signal(signal_type, entity_id, weight, timestamp)?; // Signal dispatch: side effects based on signal type and context. if let Some(user_id) = for_user { let item_u32 = entity_id.as_u64() as u32; // 1. Hard negatives. if HardNegIndex::is_hard_neg_signal(signal_type) { self.hard_negatives.add(user_id, item_u32); } // 2. Seen tracking. self.user_state.mark_seen(user_id, item_u32); // 3. Interaction weight: if creator is known, update the // (user, creator) interaction strength. if let Some(cid) = creator_id { self.interaction_ledger .record(user_id, cid, weight, timestamp.as_nanos()); } // 4. Preference vector + co-engagement: for positive engagement signals, // look up the item's embedding and blend into the user's taste vector, // and record co-occurrence with user's recent positively-engaged items. if self.is_positive_engagement(signal_type) { self.try_update_preference_vector(user_id, entity_id); self.co_engagement.record_positive(user_id, entity_id); } // 5. Per-user signal tracking for social-graph-scoped trending. if let Ok(ledger) = self.ledger() && let Ok(type_id) = ledger.resolve_signal_type(signal_type) { self.user_signal_index .record(user_id, entity_id, type_id, timestamp.as_nanos()); } // 6. Cohort attribution (M6): read user metadata, resolve cohorts, // record signal into each matching cohort's ledger. self.try_cohort_attribution(signal_type, entity_id, weight, timestamp, user_id); } Ok(()) } /// Best-effort cohort attribution: resolves user cohorts and records the signal /// into each matching cohort's signal ledger. /// /// Failures are silently ignored -- cohort attribution must not fail the /// primary signal write path. fn try_cohort_attribution( &self, signal_type: &str, entity_id: EntityId, weight: f64, timestamp: Timestamp, user_id: u64, ) { // Skip if no cohorts are defined. if self.cohort_registry.is_empty() { return; } // Resolve signal type to ID. If unknown, skip (should not happen if // the base signal succeeded, but defensive). let Ok(type_id) = self.cohort_ledger.resolve_signal_type(signal_type) else { return; }; // Read user metadata from users_engine. let Ok(Some(metadata)) = self.get_user_metadata(EntityId::new(user_id)) else { return; // No metadata or storage error -- skip cohort attribution. }; // Resolve cohort memberships. let cohorts = self.cohort_resolver.resolve(user_id, &metadata); // Record the signal into each matching cohort's ledger. let ts_ns = timestamp.as_nanos(); for cohort_name in &cohorts { self.cohort_ledger .record(cohort_name, entity_id, type_id, weight, ts_ns); } } /// Whether `signal_type` is a positive-engagement signal — one that should /// fold the signalled item's embedding into the acting user's preference /// vector. /// /// Schema-driven with a back-compat fallback: when the schema declares any /// positive-engagement signal (`Schema::positive_engagement_enabled`), the /// declarations are authoritative — a signal is positive iff its /// `SignalTypeDef::is_positive_engagement()` is set. When the schema declares /// none (or there is no schema), the engine falls back to the built-in /// default set so apps predating the flag keep their prior behavior. pub(super) fn is_positive_engagement(&self, signal_type: &str) -> bool { match self.schema_def.as_ref() { Some(schema) if schema.positive_engagement_enabled() => schema .signal(signal_type) .is_some_and(crate::schema::SignalTypeDef::is_positive_engagement), _ => is_positive_engagement_signal(signal_type), } } /// The embedding slot name used for item content embeddings. /// /// Single source of truth for every path that reads or writes an item /// embedding (`write_item_embedding`, `read_item_embedding`, /// `try_update_preference_vector`, and the retrieve-time preference boost): /// the schema's first Item embedding slot, falling back to "content" when no /// schema declares one. Keeping all paths on this resolver is what makes the /// preference-vector chain coherent regardless of how an app names its slot. pub(super) fn item_embedding_slot(&self) -> &str { self.schema_def .as_ref() .and_then(|s| { s.embedding_slots() .iter() .find(|slot| slot.entity_kind == EntityKind::Item) .map(|slot| slot.name.as_str()) }) .unwrap_or("content") } /// Attempt to update a user's preference vector from the item's stored embedding. /// /// Reads the item's embedding from durable storage (entity store) and blends it /// into the user's preference vector via `PreferenceVectors::update()`. This is /// a best-effort operation: if the item has no embedding, no storage is wired, or /// the embedding cannot be deserialized, the update is silently skipped. /// /// The slot name is resolved by [`Self::item_embedding_slot`] (the schema's /// first Item embedding slot, falling back to "content"). pub(super) fn try_update_preference_vector(&self, user_id: u64, entity_id: EntityId) { // Determine which embedding slot to read. let slot_name = self.item_embedding_slot(); // Read the item's embedding from durable storage. let Some(storage) = self.storage.as_ref() else { return; }; let key = crate::storage::vector::embedding_store_key(entity_id, slot_name); let embedding_bytes = match storage.items_engine().get(&key) { Ok(Some(bytes)) => bytes, Ok(None) => { tracing::debug!( entity_id = entity_id.as_u64(), slot = slot_name, "preference vector update skipped: item has no stored embedding" ); return; } Err(e) => { tracing::debug!( entity_id = entity_id.as_u64(), error = %e, "preference vector update skipped: storage read failed" ); return; } }; // Deserialize the embedding. let embedding = match crate::storage::vector::deserialize_embedding(&embedding_bytes) { Ok(v) => v, Err(e) => { tracing::debug!( entity_id = entity_id.as_u64(), error = %e, "preference vector update skipped: embedding deserialization failed" ); return; } }; // Blend into the user's preference vector. if !self.preference_vectors.update(user_id, &embedding) { tracing::debug!( user_id, entity_id = entity_id.as_u64(), embedding_dim = embedding.len(), "preference vector update skipped: dimension mismatch" ); } } }