//! Session-TTL sweeper and internal accessor helpers for [`TidalDb`]. use std::sync::{Arc, atomic::Ordering}; use super::TidalDb; /// Number of trailing days the notification tracker retains beyond today. /// /// Notification caps are per-calendar-day and only "today" is read, so a tiny /// window is ample; the extra days only guard against clock skew / day-boundary /// races. `evict_old_days(today, N)` keeps the last `N + 1` days. const NOTIFICATION_KEEP_DAYS: u32 = 2; impl TidalDb { // ── M7p2 Session Sweeper ────────────────────────────────────────────────── /// Scan all active sessions and close any that have exceeded their /// policy's `max_session_duration`. /// /// Called every 60s by the sweeper thread, and once during shutdown. /// /// Age is measured from the **durable** `started_at_ns` (nanoseconds since /// the Unix epoch), never from the monotonic `started_at` `Instant`. A /// session restored on open re-seeds (and back-dates) `started_at`, but only /// `started_at_ns` is the durable wall-clock anchor: measuring off it ensures /// a session that was already over its `max_session_duration` at shutdown is /// reaped on the first startup sweep, exactly as `start_sweeper` documents /// ("sessions that expired while the process was down ... are reaped promptly /// at startup"). Reading the monotonic clock here would reset every restored /// session's age to ~zero and silently void the TTL bound across restarts. pub(crate) fn sweep_expired_sessions(&self) { let now_ns = crate::schema::Timestamp::now().as_nanos(); let mut expired_ids = Vec::new(); for entry in &self.sessions { let state = entry.value(); let elapsed = std::time::Duration::from_nanos(now_ns.saturating_sub(state.started_at_ns)); // Look up the policy's max_session_duration. let max_duration = self .schema_def .as_ref() .and_then(|s| s.session_policy(&state.policy_name)) .map(|p| p.max_session_duration); if let Some(max) = max_duration && elapsed > max { expired_ids.push(state.id); } } if !expired_ids.is_empty() { tracing::info!( count = expired_ids.len(), "sweeper: closing expired sessions" ); } for session_id in expired_ids { if let Err(e) = self.close_session_internal(session_id, true) { tracing::warn!( error = %e, session_id = %session_id, "sweeper: failed to close expired session" ); } } } /// Start the session TTL sweeper background thread. /// /// The sweeper performs an initial sweep promptly at startup (reaping /// sessions that expired while the process was down), then wakes every 60 /// seconds (interruptible via 1s sleep intervals) to close any sessions /// that have exceeded their policy's `max_session_duration`. It shuts down /// within ~1s when `db.close()` is called or the last strong `Arc` is /// dropped. /// /// The sweeper holds a `Weak` so it does not keep the database /// alive on its own. The `Arc` passed here MUST be the **final, /// long-lived** handle the owner retains — otherwise the upgrade target is /// destroyed and the sweeper is dead-on-arrival (CONCURRENCY-1). Use /// [`TidalDbBuilder::open_arc`](crate::TidalDbBuilder::open_arc) or wrap the /// db in the Arc the server keeps (e.g. `ServerState`) and call this once on /// it; never spawn it against a temporary `Arc` that is about to be /// `Arc::try_unwrap`'d. For ephemeral/test usage, use `force_sweep()`. /// /// In **ephemeral** mode this is a no-op: sessions are never persisted, so /// there is nothing for a background sweeper to reap across restarts. /// Centralizing the mode gate here keeps every long-lived-Arc owner correct /// without re-implementing the check. /// /// If the OS refuses to spawn the sweeper thread (extremely rare), the /// database degrades rather than panicking: the error is logged and the db /// runs without a background sweeper. Expired sessions are then reaped only /// on the next explicit sweep / shutdown rather than on a timer — a /// graceful degradation, not a crash on a path that has already committed to /// returning an open database. pub fn start_sweeper(db: &Arc) { if db.config.mode != crate::db::config::StorageMode::Persistent { tracing::debug!("ephemeral mode: session TTL sweeper not started (no-op)"); return; } let db_weak = Arc::downgrade(db); let shutdown = Arc::clone(&db.shutdown_sweeper); let spawn_result = std::thread::Builder::new() .name("tidaldb-session-sweeper".into()) .spawn(move || { tracing::info!("session TTL sweeper started"); loop { // Sweep first, then wait. Sweeping at the top of the loop // means sessions that expired while the process was down // (or between sweeps) are reaped promptly at startup rather // than lingering up to a full interval. Upgrade the Weak -- // if the db was dropped, exit silently. let Some(db) = db_weak.upgrade() else { return; }; db.sweep_expired_sessions(); db.sweep_notification_tracker(); drop(db); // release the temporary strong ref immediately // Wait one 60s interval, broken into 1s steps so shutdown // is detected within ~1s. for _ in 0u32..60 { if shutdown.load(Ordering::Relaxed) { tracing::info!("session TTL sweeper shutting down"); return; } std::thread::sleep(std::time::Duration::from_secs(1)); } if shutdown.load(Ordering::Relaxed) { return; } } }); match spawn_result { Ok(handle) => { if let Ok(mut guard) = db.sweeper_thread.lock() { *guard = Some(handle); } } Err(e) => { // Do not panic on a path that returns an already-open database. // Degrade: no background sweeper; sessions are reaped on the next // explicit sweep / shutdown instead of on a timer. tracing::error!( error = %e, "failed to spawn session sweeper thread; running without a background sweeper" ); } } } /// Evict notification-tracker entries older than the retention window. /// /// The notification caps are per-calendar-day and only "today" is ever /// consulted, so retaining a small window is sufficient. Without this the /// `(user, creator, day)` and `(user, day)` maps grow by O(active users × /// creators) per day of uptime and are never trimmed — an unbounded leak for /// a long-lived server (CRITICAL: `evict_old_days` previously had zero /// callers, so its documented mitigation never ran). Driven on the same /// cadence as the session sweep; the sweep-at-top-of-loop also runs it /// promptly at startup so stale days from a prior run are trimmed on reopen. pub(crate) fn sweep_notification_tracker(&self) { const NANOS_PER_DAY: u64 = 86_400 * 1_000_000_000; let today = (crate::schema::Timestamp::now().as_nanos() / NANOS_PER_DAY) as u32; self.notification_tracker .evict_old_days(today, NOTIFICATION_KEEP_DAYS); } /// Test-only: trigger a session sweep without waiting for the background thread. #[cfg(any(test, feature = "test-utils"))] pub fn force_sweep(&self) { self.sweep_expired_sessions(); self.sweep_notification_tracker(); } /// Test-only: `true` if the session-sweeper thread is spawned and has not /// yet exited. Proves [`TidalDbBuilder::open_arc`] actually starts a live /// sweeper against the long-lived `Arc` (CONCURRENCY-1 regression guard). #[cfg(any(test, feature = "test-utils"))] #[must_use] pub fn sweeper_is_running(&self) -> bool { match self.sweeper_thread.lock() { Ok(guard) => guard.as_ref().is_some_and(|h| !h.is_finished()), Err(_) => false, } } /// Test-only: return the number of active rate-limiter buckets. #[cfg(any(test, feature = "test-utils"))] pub fn rate_limiter_bucket_count(&self) -> usize { self.rate_limiter.active_buckets() } /// Test-only: reference to the backup-in-progress flag. /// /// Used by integration tests to verify the backpressure mechanism /// without requiring a racy concurrent backup. #[cfg(any(test, feature = "test-utils"))] #[allow(clippy::missing_const_for_fn)] // Arc field prevents const in practice pub fn backup_in_progress_flag(&self) -> &Arc { &self.backup_in_progress } /// Store the directory lock file so it is held for the database lifetime. /// /// The advisory flock is released automatically when the `File` is dropped. pub(crate) fn set_lock_file(&mut self, file: std::fs::File) { self.lock_file = Some(file); } /// Attach the running metrics HTTP server handle. /// /// Started by [`TidalDbBuilder::open`] *after* construction (so the /// control plane is already installed on the uniquely owned metrics state) /// and stopped on `Drop`. #[cfg(feature = "metrics")] pub(crate) fn set_metrics_handle(&mut self, handle: super::http::MetricsHandle) { self.metrics_handle = Some(handle); } /// Replace the rate limiter with one using the given config. /// /// Called by [`TidalDbBuilder::open`] before any sessions are opened, so /// there is no risk of orphaned bucket references. pub(crate) fn with_rate_limiter_config( mut self, config: crate::load::RateLimiterConfig, ) -> Self { self.rate_limiter = Arc::new(crate::load::RateLimiter::new(config)); self } /// Reject write operations on follower nodes. /// /// Returns `TidalError::ReadOnly` when the node's role is `Follower`. /// Must be the **first** statement in every public write method so the /// guard fires before any validation or allocation. pub(crate) fn require_writeable(&self, operation: &str) -> crate::Result<()> { if self.config.cluster.role == crate::db::config::NodeRole::Follower { return Err(crate::schema::TidalError::read_only(operation)); } Ok(()) } // ── Internal helpers ───────────────────────────────────────────────────── /// Borrow the storage backend, or error if the database was opened without a schema. pub(crate) fn storage(&self) -> crate::Result<&super::storage_box::StorageBox> { self.storage.as_ref().ok_or_else(|| { crate::schema::TidalError::internal( "storage_access", "no storage: open with with_schema()", ) }) } /// Borrow the signal ledger, or error if the database was opened without a schema. pub fn ledger(&self) -> crate::Result<&Arc> { self.ledger.as_ref().ok_or_else(|| { crate::schema::TidalError::internal( "ledger_access", "no ledger: open with with_schema()", ) }) } /// Reference to the co-engagement index (M6). #[must_use] pub fn co_engagement(&self) -> &crate::entities::CoEngagementIndex { &self.co_engagement } /// Reference to the cohort signal ledger (M6). #[must_use] pub fn cohort_ledger(&self) -> &crate::cohort::CohortSignalLedger { &self.cohort_ledger } } #[cfg(test)] #[allow(clippy::unwrap_used)] mod tests { use crate::TidalDb; /// CRITICAL regression: the sweeper actually drives notification eviction. /// `evict_old_days` previously had zero callers, so the tracker leaked /// unbounded. `sweep_notification_tracker` (run by the sweeper loop and /// `force_sweep`) must remove entries from days outside the retention window. #[test] fn sweep_notification_tracker_evicts_stale_days() { // Far in the past relative to the real clock (~20k days since epoch). const NANOS_PER_DAY: u64 = 86_400 * 1_000_000_000; let db = TidalDb::builder().ephemeral().open().unwrap(); // Record a delivery on day 1 and one on "today". let today = (crate::schema::Timestamp::now().as_nanos() / NANOS_PER_DAY) as u32; db.notification_tracker.record(1, 100, 1); db.notification_tracker.record(1, 100, today); assert_eq!(db.notification_tracker.count_today(1, 100, 1), 1); db.sweep_notification_tracker(); assert_eq!( db.notification_tracker.count_today(1, 100, 1), 0, "stale day-1 entry must be evicted by the sweeper" ); assert_eq!( db.notification_tracker.count_today(1, 100, today), 1, "today's entry must be retained" ); } /// CRITICAL/BLOCKER regression: the sweeper must TTL-expire a *restored* /// session. A restored session re-seeds the monotonic `started_at` with a /// fresh `Instant::now()`; only the durable `started_at_ns` carries the true /// age. Measuring age off the monotonic clock would hand every restored /// session a fresh full lease on each restart and never reap it. The sweeper /// now measures off `started_at_ns`, so a session that was already past its /// `max_session_duration` at shutdown is reaped on the first startup sweep. #[test] fn sweeper_expires_restored_over_age_session() { use std::{ sync::{ Arc, atomic::{AtomicBool, AtomicU64}, }, time::{Duration, Instant}, }; use crate::{ schema::{AgentPolicy, DecaySpec, EntityKind, SchemaBuilder, Timestamp}, session::{AgentId, AuditLog, SessionId, SessionState}, }; let mut b = SchemaBuilder::new(); let _ = b .signal("view", EntityKind::Item, DecaySpec::Permanent) .add(); b.session_policy( "short", AgentPolicy { allowed_signals: vec![], denied_signals: vec![], // 1-hour cap; our restored session is 2h old. max_session_duration: Duration::from_secs(3600), max_signals_per_session: 0, }, ); let schema = b.build().unwrap(); let db = TidalDb::builder() .ephemeral() .with_schema(schema) .open() .unwrap(); // Restore shape: monotonic clock fresh, durable start 2h ago. let now_ns = Timestamp::now().as_nanos(); let two_hours_ns = 2 * 3600 * 1_000_000_000u64; let session_id = SessionId::from_raw(1); let state = Arc::new(SessionState { id: session_id, user_id: 7, agent_id: AgentId::new("restored").unwrap(), policy_name: "short".to_string(), started_at: Instant::now(), started_at_ns: now_ns.saturating_sub(two_hours_ns), metadata: std::collections::HashMap::new(), signals: dashmap::DashMap::new(), signaled_entities: dashmap::DashMap::new(), annotations: std::sync::Mutex::new(Vec::new()), signals_written: AtomicU64::new(0), signals_rejected: AtomicU64::new(0), audit_log: std::sync::Mutex::new(AuditLog::new()), closed: Arc::new(AtomicBool::new(false)), }); db.sessions.insert(session_id, state); assert_eq!(db.active_sessions().len(), 1); db.force_sweep(); assert_eq!( db.active_sessions().len(), 0, "a restored session 2h past a 1h cap must be reaped on the startup sweep" ); } }