9728194f16
Resolves every finding in docs/reviews/M0-M10-code-review-2026-06-08-pass2.md across the engine, network, server, and CLI crates: session restore, replication/CRDT, WAL format and recovery, storage indexes, query/ranking executors, cohort/community governance, and scatter-gather routing. Adds regression tests: - review_pass2_creator_search_filter - review_pass2_d_replication - review_pass2_query_for_session - review_pass2_storage_indexes_bitmap_cache - review_pass2_zone_a_sessions Verified: cargo clippy -D warnings and full test suite green across all crates.
399 lines
17 KiB
Rust
399 lines
17 KiB
Rust
//! Session-TTL sweeper and internal accessor helpers for [`TidalDb`].
|
||
|
||
use std::sync::{Arc, atomic::Ordering};
|
||
|
||
use super::TidalDb;
|
||
|
||
/// Number of trailing days the notification tracker retains beyond today.
|
||
///
|
||
/// Notification caps are per-calendar-day and only "today" is read, so a tiny
|
||
/// window is ample; the extra days only guard against clock skew / day-boundary
|
||
/// races. `evict_old_days(today, N)` keeps the last `N + 1` days.
|
||
const NOTIFICATION_KEEP_DAYS: u32 = 2;
|
||
|
||
impl TidalDb {
|
||
// ── M7p2 Session Sweeper ──────────────────────────────────────────────────
|
||
|
||
/// Scan all active sessions and close any that have exceeded their
|
||
/// policy's `max_session_duration`.
|
||
///
|
||
/// Called every 60s by the sweeper thread, and once during shutdown.
|
||
///
|
||
/// Age is measured from the **durable** `started_at_ns` (nanoseconds since
|
||
/// the Unix epoch), never from the monotonic `started_at` `Instant`. A
|
||
/// session restored on open re-seeds (and back-dates) `started_at`, but only
|
||
/// `started_at_ns` is the durable wall-clock anchor: measuring off it ensures
|
||
/// a session that was already over its `max_session_duration` at shutdown is
|
||
/// reaped on the first startup sweep, exactly as `start_sweeper` documents
|
||
/// ("sessions that expired while the process was down ... are reaped promptly
|
||
/// at startup"). Reading the monotonic clock here would reset every restored
|
||
/// session's age to ~zero and silently void the TTL bound across restarts.
|
||
pub(crate) fn sweep_expired_sessions(&self) {
|
||
let now_ns = crate::schema::Timestamp::now().as_nanos();
|
||
let mut expired_ids = Vec::new();
|
||
|
||
for entry in &self.sessions {
|
||
let state = entry.value();
|
||
let elapsed =
|
||
std::time::Duration::from_nanos(now_ns.saturating_sub(state.started_at_ns));
|
||
|
||
// Look up the policy's max_session_duration.
|
||
let max_duration = self
|
||
.schema_def
|
||
.as_ref()
|
||
.and_then(|s| s.session_policy(&state.policy_name))
|
||
.map(|p| p.max_session_duration);
|
||
|
||
if let Some(max) = max_duration
|
||
&& elapsed > max
|
||
{
|
||
expired_ids.push(state.id);
|
||
}
|
||
}
|
||
|
||
if !expired_ids.is_empty() {
|
||
tracing::info!(
|
||
count = expired_ids.len(),
|
||
"sweeper: closing expired sessions"
|
||
);
|
||
}
|
||
|
||
for session_id in expired_ids {
|
||
if let Err(e) = self.close_session_internal(session_id, true) {
|
||
tracing::warn!(
|
||
error = %e,
|
||
session_id = %session_id,
|
||
"sweeper: failed to close expired session"
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Start the session TTL sweeper background thread.
|
||
///
|
||
/// The sweeper performs an initial sweep promptly at startup (reaping
|
||
/// sessions that expired while the process was down), then wakes every 60
|
||
/// seconds (interruptible via 1s sleep intervals) to close any sessions
|
||
/// that have exceeded their policy's `max_session_duration`. It shuts down
|
||
/// within ~1s when `db.close()` is called or the last strong `Arc` is
|
||
/// dropped.
|
||
///
|
||
/// The sweeper holds a `Weak<TidalDb>` so it does not keep the database
|
||
/// alive on its own. The `Arc` passed here MUST be the **final,
|
||
/// long-lived** handle the owner retains — otherwise the upgrade target is
|
||
/// destroyed and the sweeper is dead-on-arrival (CONCURRENCY-1). Use
|
||
/// [`TidalDbBuilder::open_arc`](crate::TidalDbBuilder::open_arc) or wrap the
|
||
/// db in the Arc the server keeps (e.g. `ServerState`) and call this once on
|
||
/// it; never spawn it against a temporary `Arc` that is about to be
|
||
/// `Arc::try_unwrap`'d. For ephemeral/test usage, use `force_sweep()`.
|
||
///
|
||
/// In **ephemeral** mode this is a no-op: sessions are never persisted, so
|
||
/// there is nothing for a background sweeper to reap across restarts.
|
||
/// Centralizing the mode gate here keeps every long-lived-Arc owner correct
|
||
/// without re-implementing the check.
|
||
///
|
||
/// If the OS refuses to spawn the sweeper thread (extremely rare), the
|
||
/// database degrades rather than panicking: the error is logged and the db
|
||
/// runs without a background sweeper. Expired sessions are then reaped only
|
||
/// on the next explicit sweep / shutdown rather than on a timer — a
|
||
/// graceful degradation, not a crash on a path that has already committed to
|
||
/// returning an open database.
|
||
pub fn start_sweeper(db: &Arc<Self>) {
|
||
if db.config.mode != crate::db::config::StorageMode::Persistent {
|
||
tracing::debug!("ephemeral mode: session TTL sweeper not started (no-op)");
|
||
return;
|
||
}
|
||
let db_weak = Arc::downgrade(db);
|
||
let shutdown = Arc::clone(&db.shutdown_sweeper);
|
||
let spawn_result = std::thread::Builder::new()
|
||
.name("tidaldb-session-sweeper".into())
|
||
.spawn(move || {
|
||
tracing::info!("session TTL sweeper started");
|
||
loop {
|
||
// Sweep first, then wait. Sweeping at the top of the loop
|
||
// means sessions that expired while the process was down
|
||
// (or between sweeps) are reaped promptly at startup rather
|
||
// than lingering up to a full interval. Upgrade the Weak --
|
||
// if the db was dropped, exit silently.
|
||
let Some(db) = db_weak.upgrade() else {
|
||
return;
|
||
};
|
||
db.sweep_expired_sessions();
|
||
db.sweep_notification_tracker();
|
||
drop(db); // release the temporary strong ref immediately
|
||
|
||
// Wait one 60s interval, broken into 1s steps so shutdown
|
||
// is detected within ~1s.
|
||
for _ in 0u32..60 {
|
||
if shutdown.load(Ordering::Relaxed) {
|
||
tracing::info!("session TTL sweeper shutting down");
|
||
return;
|
||
}
|
||
std::thread::sleep(std::time::Duration::from_secs(1));
|
||
}
|
||
|
||
if shutdown.load(Ordering::Relaxed) {
|
||
return;
|
||
}
|
||
}
|
||
});
|
||
|
||
match spawn_result {
|
||
Ok(handle) => {
|
||
if let Ok(mut guard) = db.sweeper_thread.lock() {
|
||
*guard = Some(handle);
|
||
}
|
||
}
|
||
Err(e) => {
|
||
// Do not panic on a path that returns an already-open database.
|
||
// Degrade: no background sweeper; sessions are reaped on the next
|
||
// explicit sweep / shutdown instead of on a timer.
|
||
tracing::error!(
|
||
error = %e,
|
||
"failed to spawn session sweeper thread; running without a background sweeper"
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Evict notification-tracker entries older than the retention window.
|
||
///
|
||
/// The notification caps are per-calendar-day and only "today" is ever
|
||
/// consulted, so retaining a small window is sufficient. Without this the
|
||
/// `(user, creator, day)` and `(user, day)` maps grow by O(active users ×
|
||
/// creators) per day of uptime and are never trimmed — an unbounded leak for
|
||
/// a long-lived server (CRITICAL: `evict_old_days` previously had zero
|
||
/// callers, so its documented mitigation never ran). Driven on the same
|
||
/// cadence as the session sweep; the sweep-at-top-of-loop also runs it
|
||
/// promptly at startup so stale days from a prior run are trimmed on reopen.
|
||
pub(crate) fn sweep_notification_tracker(&self) {
|
||
const NANOS_PER_DAY: u64 = 86_400 * 1_000_000_000;
|
||
let today = (crate::schema::Timestamp::now().as_nanos() / NANOS_PER_DAY) as u32;
|
||
self.notification_tracker
|
||
.evict_old_days(today, NOTIFICATION_KEEP_DAYS);
|
||
}
|
||
|
||
/// Test-only: trigger a session sweep without waiting for the background thread.
|
||
#[cfg(any(test, feature = "test-utils"))]
|
||
pub fn force_sweep(&self) {
|
||
self.sweep_expired_sessions();
|
||
self.sweep_notification_tracker();
|
||
}
|
||
|
||
/// Test-only: `true` if the session-sweeper thread is spawned and has not
|
||
/// yet exited. Proves [`TidalDbBuilder::open_arc`] actually starts a live
|
||
/// sweeper against the long-lived `Arc` (CONCURRENCY-1 regression guard).
|
||
#[cfg(any(test, feature = "test-utils"))]
|
||
#[must_use]
|
||
pub fn sweeper_is_running(&self) -> bool {
|
||
match self.sweeper_thread.lock() {
|
||
Ok(guard) => guard.as_ref().is_some_and(|h| !h.is_finished()),
|
||
Err(_) => false,
|
||
}
|
||
}
|
||
|
||
/// Test-only: return the number of active rate-limiter buckets.
|
||
#[cfg(any(test, feature = "test-utils"))]
|
||
pub fn rate_limiter_bucket_count(&self) -> usize {
|
||
self.rate_limiter.active_buckets()
|
||
}
|
||
|
||
/// Test-only: reference to the backup-in-progress flag.
|
||
///
|
||
/// Used by integration tests to verify the backpressure mechanism
|
||
/// without requiring a racy concurrent backup.
|
||
#[cfg(any(test, feature = "test-utils"))]
|
||
#[allow(clippy::missing_const_for_fn)] // Arc field prevents const in practice
|
||
pub fn backup_in_progress_flag(&self) -> &Arc<std::sync::atomic::AtomicBool> {
|
||
&self.backup_in_progress
|
||
}
|
||
|
||
/// Store the directory lock file so it is held for the database lifetime.
|
||
///
|
||
/// The advisory flock is released automatically when the `File` is dropped.
|
||
pub(crate) fn set_lock_file(&mut self, file: std::fs::File) {
|
||
self.lock_file = Some(file);
|
||
}
|
||
|
||
/// Attach the running metrics HTTP server handle.
|
||
///
|
||
/// Started by [`TidalDbBuilder::open`] *after* construction (so the
|
||
/// control plane is already installed on the uniquely owned metrics state)
|
||
/// and stopped on `Drop`.
|
||
#[cfg(feature = "metrics")]
|
||
pub(crate) fn set_metrics_handle(&mut self, handle: super::http::MetricsHandle) {
|
||
self.metrics_handle = Some(handle);
|
||
}
|
||
|
||
/// Replace the rate limiter with one using the given config.
|
||
///
|
||
/// Called by [`TidalDbBuilder::open`] before any sessions are opened, so
|
||
/// there is no risk of orphaned bucket references.
|
||
pub(crate) fn with_rate_limiter_config(
|
||
mut self,
|
||
config: crate::load::RateLimiterConfig,
|
||
) -> Self {
|
||
self.rate_limiter = Arc::new(crate::load::RateLimiter::new(config));
|
||
self
|
||
}
|
||
|
||
/// Reject write operations on follower nodes.
|
||
///
|
||
/// Returns `TidalError::ReadOnly` when the node's role is `Follower`.
|
||
/// Must be the **first** statement in every public write method so the
|
||
/// guard fires before any validation or allocation.
|
||
pub(crate) fn require_writeable(&self, operation: &str) -> crate::Result<()> {
|
||
if self.config.cluster.role == crate::db::config::NodeRole::Follower {
|
||
return Err(crate::schema::TidalError::read_only(operation));
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
// ── Internal helpers ─────────────────────────────────────────────────────
|
||
|
||
/// Borrow the storage backend, or error if the database was opened without a schema.
|
||
pub(crate) fn storage(&self) -> crate::Result<&super::storage_box::StorageBox> {
|
||
self.storage.as_ref().ok_or_else(|| {
|
||
crate::schema::TidalError::internal(
|
||
"storage_access",
|
||
"no storage: open with with_schema()",
|
||
)
|
||
})
|
||
}
|
||
|
||
/// Borrow the signal ledger, or error if the database was opened without a schema.
|
||
pub fn ledger(&self) -> crate::Result<&Arc<crate::signals::SignalLedger>> {
|
||
self.ledger.as_ref().ok_or_else(|| {
|
||
crate::schema::TidalError::internal(
|
||
"ledger_access",
|
||
"no ledger: open with with_schema()",
|
||
)
|
||
})
|
||
}
|
||
|
||
/// Reference to the co-engagement index (M6).
|
||
#[must_use]
|
||
pub fn co_engagement(&self) -> &crate::entities::CoEngagementIndex {
|
||
&self.co_engagement
|
||
}
|
||
|
||
/// Reference to the cohort signal ledger (M6).
|
||
#[must_use]
|
||
pub fn cohort_ledger(&self) -> &crate::cohort::CohortSignalLedger {
|
||
&self.cohort_ledger
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
#[allow(clippy::unwrap_used)]
|
||
mod tests {
|
||
use crate::TidalDb;
|
||
|
||
/// CRITICAL regression: the sweeper actually drives notification eviction.
|
||
/// `evict_old_days` previously had zero callers, so the tracker leaked
|
||
/// unbounded. `sweep_notification_tracker` (run by the sweeper loop and
|
||
/// `force_sweep`) must remove entries from days outside the retention window.
|
||
#[test]
|
||
fn sweep_notification_tracker_evicts_stale_days() {
|
||
// Far in the past relative to the real clock (~20k days since epoch).
|
||
const NANOS_PER_DAY: u64 = 86_400 * 1_000_000_000;
|
||
|
||
let db = TidalDb::builder().ephemeral().open().unwrap();
|
||
|
||
// Record a delivery on day 1 and one on "today".
|
||
let today = (crate::schema::Timestamp::now().as_nanos() / NANOS_PER_DAY) as u32;
|
||
db.notification_tracker.record(1, 100, 1);
|
||
db.notification_tracker.record(1, 100, today);
|
||
assert_eq!(db.notification_tracker.count_today(1, 100, 1), 1);
|
||
|
||
db.sweep_notification_tracker();
|
||
|
||
assert_eq!(
|
||
db.notification_tracker.count_today(1, 100, 1),
|
||
0,
|
||
"stale day-1 entry must be evicted by the sweeper"
|
||
);
|
||
assert_eq!(
|
||
db.notification_tracker.count_today(1, 100, today),
|
||
1,
|
||
"today's entry must be retained"
|
||
);
|
||
}
|
||
|
||
/// CRITICAL/BLOCKER regression: the sweeper must TTL-expire a *restored*
|
||
/// session. A restored session re-seeds the monotonic `started_at` with a
|
||
/// fresh `Instant::now()`; only the durable `started_at_ns` carries the true
|
||
/// age. Measuring age off the monotonic clock would hand every restored
|
||
/// session a fresh full lease on each restart and never reap it. The sweeper
|
||
/// now measures off `started_at_ns`, so a session that was already past its
|
||
/// `max_session_duration` at shutdown is reaped on the first startup sweep.
|
||
#[test]
|
||
fn sweeper_expires_restored_over_age_session() {
|
||
use std::{
|
||
sync::{
|
||
Arc,
|
||
atomic::{AtomicBool, AtomicU64},
|
||
},
|
||
time::{Duration, Instant},
|
||
};
|
||
|
||
use crate::{
|
||
schema::{AgentPolicy, DecaySpec, EntityKind, SchemaBuilder, Timestamp},
|
||
session::{AgentId, AuditLog, SessionId, SessionState},
|
||
};
|
||
|
||
let mut b = SchemaBuilder::new();
|
||
let _ = b
|
||
.signal("view", EntityKind::Item, DecaySpec::Permanent)
|
||
.add();
|
||
b.session_policy(
|
||
"short",
|
||
AgentPolicy {
|
||
allowed_signals: vec![],
|
||
denied_signals: vec![],
|
||
// 1-hour cap; our restored session is 2h old.
|
||
max_session_duration: Duration::from_secs(3600),
|
||
max_signals_per_session: 0,
|
||
},
|
||
);
|
||
let schema = b.build().unwrap();
|
||
|
||
let db = TidalDb::builder()
|
||
.ephemeral()
|
||
.with_schema(schema)
|
||
.open()
|
||
.unwrap();
|
||
|
||
// Restore shape: monotonic clock fresh, durable start 2h ago.
|
||
let now_ns = Timestamp::now().as_nanos();
|
||
let two_hours_ns = 2 * 3600 * 1_000_000_000u64;
|
||
let session_id = SessionId::from_raw(1);
|
||
let state = Arc::new(SessionState {
|
||
id: session_id,
|
||
user_id: 7,
|
||
agent_id: AgentId::new("restored").unwrap(),
|
||
policy_name: "short".to_string(),
|
||
started_at: Instant::now(),
|
||
started_at_ns: now_ns.saturating_sub(two_hours_ns),
|
||
metadata: std::collections::HashMap::new(),
|
||
signals: dashmap::DashMap::new(),
|
||
signaled_entities: dashmap::DashMap::new(),
|
||
annotations: std::sync::Mutex::new(Vec::new()),
|
||
signals_written: AtomicU64::new(0),
|
||
signals_rejected: AtomicU64::new(0),
|
||
audit_log: std::sync::Mutex::new(AuditLog::new()),
|
||
closed: Arc::new(AtomicBool::new(false)),
|
||
});
|
||
db.sessions.insert(session_id, state);
|
||
assert_eq!(db.active_sessions().len(), 1);
|
||
|
||
db.force_sweep();
|
||
|
||
assert_eq!(
|
||
db.active_sessions().len(),
|
||
0,
|
||
"a restored session 2h past a 1h cap must be reaped on the startup sweep"
|
||
);
|
||
}
|
||
}
|