tidaldb/tidal/src/signals/trimmer.rs
jordan 98bdc18a49 feat: add iknowyou app + complete M8 replication extensions + Aeries agents/skills
- applications/iknowyou: new Next.js chat application with persona-aware conversations,
  briefing API, cohort logic, vLLM streaming, and sidebar navigation
- tidal M8: add replication control plane (control.rs), tenant migration state machine
  (migration.rs), tenant/upgrade coordinators, cluster/fault test harnesses
- tidal M8 tests: expand m8p2/m8p3/m8p4 test suites; add m8p5_multitenancy and m8_uat
- tidal db: split replication_ops out of db/mod.rs (was 647 lines, now 574)
- .claude: add kai-park, kaya-osei, mira-vasquez agents; add aeries-design-architect,
  aeries-fullstack-engineer, aeries-product-visionary skills
- docs: update ROADMAP.md

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-02-24 21:09:11 -07:00

285 lines
9.8 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

//! Signal ledger LRU trimmer.
//!
//! At 1M entities × 10 signal types per entity, the `SignalLedger`'s in-memory
//! `DashMap` would grow to ~10M entries. The per-entry memory cost breaks down as:
//!
//! | Component | Size |
//! |-----------------------|--------|
//! | `HotSignalState` | 64 B |
//! | `BucketedCounter` | 944 B |
//! | `EntitySignalEntry` | 1008 B |
//! | `DashMap` per-entry overhead | ~80 B |
//! | **Total per entry** | **~1088 B** |
//!
//! Projection: 10M entries × 1088 B ≈ **10.4 GB** — marginally over the 10 GB
//! budget. This module implements time-ordered eviction to bound memory usage.
//!
//! # Policy
//!
//! Entries are evicted by their `last_update_ns` timestamp (least-recently-used).
//! Oldest entries are removed first, preferring to evict signal state for
//! entities that have not been signalled recently. This is conservative:
//! re-signalling an evicted entity simply creates a fresh entry.
//!
//! The default threshold is **5 million entries** (~5.4 GB), providing a 2×
//! safety margin below the 10 GB limit.
//!
//! # When is trimming triggered?
//!
//! The checkpoint background thread calls `trim_if_needed()` on every checkpoint
//! cycle. This amortises the O(N log N) sort cost across many signal writes.
use dashmap::DashMap;
use crate::schema::EntityId;
use crate::signals::SignalTypeId;
use crate::signals::ledger::EntitySignalEntry;
/// Default maximum number of `(entity, signal_type)` entries to keep in the hot path.
///
/// At ~1088 bytes per entry: 5M × 1088 ≈ 5.44 GB — comfortably under the 10 GB budget.
pub const DEFAULT_MAX_SIGNAL_ENTRIES: usize = 5_000_000;
/// Evict the oldest `(entity, signal_type)` entries from `entries` until
/// `entries.len() <= max_entries`.
///
/// Entries are sorted by `last_update_ns` ascending (oldest first) and the
/// bottom `over` entries are removed.
///
/// # Complexity
///
/// O(N log N) in the number of entries when eviction fires, with N being the
/// current number of entries. At 10M entries this is ~10M × 23 ≈ 230M comparisons,
/// which takes ~2–3 seconds on modern hardware. Amortised over the checkpoint
/// interval (typically minutes), this is acceptable.
///
/// # Thread safety
///
/// Uses `DashMap`'s snapshot iteration (`iter()` holds shard read-locks one-at-a-time).
/// Removals use `remove()` which briefly holds the shard write-lock. Concurrent
/// signal writes during trim may re-insert an evicted entry immediately — this is
/// acceptable: the entry will be re-populated and will not be stale.
/// Returns the number of entries evicted.
#[must_use]
pub fn trim_cold_entries(
entries: &DashMap<(EntityId, SignalTypeId), EntitySignalEntry>,
max_entries: usize,
) -> usize {
let len = entries.len();
if len <= max_entries {
return 0;
}
let over = len - max_entries;
// Collect (last_update_ns, key) pairs for sorting.
// We allocate a full snapshot because DashMap does not support sorted iteration.
let mut snapshot: Vec<(u64, (EntityId, SignalTypeId))> = entries
.iter()
.map(|r| (r.value().hot.last_update_ns(), *r.key()))
.collect();
// Sort ascending: oldest (lowest timestamp) first.
snapshot.sort_unstable_by_key(|(ts, _)| *ts);
// Remove the `over` oldest entries.
let mut evicted = 0;
for (_, key) in snapshot.into_iter().take(over) {
entries.remove(&key);
evicted += 1;
}
evicted
}
#[cfg(test)]
#[allow(clippy::unwrap_used)]
mod tests {
use std::time::Duration;
use super::*;
use crate::schema::{DecaySpec, EntityId, EntityKind, SchemaBuilder, Window};
use crate::signals::{NoopWalWriter, SignalLedger};
fn make_ledger() -> SignalLedger {
let mut builder = SchemaBuilder::new();
let _ = builder
.signal(
"view",
EntityKind::Item,
DecaySpec::Exponential {
half_life: Duration::from_secs(7 * 24 * 3600),
},
)
.windows(&[Window::TwentyFourHours])
.velocity(false)
.add();
let schema = builder.build().unwrap();
SignalLedger::new(schema, Box::new(NoopWalWriter))
}
/// After trim, `len() <= max_entries` invariant holds.
#[test]
fn trim_reduces_to_max_entries() {
let ledger = make_ledger();
// Insert 200 entries across 200 entities.
for i in 1u64..=200 {
ledger
.record_signal(
"view",
EntityId::new(i),
1.0,
crate::schema::Timestamp::from_nanos(i * 1_000_000_000),
)
.unwrap();
}
assert_eq!(ledger.entries().len(), 200);
// Trim to 100.
let _ = trim_cold_entries(ledger.entries(), 100);
assert!(
ledger.entries().len() <= 100,
"expected <= 100 entries, got {}",
ledger.entries().len()
);
}
/// After trim, evicted entries have older timestamps than retained entries.
#[test]
fn trim_evicts_oldest_entries() {
let ledger = make_ledger();
// Insert 10 entries with timestamps 1s..10s.
for i in 1u64..=10 {
ledger
.record_signal(
"view",
EntityId::new(i),
1.0,
crate::schema::Timestamp::from_nanos(i * 1_000_000_000),
)
.unwrap();
}
// Trim to 5: should keep the 5 most recently updated (entities 6..=10).
let _ = trim_cold_entries(ledger.entries(), 5);
assert_eq!(ledger.entries().len(), 5);
// Entities 1..=5 (oldest) should be evicted; 6..=10 should remain.
let type_id = ledger.resolve_signal_type("view").unwrap();
for i in 1u64..=5 {
assert!(
ledger.entries().get(&(EntityId::new(i), type_id)).is_none(),
"entity {i} should have been evicted"
);
}
for i in 6u64..=10 {
assert!(
ledger.entries().get(&(EntityId::new(i), type_id)).is_some(),
"entity {i} should have been retained"
);
}
}
/// No-op when entries already at or below max.
#[test]
fn trim_noop_when_under_limit() {
let ledger = make_ledger();
for i in 1u64..=10 {
ledger
.record_signal(
"view",
EntityId::new(i),
1.0,
crate::schema::Timestamp::from_nanos(i * 1_000_000_000),
)
.unwrap();
}
let before = ledger.entries().len();
trim_cold_entries(ledger.entries(), 100);
assert_eq!(
ledger.entries().len(),
before,
"no eviction should occur when under limit"
);
}
/// Memory size documentation: compile-time assertion that struct sizes are as expected.
#[test]
fn signal_state_size_documentation() {
use crate::signals::ledger::EntitySignalEntry;
use crate::signals::{BucketedCounter, HotSignalState};
// HotSignalState = exactly 64 bytes (cache-line aligned, verified by const assert).
assert_eq!(std::mem::size_of::<HotSignalState>(), 64);
// BucketedCounter = 60×4 + 168×4 + 2×1 + 3×8 = 240+672+2+24 = 938, padded to 944.
let warm_size = std::mem::size_of::<BucketedCounter>();
// Allow ±8 bytes for platform-specific padding.
assert!(
warm_size >= 938 && warm_size <= 952,
"BucketedCounter size={warm_size}, expected ~944"
);
let entry_size = std::mem::size_of::<EntitySignalEntry>();
assert!(
entry_size >= 1000 && entry_size <= 1100,
"EntitySignalEntry size={entry_size}, expected ~1008"
);
}
mod proptests {
use super::*;
use crate::schema::{DecaySpec, EntityKind, SchemaBuilder, Window};
use crate::signals::{NoopWalWriter, SignalLedger};
use proptest::prelude::*;
use std::time::Duration;
proptest! {
/// After trimming, the entry count never exceeds `max_entries`,
/// regardless of how many entries existed before.
#[test]
fn trim_never_exceeds_max(
entry_count in 1usize..=500,
max_entries in 1usize..=499,
) {
let mut builder = SchemaBuilder::new();
let _ = builder
.signal(
"view",
EntityKind::Item,
DecaySpec::Exponential {
half_life: Duration::from_secs(7 * 24 * 3600),
},
)
.windows(&[Window::TwentyFourHours])
.velocity(false)
.add();
let schema = builder.build().unwrap();
let ledger = SignalLedger::new(schema, Box::new(NoopWalWriter));
for i in 1..=(entry_count as u64) {
ledger
.record_signal(
"view",
crate::schema::EntityId::new(i),
1.0,
crate::schema::Timestamp::from_nanos(i * 1_000_000_000),
)
.unwrap();
}
trim_cold_entries(ledger.entries(), max_entries);
prop_assert!(
ledger.entries().len() <= max_entries,
"after trim, {} entries remain but max_entries={}",
ledger.entries().len(),
max_entries
);
}
}
}
}