tidaldb/tidal/tests/m0m10_durability.rs

//! Crash-recovery durability tests for the per-user / community signal
//! side-effects that the M0–M10 review flagged as in-memory-only.
//!
//! Every test here proves a side-effect survives a crash that does NOT run a
//! graceful checkpoint of the in-memory state. The mechanism is the existing
//! crash-injection harness: we arm `CrashPoint::CheckpointPreFlush` and then
//! drive a clean `close()` whose signal-ledger checkpoint panics BEFORE it
//! flushes. That guarantees none of the in-memory side-effect indexes are
//! checkpointed on the way down — so anything that survives reopen survived
//! purely on its own durable path (write-through rows, or a checkpoint that
//! happened strictly before the crash), exactly the crash the review demands.
//!
//! Coverage:
//! - BLOCKER 2 — hard negatives (skip) stay excluded after a crash.
//! - BLOCKER 3 — interaction ledger `top_creators` survives within decay tolerance.
//! - CRITICAL 3 — completion (`InProgress`) + save (`DateSaved`) survive a crash.
//! - CRITICAL 4 — preference vectors survive a crash after a checkpoint (bounded).
//! - BLOCKER 1 / CRITICAL 5 — accepted community contribution survives a crash
//!   with ZERO loss window (synchronous per-contribution checkpoint).
//! - BLOCKER 4 — agent-scope purge stays applied after a crash (durable tombstone).

#![allow(
    clippy::too_many_lines,
    clippy::unwrap_used,
    clippy::cast_precision_loss
)]

use std::{collections::HashMap, time::Duration};

use tidaldb::{
    CommunityId, RemoveScope, ScopeClass, ScopePermission, ShareIntent, SharePolicy, TempTidalHome,
    TidalDb, UserId,
    query::retrieve::Retrieve,
    schema::{DecaySpec, EntityId, EntityKind, SchemaBuilder, Timestamp, Window},
    storage::indexes::filter::FilterExpr,
    testing::{CrashInjector, CrashPoint, crash_injector::run_with_crash},
};

// ── Shared helpers ────────────────────────────────────────────────────────────

/// Schema with the engagement signals these tests drive through
/// `signal_with_context`, plus a `content` embedding slot for preference vectors.
fn durability_schema(dims: usize) -> tidaldb::schema::Schema {
    let mut b = SchemaBuilder::new();
    for &(name, half_life_days) in &[
        ("view", 7u64),
        ("like", 14),
        ("skip", 1),
        ("completion", 30),
        ("save", 30),
    ] {
        let _ = b
            .signal(
                name,
                EntityKind::Item,
                DecaySpec::Exponential {
                    half_life: Duration::from_secs(half_life_days * 24 * 3600),
                },
            )
            .windows(&[Window::AllTime])
            .velocity(false)
            .add();
    }
    b.embedding_slot("content", EntityKind::Item, dims);
    b.build().unwrap()
}

fn item_meta(creator_id: u64) -> HashMap<String, String> {
    let mut m = HashMap::new();
    m.insert("title".to_string(), format!("item from {creator_id}"));
    m.insert("category".to_string(), "music".to_string());
    m.insert("creator_id".to_string(), creator_id.to_string());
    m
}

/// Run `f` against a freshly-opened DB on `home`, then crash before the shutdown
/// checkpoint flushes any in-memory side-effect state. A local signal is written
/// first so the shutdown's signal-ledger checkpoint actually reaches
/// `CheckpointPreFlush` (community/per-user side-effects never touch the local
/// aggregate, so without it the checkpoint over an empty ledger would not fire).
fn crash_before_checkpoint(
    home: &TempTidalHome,
    schema: tidaldb::schema::Schema,
    f: impl FnOnce(&TidalDb),
) {
    let injector = CrashInjector::new(CrashPoint::CheckpointPreFlush, 0);
    let outcome = run_with_crash(&injector, || {
        let db = TidalDb::builder()
            .with_data_dir(home.path())
            .with_schema(schema)
            .open()
            .unwrap();
        f(&db);
        // Ensure the shutdown checkpoint has a local entry to flush so the crash
        // point is crossed. `like` is defined in every schema these tests use and
        // lands in the LOCAL signal ledger (community/per-user side-effects never
        // touch it), so this is independent of the state under assertion.
        db.signal("like", EntityId::new(9_999_999), 1.0, Timestamp::now())
            .unwrap();
        let _ = db.close();
    });
    assert!(
        matches!(outcome, Err(CrashPoint::CheckpointPreFlush)),
        "crash injector must fire during the shutdown checkpoint, got {outcome:?}"
    );
}

fn reopen(home: &TempTidalHome, schema: tidaldb::schema::Schema) -> TidalDb {
    TidalDb::builder()
        .with_data_dir(home.path())
        .with_schema(schema)
        .open()
        .unwrap()
}

// ── BLOCKER 2: hard negatives survive a crash ────────────────────────────────

#[test]
fn hard_negative_skip_survives_crash_before_checkpoint() {
    let home = TempTidalHome::new().unwrap();

    crash_before_checkpoint(&home, durability_schema(4), |db| {
        // Two items; user 7 skips item 1 (a hard negative).
        for id in 1u64..=2 {
            db.write_item_with_metadata(EntityId::new(id), &item_meta(1))
                .unwrap();
        }
        db.signal_with_context(
            "skip",
            EntityId::new(1),
            1.0,
            Timestamp::now(),
            Some(7),
            Some(1),
        )
        .unwrap();
        // Live (pre-crash) state already excludes it.
        assert!(db.hard_negatives().is_negative(7, 1));
    });

    // Reopen after the crash: the durable Tag::HardNeg row must repopulate the
    // bitmap so item 1 stays excluded from user 7's retrieve.
    let db = reopen(&home, durability_schema(4));
    assert!(
        db.hard_negatives().is_negative(7, 1),
        "hard-negative bitmap must be restored from durable rows after a crash"
    );

    let results = db
        .retrieve(
            &Retrieve::builder()
                .profile("new")
                .for_user(7)
                .limit(10)
                .build()
                .unwrap(),
        )
        .unwrap();
    let ids: Vec<u64> = results.items.iter().map(|r| r.entity_id.as_u64()).collect();
    assert!(
        !ids.contains(&1),
        "skip-rejected item 1 must NOT resurface in user 7's feed after a crash; got {ids:?}"
    );
    assert!(ids.contains(&2), "the un-rejected item 2 must still appear");
    db.close().unwrap();
}

// ── BLOCKER 3: interaction ledger survives a crash ───────────────────────────

#[test]
fn interaction_ledger_survives_crash_before_checkpoint() {
    let home = TempTidalHome::new().unwrap();
    let now = Timestamp::now();

    crash_before_checkpoint(&home, durability_schema(4), |db| {
        for id in 1u64..=3 {
            db.write_item_with_metadata(EntityId::new(id), &item_meta(id))
                .unwrap();
        }
        // User 5 interacts with three creators at different strengths.
        db.signal_with_context("like", EntityId::new(1), 5.0, now, Some(5), Some(100))
            .unwrap();
        db.signal_with_context("like", EntityId::new(2), 10.0, now, Some(5), Some(200))
            .unwrap();
        db.signal_with_context("like", EntityId::new(3), 1.0, now, Some(5), Some(300))
            .unwrap();
        // Pre-crash: creator 200 is the strongest.
        let top = db.interaction_ledger().top_creators(5, 3, now.as_nanos());
        assert_eq!(top[0].0, 200);
    });

    let db = reopen(&home, durability_schema(4));
    let top = db.interaction_ledger().top_creators(5, 3, now.as_nanos());
    assert_eq!(
        top.len(),
        3,
        "all three (user, creator) interaction edges must be reconstructed after a crash"
    );
    assert_eq!(top[0].0, 200, "creator ordering must survive the crash");
    assert_eq!(top[1].0, 100);
    assert_eq!(top[2].0, 300);
    // The strongest score must be reconstructed within decay tolerance.
    assert!(
        (top[0].1 - 10.0).abs() < 0.01,
        "reconstructed interaction score must match within decay tolerance, got {}",
        top[0].1
    );
    db.close().unwrap();
}

// ── CRITICAL 3: completion (InProgress) + save (DateSaved) survive a crash ───

#[test]
fn completion_and_save_survive_crash_before_checkpoint() {
    let home = TempTidalHome::new().unwrap();
    let now = Timestamp::now();

    crash_before_checkpoint(&home, durability_schema(4), |db| {
        for id in 1u64..=3 {
            db.write_item_with_metadata(EntityId::new(id), &item_meta(1))
                .unwrap();
            db.signal("view", EntityId::new(id), 1.0, now).unwrap();
        }
        // completion signal => completion ratio (weight is the ratio).
        db.signal_with_context("completion", EntityId::new(1), 0.5, now, Some(42), Some(1))
            .unwrap();
        // save signals at increasing timestamps for DateSaved ordering.
        db.signal_with_context(
            "save",
            EntityId::new(2),
            1.0,
            Timestamp::from_nanos(1000),
            Some(42),
            Some(1),
        )
        .unwrap();
        db.signal_with_context(
            "save",
            EntityId::new(3),
            1.0,
            Timestamp::from_nanos(3000),
            Some(42),
            Some(1),
        )
        .unwrap();
        // Pre-crash: item 1 is in progress, save timestamps recorded.
        assert!(db.user_state().is_in_progress(42, 1, 0.8));
        assert_eq!(db.user_state().get_save_timestamp(42, 3), Some(3000));
    });

    let db = reopen(&home, durability_schema(4));

    // InProgress filter must see the recovered completion ratio.
    assert!(
        db.user_state().is_in_progress(42, 1, 0.8),
        "completion ratio must be restored after a crash"
    );
    let in_progress = db
        .retrieve(
            &Retrieve::builder()
                .profile("new")
                .for_user(42)
                .filter(FilterExpr::InProgress {
                    user_id: 42,
                    threshold: 0.8,
                })
                .limit(10)
                .build()
                .unwrap(),
        )
        .unwrap();
    let ids: Vec<u64> = in_progress
        .items
        .iter()
        .map(|r| r.entity_id.as_u64())
        .collect();
    assert!(
        ids.contains(&1),
        "InProgress filter must match the recovered completion after a crash; got {ids:?}"
    );

    // DateSaved sort must see the recovered timestamps (item 3 saved latest).
    assert_eq!(
        db.user_state().get_save_timestamp(42, 3),
        Some(3000),
        "save timestamp must be restored after a crash"
    );
    let saved = db
        .retrieve(
            &Retrieve::builder()
                .profile("date_saved")
                .for_user(42)
                .limit(10)
                .build()
                .unwrap(),
        )
        .unwrap();
    let saved_ids: Vec<u64> = saved.items.iter().map(|r| r.entity_id.as_u64()).collect();
    let pos3 = saved_ids.iter().position(|&i| i == 3);
    let pos2 = saved_ids.iter().position(|&i| i == 2);
    assert!(
        pos3.is_some() && pos2.is_some() && pos3 < pos2,
        "DateSaved must order the later-saved item 3 before item 2 after a crash; got {saved_ids:?}"
    );
    db.close().unwrap();
}

// ── CRITICAL 4: preference vectors survive a crash (bounded window) ───────────

#[test]
fn preference_vectors_survive_crash_after_checkpoint() {
    let home = TempTidalHome::new().unwrap();
    let backup_home = TempTidalHome::new().unwrap();
    let now = Timestamp::now();

    // Phase 1: build a preference vector via a positive-engagement signal, then
    // force a checkpoint (create_backup checkpoints preference vectors into the
    // LIVE data dir before copying), then clean shutdown.
    let pref_before;
    {
        let db = reopen(&home, durability_schema(4));
        db.write_item_with_metadata(EntityId::new(1), &item_meta(1))
            .unwrap();
        db.write_item_embedding(EntityId::new(1), &[1.0, 0.0, 0.0, 0.0])
            .unwrap();
        db.signal_with_context("like", EntityId::new(1), 1.0, now, Some(77), Some(1))
            .unwrap();
        pref_before = db
            .preference_vectors()
            .get(77)
            .expect("preference vector must exist after a positive-engagement signal");
        // create_backup checkpoints the preference vectors to the live store.
        db.create_backup(backup_home.path()).unwrap();
        db.shutdown().unwrap();
    }

    // Phase 2: reopen (restores prefs from the checkpoint), then crash before the
    // next checkpoint without adding any new preference state. The restored vector
    // must still be present after the crash — it survived on the checkpoint alone.
    crash_before_checkpoint(&home, durability_schema(4), |db| {
        let p = db
            .preference_vectors()
            .get(77)
            .expect("preference vector must be restored on reopen from the checkpoint");
        assert_eq!(
            p, pref_before,
            "restored vector must equal the checkpointed one"
        );
    });

    // Phase 3: reopen after the crash — the vector must STILL be there (it was on
    // disk before the crash; the crash only skipped a fresh checkpoint).
    let db = reopen(&home, durability_schema(4));
    let pref_after = db.preference_vectors().get(77);
    assert_eq!(
        pref_after,
        Some(pref_before),
        "preference vector must survive a crash that happens after a checkpoint"
    );
    db.close().unwrap();
}

// ── BLOCKER 1 / CRITICAL 5: community contribution survives a crash ──────────

fn community_schema() -> tidaldb::schema::Schema {
    let mut b = SchemaBuilder::new();
    let _ = b
        .signal(
            "like",
            EntityKind::Item,
            DecaySpec::Exponential {
                half_life: Duration::from_secs(7 * 24 * 3600),
            },
        )
        .windows(&[Window::AllTime])
        .velocity(false)
        .add();
    b.build().unwrap()
}

#[test]
fn community_contribution_survives_crash_zero_loss_window() {
    let home = TempTidalHome::new().unwrap();
    let (community, item) = (CommunityId(1), EntityId::new(100));

    // Contribute, then CRASH before any checkpoint runs. Because the accepted
    // contribution synchronously checkpoints the community ledger before
    // returning Ok(true), the durable Tag::Community state already reflects it —
    // ZERO loss window even though the shutdown checkpoint never flushes.
    crash_before_checkpoint(&home, community_schema(), |db| {
        let policy = SharePolicy::community_share(1, [ShareIntent::Like]);
        for u in 1u64..=2 {
            db.join_community(UserId(u), community, policy.clone())
                .unwrap();
            assert!(
                db.signal_for_community("like", item, 1.0, Timestamp::now(), UserId(u), community)
                    .unwrap(),
                "contribution must be accepted (Ok(true))"
            );
        }
        assert_eq!(
            db.read_community_windowed_count(community, item, "like", Window::AllTime)
                .unwrap(),
            2,
            "pre-crash aggregate counts both contributors"
        );
    });

    let db = reopen(&home, community_schema());
    assert_eq!(
        db.read_community_windowed_count(community, item, "like", Window::AllTime)
            .unwrap(),
        2,
        "accepted community contributions must survive a crash (zero loss window)"
    );
    assert!(
        db.read_community_decay_score(community, item, "like", 0)
            .unwrap()
            .is_some(),
        "community decay score must be preserved after a crash"
    );
    db.close().unwrap();
}

// ── BLOCKER 4: agent-scope purge survives a crash ────────────────────────────

#[test]
fn agent_purge_survives_crash_before_checkpoint() {
    let home = TempTidalHome::new().unwrap();
    let (community, item) = (CommunityId(1), EntityId::new(100));
    let agent = "agent-x".to_string();

    // Phase 1: an agent contributes on behalf of users, then CLEAN shutdown so
    // the PRE-purge Tag::Community snapshot (with the agent's contributions) is on
    // disk. (community_contribute checkpoints synchronously, so the agent's writes
    // are durable after this phase.)
    {
        let db = reopen(&home, community_schema());
        let policy = SharePolicy::community_share(1, [ShareIntent::Like]);
        let now = Timestamp::now();
        db.grant_capability(
            &agent,
            vec![ScopePermission::read_write(ScopeClass::Community)],
            None,
        )
        .unwrap();
        for u in 1u64..=2 {
            db.join_community(UserId(u), community, policy.clone())
                .unwrap();
            assert!(
                db.agent_signal_for_community(&agent, "like", item, 1.0, now, UserId(u), community)
                    .unwrap(),
                "agent contribution must be accepted"
            );
        }
        assert_eq!(
            db.read_community_windowed_count(community, item, "like", Window::AllTime)
                .unwrap(),
            2
        );
        db.shutdown().unwrap();
    }

    // Phase 2: reopen, PURGE the agent (writes a durable AgentPurgeTombstone +
    // mutates the in-memory aggregate -> count 0), then CRASH before the next
    // checkpoint flushes. On-disk state is the (re-checkpointed) post-purge
    // snapshot AND the durable agent tombstone.
    crash_before_checkpoint(&home, community_schema(), |db| {
        assert_eq!(
            db.read_community_windowed_count(community, item, "like", Window::AllTime)
                .unwrap(),
            2,
            "reopen restores the pre-purge aggregate"
        );
        let receipt = db
            .remove_from_personalization(RemoveScope::CommunityAgent {
                writer_agent: agent.clone(),
                community,
            })
            .unwrap();
        assert_eq!(receipt.contributions_removed, 2);
        assert_eq!(
            db.read_community_windowed_count(community, item, "like", Window::AllTime)
                .unwrap(),
            0,
            "in-memory aggregate reflects the agent purge before the crash"
        );
    });

    // Phase 3: reopen after the crash. The agent's contributions must NOT
    // resurrect — the durable AgentPurgeTombstone is replayed over the restored
    // aggregate (and the re-checkpoint already reflected the purge).
    let db = reopen(&home, community_schema());
    assert_eq!(
        db.read_community_windowed_count(community, item, "like", Window::AllTime)
            .unwrap(),
        0,
        "agent purge must stay applied after a crash (tombstone replayed; \
         revoked agent's contributions must NOT resurrect)"
    );
    assert!(
        db.community_purge_watermark(community).is_some(),
        "the agent-purge watermark must be re-established after a crash"
    );
    db.close().unwrap();
}