tidaldb/tidal/tests/m8p3_reconcile_production.rs

//! Production CRDT reconciliation: `TidalDb::take_crdt_snapshot` +
//! `TidalDb::reconcile_with` heal a partition between two in-process nodes.
//!
//! These tests close the "CRDT engine is never invoked in production" gap: they
//! drive the LIVE production entry points on real `TidalDb` instances (not the
//! `ReconciliationEngine` directly), diverge two nodes, exchange snapshots,
//! reconcile, and assert what the heal actually guarantees.
//!
//! # The contract these tests hold (read before changing an expectation)
//!
//! `take_crdt_snapshot` keys every signal contribution to ONE canonical
//! contributor ([`ShardId::SINGLE`]), not to the local shard. Signals are
//! replicated from a single writer through the WAL relay, so each node's hot
//! accumulator ALREADY contains the other nodes' relayed events. Attributing it
//! per-node would fabricate N disjoint contributions for one logical stream,
//! which `CrdtSignalState::merge` then sums - double-counting every replicated
//! event on every reconcile (the creep a UAT caught). See the rationale on
//! `TidalDb::take_crdt_snapshot`.
//!
//! With one contributor the merge is therefore deterministic convergence, not
//! addition: the decay score is last-writer-wins on `(last_update_ns, score)`,
//! and the windowed bucket count is the PN-counter per-node max. Both nodes
//! converge on the MORE COMPLETE accumulator and stay there under repeated
//! exchange. A test that asserts `3 + 5 == 8` here is asserting the bug.
#![allow(clippy::unwrap_used, clippy::float_cmp)]

use std::time::Duration;

use tidaldb::{
    TidalDb,
    db::config::{NodeConfig, NodeRole},
    replication::ShardId,
    schema::{DecaySpec, EntityId, EntityKind, Schema, SchemaBuilder, Timestamp, Window},
};

/// A schema with a decaying "view" signal carrying an `AllTime` window (so the
/// windowed bucket count is observable through reconciliation — finding 6) and a
/// "skip" hard-negative signal.
fn schema() -> Schema {
    let mut builder = SchemaBuilder::new();
    let _ = builder
        .signal(
            "view",
            EntityKind::Item,
            DecaySpec::Exponential {
                half_life: Duration::from_secs(7 * 24 * 3600),
            },
        )
        .windows(&[Window::AllTime])
        .velocity(false)
        .add();
    let _ = builder
        .signal(
            "skip",
            EntityKind::Item,
            DecaySpec::Exponential {
                half_life: Duration::from_secs(7 * 24 * 3600),
            },
        )
        .windows(&[Window::AllTime])
        .velocity(false)
        .add();
    builder.build().unwrap()
}

/// Open an ephemeral node bound to `shard` (so its CRDT contributions are
/// attributed to a distinct node).
fn node(shard: ShardId) -> TidalDb {
    TidalDb::builder()
        .ephemeral()
        .with_schema(schema())
        .with_cluster(NodeConfig {
            role: NodeRole::Single,
            shard_id: shard,
            ..NodeConfig::default()
        })
        .open()
        .expect("ephemeral node opens")
}

/// Two diverged nodes exchange snapshots and reconcile. Both converge on the
/// more complete accumulator, the windowed count survives the round trip (it
/// does not drop to 0 - the original finding), and neither side inflates past
/// the true event total.
#[test]
fn two_node_divergence_converges_on_the_more_complete_accumulator() {
    let node_a = node(ShardId(0));
    let node_b = node(ShardId(1));
    let item = EntityId::new(42);

    // Node A saw 3 views, node B saw 5 of the same logical stream (B is the
    // more complete replica). Same fixed timestamp so decay is negligible.
    let ts = Timestamp::now();
    for _ in 0..3 {
        node_a.signal("view", item, 1.0, ts).unwrap();
    }
    for _ in 0..5 {
        node_b.signal("view", item, 1.0, ts).unwrap();
    }

    let a_count_before = node_a
        .read_windowed_count(item, "view", Window::AllTime)
        .unwrap();
    let b_count_before = node_b
        .read_windowed_count(item, "view", Window::AllTime)
        .unwrap();
    assert_eq!(a_count_before, 3, "node A sees only its 3 events pre-heal");
    assert_eq!(b_count_before, 5, "node B sees only its 5 events pre-heal");

    let b_score_before = node_b
        .read_decay_score(item, "view", 0)
        .unwrap()
        .unwrap_or(0.0);

    // ── Heal: exchange snapshots and reconcile each side. ──
    let snap_a = node_a.take_crdt_snapshot().unwrap();
    let snap_b = node_b.take_crdt_snapshot().unwrap();

    let ops_a = node_a.reconcile_with(&snap_b).unwrap();
    let ops_b = node_b.reconcile_with(&snap_a).unwrap();
    assert!(ops_a >= 1, "reconcile must apply at least the signal merge");
    assert!(ops_b >= 1);

    let a_count_after = node_a
        .read_windowed_count(item, "view", Window::AllTime)
        .unwrap();
    let b_count_after = node_b
        .read_windowed_count(item, "view", Window::AllTime)
        .unwrap();

    // Convergence is the property that matters: both sides agree.
    assert_eq!(
        a_count_after, b_count_after,
        "both nodes must converge on one windowed count after the exchange"
    );
    // ... on the more complete accumulator (PN-counter per-node max of 3 and 5),
    // never 0 (the count survives the snapshot round trip) and never 8 (summing
    // one logical stream twice is the double-count bug).
    assert_eq!(
        a_count_after, 5,
        "converged count is the more complete accumulator, not a sum"
    );

    // Decay score: LWW on (last_update_ns, score) with a single contributor, so
    // both sides hold node B's larger accumulator.
    let a_score_after = node_a
        .read_decay_score(item, "view", 0)
        .unwrap()
        .unwrap_or(0.0);
    let b_score_after = node_b
        .read_decay_score(item, "view", 0)
        .unwrap()
        .unwrap_or(0.0);
    // Tolerance: decay over the few-ms reconcile window is negligible but nonzero.
    assert!(
        (a_score_after - b_score_after).abs() < 1e-3,
        "both nodes must converge on one score: A {a_score_after} vs B {b_score_after}"
    );
    assert!(
        (a_score_after - b_score_before).abs() < 1e-3,
        "converged score {a_score_after} should be node B's {b_score_before}"
    );
}

/// Repeated exchange between already-converged nodes changes nothing.
///
/// This is the regression guard for the creep that per-node attribution caused:
/// with the whole accumulator attributed per shard, every reconcile re-summed
/// the same relayed events and the count grew without any new signal being
/// written. Under the canonical-contributor keying the second and third rounds
/// are exact no-ops.
#[test]
fn repeated_reconcile_of_converged_nodes_does_not_creep() {
    let node_a = node(ShardId(0));
    let node_b = node(ShardId(1));
    let item = EntityId::new(99);

    let ts = Timestamp::now();
    for _ in 0..4 {
        node_a.signal("view", item, 1.0, ts).unwrap();
    }
    for _ in 0..4 {
        node_b.signal("view", item, 1.0, ts).unwrap();
    }

    // First exchange converges the pair.
    let snap_b = node_b.take_crdt_snapshot().unwrap();
    node_a.reconcile_with(&snap_b).unwrap();
    let converged = node_a
        .read_windowed_count(item, "view", Window::AllTime)
        .unwrap();
    assert_eq!(converged, 4, "converged on the shared 4-event accumulator");

    // Two more rounds with FRESH snapshots taken after the merge - the shape
    // anti-entropy would actually run - must not move the count or the score.
    let score_after_first = node_a
        .read_decay_score(item, "view", 0)
        .unwrap()
        .unwrap_or(0.0);
    for round in 1..=2 {
        let fresh_a = node_a.take_crdt_snapshot().unwrap();
        let fresh_b = node_b.take_crdt_snapshot().unwrap();
        node_b.reconcile_with(&fresh_a).unwrap();
        node_a.reconcile_with(&fresh_b).unwrap();

        let count = node_a
            .read_windowed_count(item, "view", Window::AllTime)
            .unwrap();
        assert_eq!(
            count, converged,
            "round {round}: repeated reconcile must not inflate the count"
        );
        let score = node_a
            .read_decay_score(item, "view", 0)
            .unwrap()
            .unwrap_or(0.0);
        assert!(
            (score - score_after_first).abs() < 1e-3,
            "round {round}: repeated reconcile must not inflate the score \
             ({score} vs {score_after_first})"
        );
    }
}

/// Reconciliation against a remote snapshot the local node already covers is a
/// no-op for the windowed count: it never shrinks a locally-durable count, and
/// re-applying the same snapshot never grows one.
#[test]
fn reconcile_with_already_absorbed_remote_is_noop_for_count() {
    let node_a = node(ShardId(0));
    let node_b = node(ShardId(1));
    let item = EntityId::new(7);

    let ts = Timestamp::now();
    for _ in 0..6 {
        node_a.signal("view", item, 1.0, ts).unwrap();
    }
    for _ in 0..4 {
        node_b.signal("view", item, 1.0, ts).unwrap();
    }

    // Snapshot B (4 events) BEFORE any merge. A is the more complete side.
    let snap_b = node_b.take_crdt_snapshot().unwrap();

    // Merging a strictly smaller remote accumulator must not shrink A.
    node_a.reconcile_with(&snap_b).unwrap();
    let after_first = node_a
        .read_windowed_count(item, "view", Window::AllTime)
        .unwrap();
    assert_eq!(
        after_first, 6,
        "merging a smaller remote accumulator keeps the local 6"
    );

    // Re-merging the same snapshot is idempotent in both directions: no shrink
    // below 6 and no growth toward 6 + 4.
    node_a.reconcile_with(&snap_b).unwrap();
    let after_second = node_a
        .read_windowed_count(item, "view", Window::AllTime)
        .unwrap();
    assert_eq!(
        after_second, 6,
        "re-merge must neither shrink nor inflate the locally-durable count"
    );
}

/// Hard-negative divergence heals through the live production path: a hide on
/// one node propagates to the other after reconciliation.
#[test]
fn two_node_partition_heals_hard_negatives() {
    let node_a = node(ShardId(0));
    let node_b = node(ShardId(1));

    let user = EntityId::new(100);
    let item = EntityId::new(200);

    // Node A records a "skip" (a hard-negative signal) for (user, item) via the
    // context path, which populates node A's hard-negative index. Node B never
    // saw it during the partition.
    node_a
        .signal_with_context(
            "skip",
            item,
            1.0,
            Timestamp::now(),
            Some(user.as_u64()),
            None,
        )
        .unwrap();

    assert!(
        node_a
            .hard_negatives()
            .is_negative(user.as_u64(), item.as_u64() as u32),
        "node A must have the hard negative before reconcile"
    );
    assert!(
        !node_b
            .hard_negatives()
            .is_negative(user.as_u64(), item.as_u64() as u32),
        "node B must NOT have it before reconcile (partitioned)"
    );

    // Exchange snapshots and reconcile node B with node A's snapshot.
    let snap_a = node_a.take_crdt_snapshot().unwrap();
    node_b.reconcile_with(&snap_a).unwrap();

    assert!(
        node_b
            .hard_negatives()
            .is_negative(user.as_u64(), item.as_u64() as u32),
        "node B must converge to the hard negative after reconcile"
    );
}