//! Production CRDT reconciliation: `TidalDb::take_crdt_snapshot` + //! `TidalDb::reconcile_with` heal a partition between two in-process nodes. //! //! These tests close the "CRDT engine is never invoked in production" gap: they //! drive the LIVE production entry points on real `TidalDb` instances (not the //! `ReconciliationEngine` directly), diverge two nodes, exchange snapshots, //! reconcile, and assert what the heal actually guarantees. //! //! # The contract these tests hold (read before changing an expectation) //! //! `take_crdt_snapshot` keys every signal contribution to ONE canonical //! contributor ([`ShardId::SINGLE`]), not to the local shard. Signals are //! replicated from a single writer through the WAL relay, so each node's hot //! accumulator ALREADY contains the other nodes' relayed events. Attributing it //! per-node would fabricate N disjoint contributions for one logical stream, //! which `CrdtSignalState::merge` then sums - double-counting every replicated //! event on every reconcile (the creep a UAT caught). See the rationale on //! `TidalDb::take_crdt_snapshot`. //! //! With one contributor the merge is therefore deterministic convergence, not //! addition: the decay score is last-writer-wins on `(last_update_ns, score)`, //! and the windowed bucket count is the PN-counter per-node max. Both nodes //! converge on the MORE COMPLETE accumulator and stay there under repeated //! exchange. A test that asserts `3 + 5 == 8` here is asserting the bug. #![allow(clippy::unwrap_used, clippy::float_cmp)] use std::time::Duration; use tidaldb::{ TidalDb, db::config::{NodeConfig, NodeRole}, replication::ShardId, schema::{DecaySpec, EntityId, EntityKind, Schema, SchemaBuilder, Timestamp, Window}, }; /// A schema with a decaying "view" signal carrying an `AllTime` window (so the /// windowed bucket count is observable through reconciliation — finding 6) and a /// "skip" hard-negative signal. fn schema() -> Schema { let mut builder = SchemaBuilder::new(); let _ = builder .signal( "view", EntityKind::Item, DecaySpec::Exponential { half_life: Duration::from_secs(7 * 24 * 3600), }, ) .windows(&[Window::AllTime]) .velocity(false) .add(); let _ = builder .signal( "skip", EntityKind::Item, DecaySpec::Exponential { half_life: Duration::from_secs(7 * 24 * 3600), }, ) .windows(&[Window::AllTime]) .velocity(false) .add(); builder.build().unwrap() } /// Open an ephemeral node bound to `shard` (so its CRDT contributions are /// attributed to a distinct node). fn node(shard: ShardId) -> TidalDb { TidalDb::builder() .ephemeral() .with_schema(schema()) .with_cluster(NodeConfig { role: NodeRole::Single, shard_id: shard, ..NodeConfig::default() }) .open() .expect("ephemeral node opens") } /// Two diverged nodes exchange snapshots and reconcile. Both converge on the /// more complete accumulator, the windowed count survives the round trip (it /// does not drop to 0 - the original finding), and neither side inflates past /// the true event total. #[test] fn two_node_divergence_converges_on_the_more_complete_accumulator() { let node_a = node(ShardId(0)); let node_b = node(ShardId(1)); let item = EntityId::new(42); // Node A saw 3 views, node B saw 5 of the same logical stream (B is the // more complete replica). Same fixed timestamp so decay is negligible. let ts = Timestamp::now(); for _ in 0..3 { node_a.signal("view", item, 1.0, ts).unwrap(); } for _ in 0..5 { node_b.signal("view", item, 1.0, ts).unwrap(); } let a_count_before = node_a .read_windowed_count(item, "view", Window::AllTime) .unwrap(); let b_count_before = node_b .read_windowed_count(item, "view", Window::AllTime) .unwrap(); assert_eq!(a_count_before, 3, "node A sees only its 3 events pre-heal"); assert_eq!(b_count_before, 5, "node B sees only its 5 events pre-heal"); let b_score_before = node_b .read_decay_score(item, "view", 0) .unwrap() .unwrap_or(0.0); // ── Heal: exchange snapshots and reconcile each side. ── let snap_a = node_a.take_crdt_snapshot().unwrap(); let snap_b = node_b.take_crdt_snapshot().unwrap(); let ops_a = node_a.reconcile_with(&snap_b).unwrap(); let ops_b = node_b.reconcile_with(&snap_a).unwrap(); assert!(ops_a >= 1, "reconcile must apply at least the signal merge"); assert!(ops_b >= 1); let a_count_after = node_a .read_windowed_count(item, "view", Window::AllTime) .unwrap(); let b_count_after = node_b .read_windowed_count(item, "view", Window::AllTime) .unwrap(); // Convergence is the property that matters: both sides agree. assert_eq!( a_count_after, b_count_after, "both nodes must converge on one windowed count after the exchange" ); // ... on the more complete accumulator (PN-counter per-node max of 3 and 5), // never 0 (the count survives the snapshot round trip) and never 8 (summing // one logical stream twice is the double-count bug). assert_eq!( a_count_after, 5, "converged count is the more complete accumulator, not a sum" ); // Decay score: LWW on (last_update_ns, score) with a single contributor, so // both sides hold node B's larger accumulator. let a_score_after = node_a .read_decay_score(item, "view", 0) .unwrap() .unwrap_or(0.0); let b_score_after = node_b .read_decay_score(item, "view", 0) .unwrap() .unwrap_or(0.0); // Tolerance: decay over the few-ms reconcile window is negligible but nonzero. assert!( (a_score_after - b_score_after).abs() < 1e-3, "both nodes must converge on one score: A {a_score_after} vs B {b_score_after}" ); assert!( (a_score_after - b_score_before).abs() < 1e-3, "converged score {a_score_after} should be node B's {b_score_before}" ); } /// Repeated exchange between already-converged nodes changes nothing. /// /// This is the regression guard for the creep that per-node attribution caused: /// with the whole accumulator attributed per shard, every reconcile re-summed /// the same relayed events and the count grew without any new signal being /// written. Under the canonical-contributor keying the second and third rounds /// are exact no-ops. #[test] fn repeated_reconcile_of_converged_nodes_does_not_creep() { let node_a = node(ShardId(0)); let node_b = node(ShardId(1)); let item = EntityId::new(99); let ts = Timestamp::now(); for _ in 0..4 { node_a.signal("view", item, 1.0, ts).unwrap(); } for _ in 0..4 { node_b.signal("view", item, 1.0, ts).unwrap(); } // First exchange converges the pair. let snap_b = node_b.take_crdt_snapshot().unwrap(); node_a.reconcile_with(&snap_b).unwrap(); let converged = node_a .read_windowed_count(item, "view", Window::AllTime) .unwrap(); assert_eq!(converged, 4, "converged on the shared 4-event accumulator"); // Two more rounds with FRESH snapshots taken after the merge - the shape // anti-entropy would actually run - must not move the count or the score. let score_after_first = node_a .read_decay_score(item, "view", 0) .unwrap() .unwrap_or(0.0); for round in 1..=2 { let fresh_a = node_a.take_crdt_snapshot().unwrap(); let fresh_b = node_b.take_crdt_snapshot().unwrap(); node_b.reconcile_with(&fresh_a).unwrap(); node_a.reconcile_with(&fresh_b).unwrap(); let count = node_a .read_windowed_count(item, "view", Window::AllTime) .unwrap(); assert_eq!( count, converged, "round {round}: repeated reconcile must not inflate the count" ); let score = node_a .read_decay_score(item, "view", 0) .unwrap() .unwrap_or(0.0); assert!( (score - score_after_first).abs() < 1e-3, "round {round}: repeated reconcile must not inflate the score \ ({score} vs {score_after_first})" ); } } /// Reconciliation against a remote snapshot the local node already covers is a /// no-op for the windowed count: it never shrinks a locally-durable count, and /// re-applying the same snapshot never grows one. #[test] fn reconcile_with_already_absorbed_remote_is_noop_for_count() { let node_a = node(ShardId(0)); let node_b = node(ShardId(1)); let item = EntityId::new(7); let ts = Timestamp::now(); for _ in 0..6 { node_a.signal("view", item, 1.0, ts).unwrap(); } for _ in 0..4 { node_b.signal("view", item, 1.0, ts).unwrap(); } // Snapshot B (4 events) BEFORE any merge. A is the more complete side. let snap_b = node_b.take_crdt_snapshot().unwrap(); // Merging a strictly smaller remote accumulator must not shrink A. node_a.reconcile_with(&snap_b).unwrap(); let after_first = node_a .read_windowed_count(item, "view", Window::AllTime) .unwrap(); assert_eq!( after_first, 6, "merging a smaller remote accumulator keeps the local 6" ); // Re-merging the same snapshot is idempotent in both directions: no shrink // below 6 and no growth toward 6 + 4. node_a.reconcile_with(&snap_b).unwrap(); let after_second = node_a .read_windowed_count(item, "view", Window::AllTime) .unwrap(); assert_eq!( after_second, 6, "re-merge must neither shrink nor inflate the locally-durable count" ); } /// Hard-negative divergence heals through the live production path: a hide on /// one node propagates to the other after reconciliation. #[test] fn two_node_partition_heals_hard_negatives() { let node_a = node(ShardId(0)); let node_b = node(ShardId(1)); let user = EntityId::new(100); let item = EntityId::new(200); // Node A records a "skip" (a hard-negative signal) for (user, item) via the // context path, which populates node A's hard-negative index. Node B never // saw it during the partition. node_a .signal_with_context( "skip", item, 1.0, Timestamp::now(), Some(user.as_u64()), None, ) .unwrap(); assert!( node_a .hard_negatives() .is_negative(user.as_u64(), item.as_u64() as u32), "node A must have the hard negative before reconcile" ); assert!( !node_b .hard_negatives() .is_negative(user.as_u64(), item.as_u64() as u32), "node B must NOT have it before reconcile (partitioned)" ); // Exchange snapshots and reconcile node B with node A's snapshot. let snap_a = node_a.take_crdt_snapshot().unwrap(); node_b.reconcile_with(&snap_a).unwrap(); assert!( node_b .hard_negatives() .is_negative(user.as_u64(), item.as_u64() as u32), "node B must converge to the hard negative after reconcile" ); }