//! A/B experiment framework for baseline comparison. //! //! Provides deterministic user-to-group assignment and metric aggregation for //! comparing personalized ranking against a non-personalized baseline feed. //! The experiment module is read-only with respect to storage -- all metrics //! are derived from existing `SignalLedger` and `UserSignalIndex` data. pub mod report; use std::time::Duration; use crate::schema::TidalError; // ── ExperimentGroup ────────────────────────────────────────────────────────── /// Which arm of an A/B experiment a user is assigned to. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum ExperimentGroup { /// The personalized (non-baseline) arm. Treatment, /// The baseline (non-personalized) arm. Control, } // ── ExperimentConfig ───────────────────────────────────────────────────────── /// Configuration for an A/B experiment comparing two ranking profiles. #[derive(Debug, Clone)] pub struct ExperimentConfig { /// Unique experiment identifier (e.g., "pg1-personalization-v1"). pub experiment_id: String, /// Fraction of users assigned to treatment (personalized). Must be in `0.0..=1.0`. /// Control gets the remainder. pub treatment_fraction: f64, /// Profile name for the treatment group (e.g., "`for_you`"). pub treatment_profile: String, /// Profile name for the control group (e.g., "chronological"). pub control_profile: String, /// Signals that count as "click" for CTR computation. pub click_signals: Vec, /// Signals that count as "completion" for completion rate. pub completion_signals: Vec, /// Observation window for return rate (e.g., 7 days). pub return_window: Duration, } impl ExperimentConfig { /// Validate the experiment configuration. /// /// # Errors /// /// Returns `TidalError::InvalidInput` if: /// - `treatment_fraction` is not in `0.0..=1.0` /// - `click_signals` is empty /// - `completion_signals` is empty pub fn validate(&self) -> crate::Result<()> { if !(0.0..=1.0).contains(&self.treatment_fraction) { return Err(TidalError::invalid_input(format!( "treatment_fraction must be in 0.0..=1.0, got {}", self.treatment_fraction ))); } if self.click_signals.is_empty() { return Err(TidalError::invalid_input("click_signals must not be empty")); } if self.completion_signals.is_empty() { return Err(TidalError::invalid_input( "completion_signals must not be empty", )); } Ok(()) } } // ── Assignment ─────────────────────────────────────────────────────────────── /// Assign a user to an experiment group. /// /// Uses FNV-1a hash over `experiment_id` and `user_id` for deterministic, /// stable assignment. The same `(user_id, experiment_id)` pair always maps /// to the same group. This is a pure function with no I/O or state. /// /// The hash is taken modulo 10,000 to produce a bucket. If the bucket is /// less than `treatment_fraction * 10_000`, the user is assigned to Treatment. #[must_use] pub fn assign_group(user_id: u64, experiment_id: &str, treatment_fraction: f64) -> ExperimentGroup { let hash = fnv1a_hash(experiment_id.as_bytes(), user_id); let bucket = hash % 10_000; #[allow(clippy::cast_sign_loss)] let threshold = (treatment_fraction * 10_000.0) as u64; if bucket < threshold { ExperimentGroup::Treatment } else { ExperimentGroup::Control } } /// FNV-1a hash combining `experiment_id` bytes and `user_id`. /// /// Deterministic across platforms: uses fixed FNV offset basis and prime, /// processes `experiment_id` bytes followed by `user_id` little-endian bytes. fn fnv1a_hash(experiment_bytes: &[u8], user_id: u64) -> u64 { const FNV_OFFSET_BASIS: u64 = 0xcbf2_9ce4_8422_2325; const FNV_PRIME: u64 = 0x0100_0000_01b3; let mut hash = FNV_OFFSET_BASIS; for &byte in experiment_bytes { hash ^= u64::from(byte); hash = hash.wrapping_mul(FNV_PRIME); } for &byte in &user_id.to_le_bytes() { hash ^= u64::from(byte); hash = hash.wrapping_mul(FNV_PRIME); } hash } // ── Report types ───────────────────────────────────────────────────────────── /// Per-group engagement metrics. #[derive(Debug, Clone, PartialEq)] pub struct GroupMetrics { /// Click-through rate: `total_clicks / total_views`. pub ctr: f64, /// Completion rate: `total_completions / total_views`. pub completion_rate: f64, /// Return rate: fraction of users who returned within the observation window. pub return_rate: f64, /// Total view events across all users in this group. pub total_views: u64, /// Total click events across all users in this group. pub total_clicks: u64, /// Total completion events across all users in this group. pub total_completions: u64, } impl GroupMetrics { /// Create zeroed-out group metrics. #[must_use] pub const fn zero() -> Self { Self { ctr: 0.0, completion_rate: 0.0, return_rate: 0.0, total_views: 0, total_clicks: 0, total_completions: 0, } } } /// Lift of treatment metrics relative to control. /// /// Each field is `(treatment - control) / control`. When the control value /// is zero and treatment is also zero, lift is `0.0`. When control is zero /// and treatment is positive, lift is `f64::INFINITY`. #[derive(Debug, Clone, PartialEq)] pub struct MetricLift { /// CTR lift: `(treatment_ctr - control_ctr) / control_ctr`. pub ctr_lift: f64, /// Completion rate lift. pub completion_lift: f64, /// Return rate lift. pub return_lift: f64, } impl MetricLift { /// Compute lift from treatment and control metrics. #[must_use] pub fn compute(treatment: &GroupMetrics, control: &GroupMetrics) -> Self { Self { ctr_lift: relative_lift(treatment.ctr, control.ctr), completion_lift: relative_lift(treatment.completion_rate, control.completion_rate), return_lift: relative_lift(treatment.return_rate, control.return_rate), } } } /// Compute relative lift: `(treatment - control) / control`. /// /// Edge cases: /// - Both zero: 0.0 /// - Control zero, treatment positive: `f64::INFINITY` /// - Control zero, treatment negative: `f64::NEG_INFINITY` fn relative_lift(treatment: f64, control: f64) -> f64 { if control == 0.0 { if treatment == 0.0 { 0.0 } else if treatment > 0.0 { f64::INFINITY } else { f64::NEG_INFINITY } } else { (treatment - control) / control } } /// Full A/B experiment comparison report. #[derive(Debug, Clone)] pub struct ExperimentReport { /// The experiment identifier. pub experiment_id: String, /// Number of users in the treatment group. pub treatment_users: usize, /// Number of users in the control group. pub control_users: usize, /// Aggregated metrics for treatment. pub treatment_metrics: GroupMetrics, /// Aggregated metrics for control. pub control_metrics: GroupMetrics, /// Lift of treatment over control. pub lift: MetricLift, } // ── Tests ──────────────────────────────────────────────────────────────────── #[cfg(test)] #[allow(clippy::unwrap_used, clippy::float_cmp)] mod tests { use super::*; #[test] fn assign_group_is_deterministic() { let group = assign_group(42, "exp-1", 0.5); for _ in 0..1000 { assert_eq!(assign_group(42, "exp-1", 0.5), group); } } #[test] fn different_experiment_id_can_change_group() { // With enough experiment IDs, at least one should differ for user 42. let base_group = assign_group(42, "exp-a", 0.5); let found_different = (0..100) .map(|i| assign_group(42, &format!("exp-{i}"), 0.5)) .any(|g| g != base_group); assert!( found_different, "different experiment IDs should sometimes produce different groups" ); } #[test] fn treatment_fraction_1_0_assigns_all_treatment() { for user_id in 0..10_000u64 { assert_eq!( assign_group(user_id, "all-treatment", 1.0), ExperimentGroup::Treatment ); } } #[test] fn treatment_fraction_0_0_assigns_all_control() { for user_id in 0..10_000u64 { assert_eq!( assign_group(user_id, "all-control", 0.0), ExperimentGroup::Control ); } } #[test] fn treatment_fraction_0_5_produces_balanced_split() { let treatment_count = (0..10_000u64) .filter(|&uid| assign_group(uid, "balanced-test", 0.5) == ExperimentGroup::Treatment) .count(); assert!( (4750..=5250).contains(&treatment_count), "expected ~5000 treatment users, got {treatment_count}" ); } #[test] fn treatment_fraction_0_1_produces_10_percent_split() { let treatment_count = (0..10_000u64) .filter(|&uid| assign_group(uid, "ten-percent", 0.1) == ExperimentGroup::Treatment) .count(); assert!( (800..=1200).contains(&treatment_count), "expected ~1000 treatment users, got {treatment_count}" ); } #[test] fn validate_rejects_out_of_range_fraction() { let config = ExperimentConfig { experiment_id: "test".into(), treatment_fraction: 1.5, treatment_profile: "for_you".into(), control_profile: "chronological".into(), click_signals: vec!["click".into()], completion_signals: vec!["complete".into()], return_window: Duration::from_secs(7 * 86400), }; assert!(config.validate().is_err()); } #[test] fn validate_rejects_negative_fraction() { let config = ExperimentConfig { experiment_id: "test".into(), treatment_fraction: -0.1, treatment_profile: "for_you".into(), control_profile: "chronological".into(), click_signals: vec!["click".into()], completion_signals: vec!["complete".into()], return_window: Duration::from_secs(7 * 86400), }; assert!(config.validate().is_err()); } #[test] fn validate_rejects_empty_click_signals() { let config = ExperimentConfig { experiment_id: "test".into(), treatment_fraction: 0.5, treatment_profile: "for_you".into(), control_profile: "chronological".into(), click_signals: vec![], completion_signals: vec!["complete".into()], return_window: Duration::from_secs(7 * 86400), }; assert!(config.validate().is_err()); } #[test] fn validate_rejects_empty_completion_signals() { let config = ExperimentConfig { experiment_id: "test".into(), treatment_fraction: 0.5, treatment_profile: "for_you".into(), control_profile: "chronological".into(), click_signals: vec!["click".into()], completion_signals: vec![], return_window: Duration::from_secs(7 * 86400), }; assert!(config.validate().is_err()); } #[test] fn validate_accepts_valid_config() { let config = ExperimentConfig { experiment_id: "test".into(), treatment_fraction: 0.5, treatment_profile: "for_you".into(), control_profile: "chronological".into(), click_signals: vec!["click".into()], completion_signals: vec!["complete".into()], return_window: Duration::from_secs(7 * 86400), }; assert!(config.validate().is_ok()); } #[test] fn validate_accepts_boundary_fractions() { let mut config = ExperimentConfig { experiment_id: "test".into(), treatment_fraction: 0.0, treatment_profile: "for_you".into(), control_profile: "chronological".into(), click_signals: vec!["click".into()], completion_signals: vec!["complete".into()], return_window: Duration::from_secs(7 * 86400), }; assert!(config.validate().is_ok()); config.treatment_fraction = 1.0; assert!(config.validate().is_ok()); } #[test] fn relative_lift_both_zero_is_zero() { assert_eq!(relative_lift(0.0, 0.0), 0.0); } #[test] fn relative_lift_control_zero_treatment_positive_is_infinity() { assert_eq!(relative_lift(0.5, 0.0), f64::INFINITY); } #[test] fn relative_lift_normal_case() { // treatment 0.3, control 0.2 => lift = (0.3 - 0.2) / 0.2 = 0.5 let lift = relative_lift(0.3, 0.2); assert!((lift - 0.5).abs() < 1e-10, "expected 0.5, got {lift}"); } #[test] fn relative_lift_treatment_worse() { // treatment 0.1, control 0.2 => lift = -0.5 let lift = relative_lift(0.1, 0.2); assert!((lift - (-0.5)).abs() < 1e-10, "expected -0.5, got {lift}"); } #[test] fn metric_lift_compute_all_fields() { let treatment = GroupMetrics { ctr: 0.3, completion_rate: 0.6, return_rate: 0.8, total_views: 100, total_clicks: 30, total_completions: 60, }; let control = GroupMetrics { ctr: 0.2, completion_rate: 0.4, return_rate: 0.5, total_views: 100, total_clicks: 20, total_completions: 40, }; let lift = MetricLift::compute(&treatment, &control); assert!((lift.ctr_lift - 0.5).abs() < 1e-10); assert!((lift.completion_lift - 0.5).abs() < 1e-10); assert!((lift.return_lift - 0.6).abs() < 1e-10); } }