foxhunt/crates/ml-ensemble/src/ab_testing.rs

//! A/B testing framework for ensemble vs single-model comparison
//!
//! This module provides statistical testing infrastructure to compare ensemble
//! predictions against single-model baselines with rigorous significance testing.

use serde::{Deserialize, Serialize};
use statrs::distribution::{ContinuousCDF, Normal, StudentsT};
use std::collections::HashMap;
use std::sync::Arc;
use thiserror::Error;
use tokio::sync::RwLock;
use uuid::Uuid;

use crate::EnsembleError;

/// A/B test group assignment
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Hash)]
pub enum ABGroup {
    /// Control group (baseline single model)
    Control,
    /// Treatment group (ensemble model)
    Treatment,
}

impl ABGroup {
    pub fn as_str(&self) -> &'static str {
        match self {
            ABGroup::Control => "control",
            ABGroup::Treatment => "treatment",
        }
    }
}

/// Configuration for an A/B test
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ABTestConfig {
    /// Unique test identifier
    pub test_id: String,
    /// Control model variant (e.g., "DQN")
    pub control_model: String,
    /// Treatment model variant (e.g., "Ensemble")
    pub treatment_model: String,
    /// Traffic split (0.0-1.0, where 0.5 = 50/50)
    pub traffic_split: f64,
    /// Minimum sample size per group before computing significance
    pub min_sample_size: usize,
    /// Statistical significance level (alpha)
    pub significance_level: f64,
    /// Maximum test duration in hours
    pub max_duration_hours: u64,
    /// Test start timestamp
    pub start_time: i64,
}

impl Default for ABTestConfig {
    fn default() -> Self {
        Self {
            test_id: Uuid::new_v4().to_string(),
            control_model: "DQN".to_owned(),
            treatment_model: "Ensemble".to_owned(),
            traffic_split: 0.5,
            min_sample_size: 1000,
            significance_level: 0.05,
            max_duration_hours: 168, // 1 week
            start_time: chrono::Utc::now().timestamp(),
        }
    }
}

/// Metrics tracked for each A/B test group
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct GroupMetrics {
    /// Total number of predictions
    pub predictions: u64,
    /// Number of correct predictions
    pub correct_predictions: u64,
    /// Total profit and loss
    pub total_pnl: f64,
    /// Individual PnL samples for statistical tests
    pub pnl_samples: Vec<f64>,
    /// Individual returns for Sharpe ratio calculation
    pub returns: Vec<f64>,
    /// Average latency in microseconds
    pub avg_latency_us: f64,
    /// Latency samples
    pub latency_samples: Vec<u64>,
}

impl GroupMetrics {
    /// Win rate (correct predictions / total predictions)
    pub fn win_rate(&self) -> f64 {
        if self.predictions == 0 {
            0.0
        } else {
            self.correct_predictions as f64 / self.predictions as f64
        }
    }

    /// Sharpe ratio (annualized)
    pub fn sharpe_ratio(&self) -> f64 {
        if self.returns.is_empty() {
            0.0
        } else {
            let mean_return = self.returns.iter().sum::<f64>() / self.returns.len() as f64;
            let variance = self
                .returns
                .iter()
                .map(|r| (r - mean_return).powi(2))
                .sum::<f64>()
                / self.returns.len() as f64;
            let std_dev = variance.sqrt();

            if std_dev < 1e-10 {
                0.0
            } else {
                // Annualize: assume 252 trading days, multiply by sqrt(252)
                mean_return / std_dev * 15.874 // sqrt(252) ≈ 15.874
            }
        }
    }

    /// Average PnL per prediction
    pub fn avg_pnl(&self) -> f64 {
        if self.predictions == 0 {
            0.0
        } else {
            self.total_pnl / self.predictions as f64
        }
    }

    /// Record a new prediction outcome
    pub fn record_prediction(&mut self, correct: bool, pnl: f64, return_pct: f64, latency_us: u64) {
        self.predictions += 1;
        if correct {
            self.correct_predictions += 1;
        }
        self.total_pnl += pnl;
        self.pnl_samples.push(pnl);
        self.returns.push(return_pct);
        self.latency_samples.push(latency_us);

        // Update rolling average latency
        let total_latency: u64 = self.latency_samples.iter().sum();
        self.avg_latency_us = total_latency as f64 / self.latency_samples.len() as f64;
    }
}

/// Statistical test results
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StatisticalTestResult {
    /// Test statistic value
    pub test_statistic: f64,
    /// P-value (probability of observing results under null hypothesis)
    pub p_value: f64,
    /// Whether result is statistically significant
    pub is_significant: bool,
    /// Confidence interval (95%)
    pub confidence_interval: (f64, f64),
}

/// Complete A/B test results with all metrics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ABTestResults {
    /// Test configuration
    pub test_id: String,
    /// Control group metrics
    pub control_group: GroupMetrics,
    /// Treatment group metrics
    pub treatment_group: GroupMetrics,

    /// Sharpe ratio difference (treatment - control)
    pub sharpe_diff: f64,
    /// Sharpe ratio test result
    pub sharpe_test: StatisticalTestResult,

    /// Win rate difference (treatment - control)
    pub win_rate_diff: f64,
    /// Win rate test result
    pub win_rate_test: StatisticalTestResult,

    /// PnL difference (treatment - control)
    pub pnl_diff: f64,
    /// PnL test result
    pub pnl_test: StatisticalTestResult,

    /// Overall recommendation
    pub recommendation: Recommendation,
}

/// Recommendation based on A/B test results
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum Recommendation {
    /// Treatment is significantly better, roll out to 100%
    RolloutTreatment(String),
    /// Control is significantly better, revert to baseline
    RevertToControl(String),
    /// No significant difference, continue testing or use simpler model
    Neutral(String),
    /// Not enough data yet, continue testing
    Inconclusive(String),
}

/// A/B test router for traffic splitting and group assignment
pub struct ABTestRouter {
    config: ABTestConfig,
    group_assignments: Arc<RwLock<HashMap<String, ABGroup>>>,
    metrics_tracker: Arc<ABMetricsTracker>,
}

impl std::fmt::Debug for ABTestRouter {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("ABTestRouter")
            .field("config", &self.config)
            .field("group_assignments", &"Arc<RwLock<HashMap<..>>>")
            .field("metrics_tracker", &"Arc<ABMetricsTracker>")
            .finish()
    }
}

impl ABTestRouter {
    /// Create a new A/B test router
    pub fn new(config: ABTestConfig) -> Self {
        Self {
            config: config.clone(),
            group_assignments: Arc::new(RwLock::new(HashMap::new())),
            metrics_tracker: Arc::new(ABMetricsTracker::new(config)),
        }
    }

    /// Get or assign a user to a group using deterministic hashing
    pub async fn get_or_assign_group(&self, user_id: &str) -> ABGroup {
        // Check if already assigned
        {
            let assignments = self.group_assignments.read().await;
            if let Some(&group) = assignments.get(user_id) {
                return group;
            }
        }

        // Assign new user using consistent hashing
        let group = self.assign_group(user_id);

        // Cache assignment
        self.group_assignments
            .write()
            .await
            .insert(user_id.to_string(), group);

        group
    }

    /// Assign user to group using deterministic hash
    fn assign_group(&self, user_id: &str) -> ABGroup {
        // Use simple hash for deterministic assignment
        let hash = user_id.bytes().enumerate().fold(0_u64, |acc, (i, b)| {
            acc.wrapping_add((b as u64).wrapping_mul((i as u64).wrapping_add(1)))
        });

        // Convert to 0-100 range
        let bucket = (hash % 100) as f64 / 100.0;

        if bucket < self.config.traffic_split {
            ABGroup::Treatment
        } else {
            ABGroup::Control
        }
    }

    /// Record prediction outcome for metrics tracking
    pub async fn record_outcome(
        &self,
        group: ABGroup,
        correct: bool,
        pnl: f64,
        return_pct: f64,
        latency_us: u64,
    ) {
        self.metrics_tracker
            .record_prediction(group, correct, pnl, return_pct, latency_us)
            .await;
    }

    /// Get current A/B test results
    pub async fn get_results(&self) -> Result<ABTestResults, ABTestError> {
        self.metrics_tracker.compute_significance().await
    }

    /// Get configuration
    pub fn config(&self) -> &ABTestConfig {
        &self.config
    }
}

/// Metrics tracker for A/B test groups
pub struct ABMetricsTracker {
    config: ABTestConfig,
    control_metrics: Arc<RwLock<GroupMetrics>>,
    treatment_metrics: Arc<RwLock<GroupMetrics>>,
}

impl std::fmt::Debug for ABMetricsTracker {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("ABMetricsTracker")
            .field("config", &self.config)
            .field("control_metrics", &"Arc<RwLock<GroupMetrics>>")
            .field("treatment_metrics", &"Arc<RwLock<GroupMetrics>>")
            .finish()
    }
}

impl ABMetricsTracker {
    /// Create new metrics tracker
    pub fn new(config: ABTestConfig) -> Self {
        Self {
            config,
            control_metrics: Arc::new(RwLock::new(GroupMetrics::default())),
            treatment_metrics: Arc::new(RwLock::new(GroupMetrics::default())),
        }
    }

    /// Record a prediction outcome
    pub async fn record_prediction(
        &self,
        group: ABGroup,
        correct: bool,
        pnl: f64,
        return_pct: f64,
        latency_us: u64,
    ) {
        match group {
            ABGroup::Control => {
                self.control_metrics
                    .write()
                    .await
                    .record_prediction(correct, pnl, return_pct, latency_us);
            },
            ABGroup::Treatment => {
                self.treatment_metrics
                    .write()
                    .await
                    .record_prediction(correct, pnl, return_pct, latency_us);
            },
        }
    }

    /// Compute statistical significance of A/B test results
    pub async fn compute_significance(&self) -> Result<ABTestResults, ABTestError> {
        let control = self.control_metrics.read().await.clone();
        let treatment = self.treatment_metrics.read().await.clone();

        // Check minimum sample size
        if control.predictions < self.config.min_sample_size as u64
            || treatment.predictions < self.config.min_sample_size as u64
        {
            return Err(ABTestError::InsufficientSamples {
                required: self.config.min_sample_size,
                control: control.predictions as usize,
                treatment: treatment.predictions as usize,
            });
        }

        // Sharpe ratio comparison (Welch's t-test on returns)
        let sharpe_diff = treatment.sharpe_ratio() - control.sharpe_ratio();
        let sharpe_test = self.welch_t_test(&control.returns, &treatment.returns)?;

        // Win rate comparison (proportion z-test)
        let win_rate_diff = treatment.win_rate() - control.win_rate();
        let win_rate_test = self.proportion_z_test(
            control.correct_predictions,
            control.predictions,
            treatment.correct_predictions,
            treatment.predictions,
        )?;

        // PnL comparison (Mann-Whitney U test for non-normal distributions)
        let pnl_diff = treatment.total_pnl - control.total_pnl;
        let pnl_test = self.mann_whitney_u_test(&control.pnl_samples, &treatment.pnl_samples)?;

        // Generate recommendation
        let recommendation =
            self.generate_recommendation(sharpe_diff, &sharpe_test, pnl_diff, &pnl_test);

        Ok(ABTestResults {
            test_id: self.config.test_id.clone(),
            control_group: control,
            treatment_group: treatment,
            sharpe_diff,
            sharpe_test,
            win_rate_diff,
            win_rate_test,
            pnl_diff,
            pnl_test,
            recommendation,
        })
    }

    /// Welch's t-test for two samples with unequal variances
    pub fn welch_t_test(
        &self,
        sample1: &[f64],
        sample2: &[f64],
    ) -> Result<StatisticalTestResult, ABTestError> {
        if sample1.is_empty() || sample2.is_empty() {
            return Err(ABTestError::EmptySamples);
        }

        let n1 = sample1.len() as f64;
        let n2 = sample2.len() as f64;

        // Calculate means
        let mean1 = sample1.iter().sum::<f64>() / n1;
        let mean2 = sample2.iter().sum::<f64>() / n2;

        // Calculate variances
        let var1 = sample1.iter().map(|x| (x - mean1).powi(2)).sum::<f64>() / (n1 - 1.0);
        let var2 = sample2.iter().map(|x| (x - mean2).powi(2)).sum::<f64>() / (n2 - 1.0);

        // Welch's t-statistic
        let t_stat = (mean1 - mean2) / ((var1 / n1) + (var2 / n2)).sqrt();

        // Welch-Satterthwaite degrees of freedom
        let numerator = ((var1 / n1) + (var2 / n2)).powi(2);
        let denominator = (var1 / n1).powi(2) / (n1 - 1.0) + (var2 / n2).powi(2) / (n2 - 1.0);
        let df = numerator / denominator;

        // Approximate p-value using t-distribution (two-tailed)
        let p_value = self.t_distribution_p_value(t_stat.abs(), df);

        // 95% confidence interval for difference in means
        let t_critical = self.t_critical_value(0.05, df);
        let se = ((var1 / n1) + (var2 / n2)).sqrt();
        let diff = mean1 - mean2;
        let ci = (diff - t_critical * se, diff + t_critical * se);

        Ok(StatisticalTestResult {
            test_statistic: t_stat,
            p_value,
            is_significant: p_value < self.config.significance_level,
            confidence_interval: ci,
        })
    }

    /// Proportion z-test for comparing two proportions (win rates)
    pub fn proportion_z_test(
        &self,
        successes1: u64,
        total1: u64,
        successes2: u64,
        total2: u64,
    ) -> Result<StatisticalTestResult, ABTestError> {
        if total1 == 0 || total2 == 0 {
            return Err(ABTestError::EmptySamples);
        }

        let p1 = successes1 as f64 / total1 as f64;
        let p2 = successes2 as f64 / total2 as f64;

        // Pooled proportion
        let p_pool = (successes1 + successes2) as f64 / (total1 + total2) as f64;

        // Standard error
        let se = (p_pool * (1.0 - p_pool) * (1.0 / total1 as f64 + 1.0 / total2 as f64)).sqrt();

        // Z-statistic
        let z_stat = (p1 - p2) / se;

        // Two-tailed p-value
        let p_value = 2.0 * (1.0 - self.standard_normal_cdf(z_stat.abs()));

        // 95% confidence interval for difference in proportions
        let z_critical = 1.96; // 95% CI
        let se_diff =
            ((p1 * (1.0 - p1) / total1 as f64) + (p2 * (1.0 - p2) / total2 as f64)).sqrt();
        let diff = p1 - p2;
        let ci = (diff - z_critical * se_diff, diff + z_critical * se_diff);

        Ok(StatisticalTestResult {
            test_statistic: z_stat,
            p_value,
            is_significant: p_value < self.config.significance_level,
            confidence_interval: ci,
        })
    }

    /// Mann-Whitney U test for non-parametric comparison (PnL distributions)
    pub fn mann_whitney_u_test(
        &self,
        sample1: &[f64],
        sample2: &[f64],
    ) -> Result<StatisticalTestResult, ABTestError> {
        if sample1.is_empty() || sample2.is_empty() {
            return Err(ABTestError::EmptySamples);
        }

        let n1 = sample1.len();
        let n2 = sample2.len();

        // Combine samples with group labels
        let mut combined: Vec<(f64, u8)> = sample1.iter().map(|&x| (x, 1)).collect();
        combined.extend(sample2.iter().map(|&x| (x, 2)));

        // Sort by value
        combined.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal));

        // Assign ranks (handle ties with average rank)
        let mut ranks1 = 0.0;
        let mut i = 0;
        while i < combined.len() {
            let mut j = i;
            while j < combined.len() && (combined[j].0 - combined[i].0).abs() < 1e-10 {
                j += 1;
            }
            // Average rank for ties
            let avg_rank = (i + j + 1) as f64 / 2.0;
            for k in i..j {
                if combined[k].1 == 1 {
                    ranks1 += avg_rank;
                }
            }
            i = j;
        }

        // U statistic for sample 1
        let u1 = ranks1 - (n1 * (n1 + 1)) as f64 / 2.0;

        // Mean and standard deviation under null hypothesis
        let mean_u = (n1 * n2) as f64 / 2.0;
        let std_u = ((n1 * n2 * (n1 + n2 + 1)) as f64 / 12.0).sqrt();

        // Z-statistic (normal approximation for large samples)
        let z_stat = (u1 - mean_u) / std_u;

        // Two-tailed p-value
        let p_value = 2.0 * (1.0 - self.standard_normal_cdf(z_stat.abs()));

        // For Mann-Whitney, CI is more complex, use point estimate difference
        let median1 = self.median(sample1);
        let median2 = self.median(sample2);
        let ci = (median1 - median2, median1 - median2); // Simplified

        Ok(StatisticalTestResult {
            test_statistic: z_stat,
            p_value,
            is_significant: p_value < self.config.significance_level,
            confidence_interval: ci,
        })
    }

    /// Generate recommendation based on test results
    fn generate_recommendation(
        &self,
        sharpe_diff: f64,
        sharpe_test: &StatisticalTestResult,
        pnl_diff: f64,
        pnl_test: &StatisticalTestResult,
    ) -> Recommendation {
        // Check if results are statistically significant
        if !sharpe_test.is_significant && !pnl_test.is_significant {
            return Recommendation::Inconclusive(
                "Results not statistically significant. Continue testing or increase sample size."
                    .to_string(),
            );
        }

        // If only one metric is significant, be cautious
        let both_significant = sharpe_test.is_significant && pnl_test.is_significant;

        // Strong positive signal: both metrics significantly better
        if both_significant && sharpe_diff > 0.2 && pnl_diff > 0.0 {
            return Recommendation::RolloutTreatment(
                format!(
                    "Ensemble significantly outperforms baseline: Sharpe +{:.2} (p={:.4}), PnL +${:.2} (p={:.4}). Roll out to 100%.",
                    sharpe_diff, sharpe_test.p_value, pnl_diff, pnl_test.p_value
                )
            );
        }

        // Strong negative signal: both metrics significantly worse
        if both_significant && sharpe_diff < -0.2 && pnl_diff < 0.0 {
            return Recommendation::RevertToControl(
                format!(
                    "Ensemble significantly underperforms baseline: Sharpe {:.2} (p={:.4}), PnL ${:.2} (p={:.4}). Revert to control.",
                    sharpe_diff, sharpe_test.p_value, pnl_diff, pnl_test.p_value
                )
            );
        }

        // Moderate positive signal
        if sharpe_diff > 0.1 && pnl_diff > 0.0 {
            return Recommendation::RolloutTreatment(
                format!(
                    "Ensemble shows improvement: Sharpe +{:.2}, PnL +${:.2}. Consider gradual rollout to 100%.",
                    sharpe_diff, pnl_diff
                )
            );
        }

        // Moderate negative signal
        if sharpe_diff < -0.1 && pnl_diff < 0.0 {
            return Recommendation::RevertToControl(
                format!(
                    "Ensemble shows degradation: Sharpe {:.2}, PnL ${:.2}. Consider reverting to control.",
                    sharpe_diff, pnl_diff
                )
            );
        }

        // No meaningful difference
        Recommendation::Neutral(
            format!(
                "No meaningful difference detected (Sharpe diff: {:.2}, PnL diff: ${:.2}). Use simpler single-model for operational efficiency.",
                sharpe_diff, pnl_diff
            )
        )
    }

    /// Calculate median of a sample
    fn median(&self, samples: &[f64]) -> f64 {
        if samples.is_empty() {
            return 0.0;
        }
        let mut sorted = samples.to_vec();
        sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
        let mid = sorted.len() / 2;
        if sorted.len() % 2 == 0 {
            (sorted[mid - 1] + sorted[mid]) / 2.0
        } else {
            sorted[mid]
        }
    }

    /// CDF of standard normal distribution using statrs
    fn standard_normal_cdf(&self, x: f64) -> f64 {
        // Fallback to 0.5 if Normal::new fails (should never happen with 0.0, 1.0)
        match Normal::new(0.0, 1.0) {
            Ok(dist) => dist.cdf(x),
            Err(_) => 0.5,
        }
    }

    /// P-value for t-distribution using statrs StudentsT
    fn t_distribution_p_value(&self, t: f64, df: f64) -> f64 {
        // Use statrs StudentsT for exact CDF computation
        match StudentsT::new(0.0, 1.0, df) {
            Ok(dist) => 2.0 * (1.0 - dist.cdf(t.abs())),
            Err(_) => {
                // Fallback to normal approximation if StudentsT::new fails
                // (e.g., df <= 0 which shouldn't happen in practice)
                2.0 * (1.0 - self.standard_normal_cdf(t.abs()))
            },
        }
    }

    /// Critical t-value for given alpha and df using statrs inverse CDF
    fn t_critical_value(&self, alpha: f64, df: f64) -> f64 {
        match StudentsT::new(0.0, 1.0, df) {
            Ok(dist) => dist.inverse_cdf(1.0 - alpha / 2.0),
            Err(_) => 1.96, // Fallback to normal approximation
        }
    }

    /// Calculate minimum sample size needed for desired power using statrs Normal inverse CDF
    pub fn calculate_min_sample_size(effect_size: f64, power: f64, alpha: f64) -> usize {
        if effect_size.abs() < 1e-15 {
            return usize::MAX;
        }

        let normal = match Normal::new(0.0, 1.0) {
            Ok(dist) => dist,
            Err(_) => return usize::MAX,
        };

        let z_alpha = normal.inverse_cdf(1.0 - alpha / 2.0);
        let z_beta = normal.inverse_cdf(power);

        let n = 2.0 * ((z_alpha + z_beta) / effect_size).powi(2);
        n.ceil() as usize
    }
}

/// Errors that can occur in A/B testing
#[derive(Error, Debug)]
pub enum ABTestError {
    #[error(
        "Insufficient samples: required {required}, got control={control}, treatment={treatment}"
    )]
    InsufficientSamples {
        required: usize,
        control: usize,
        treatment: usize,
    },

    #[error("Empty samples provided for statistical test")]
    EmptySamples,

    #[error("Invalid configuration: {0}")]
    InvalidConfiguration(String),

    #[error("Test expired: duration {elapsed_hours}h exceeds maximum {max_hours}h")]
    TestExpired { elapsed_hours: u64, max_hours: u64 },
}

impl From<ABTestError> for EnsembleError {
    fn from(err: ABTestError) -> Self {
        EnsembleError::AggregationFailed(err.to_string())
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn test_group_assignment_deterministic() {
        let config = ABTestConfig {
            traffic_split: 0.5,
            ..Default::default()
        };
        let router = ABTestRouter::new(config);

        // Same user ID should always get same group
        let user_id = "user123";
        let group1 = router.get_or_assign_group(user_id).await;
        let group2 = router.get_or_assign_group(user_id).await;
        assert_eq!(group1, group2);
    }

    #[tokio::test]
    async fn test_traffic_split() {
        let config = ABTestConfig {
            traffic_split: 0.3, // 30% treatment, 70% control
            ..Default::default()
        };
        let router = ABTestRouter::new(config);

        let mut treatment_count = 0;
        let mut _control_count = 0;
        let total_users = 10000;

        for i in 0..total_users {
            let user_id = format!("user{}", i);
            let group = router.get_or_assign_group(&user_id).await;
            match group {
                ABGroup::Treatment => treatment_count += 1,
                ABGroup::Control => _control_count += 1,
            }
        }

        let treatment_pct = treatment_count as f64 / total_users as f64;
        // Should be close to 30% (within 2%)
        assert!(
            (treatment_pct - 0.3).abs() < 0.02,
            "Treatment percentage {} not close to 30%",
            treatment_pct
        );
    }

    #[tokio::test]
    async fn test_welch_t_test_significant_difference() {
        let config = ABTestConfig::default();
        let tracker = ABMetricsTracker::new(config);

        // Sample 1: mean = 0, std = 1
        let sample1: Vec<f64> = (0..1000).map(|_| rand::random::<f64>() - 0.5).collect();

        // Sample 2: mean = 0.3, std = 1 (shifted distribution)
        let sample2: Vec<f64> = (0..1000).map(|_| rand::random::<f64>() - 0.2).collect();

        let result = tracker.welch_t_test(&sample1, &sample2).unwrap();

        // Should detect significant difference
        assert!(
            result.is_significant,
            "Should detect significant difference"
        );
        assert!(result.p_value < 0.05, "P-value should be < 0.05");
    }

    #[tokio::test]
    async fn test_proportion_z_test() {
        let config = ABTestConfig::default();
        let tracker = ABMetricsTracker::new(config);

        // Control: 52% win rate
        let result = tracker.proportion_z_test(520, 1000, 580, 1000).unwrap();

        // Should detect significant difference (52% vs 58%)
        assert!(
            result.is_significant,
            "Should detect significant win rate difference"
        );
    }

    #[tokio::test]
    async fn test_min_sample_size_calculation() {
        // Detect 10% Sharpe improvement with 80% power
        let effect_size = 0.2; // Small to medium effect
        let power = 0.8;
        let alpha = 0.05;

        let min_n = ABMetricsTracker::calculate_min_sample_size(effect_size, power, alpha);

        // Should be around 393 per group for 0.2 effect size
        assert!(
            min_n > 300 && min_n < 500,
            "Min sample size {} out of expected range",
            min_n
        );
    }

    #[tokio::test]
    async fn test_sharpe_ratio_calculation() {
        let mut metrics = GroupMetrics::default();

        // Add some sample returns
        let returns = vec![0.01, -0.005, 0.02, 0.015, -0.01, 0.03, 0.005, -0.002];
        for &ret in &returns {
            metrics.record_prediction(true, ret * 10000.0, ret, 50);
        }

        let sharpe = metrics.sharpe_ratio();

        // Should be positive with these returns
        assert!(sharpe > 0.0, "Sharpe ratio should be positive");

        // Rough check: mean ~0.0085, std ~0.013, annualized Sharpe ~10
        assert!(
            sharpe > 5.0 && sharpe < 15.0,
            "Sharpe ratio {} out of expected range",
            sharpe
        );
    }

    #[tokio::test]
    async fn test_full_ab_test_workflow() {
        let config = ABTestConfig {
            min_sample_size: 90, // Slightly under to handle split variance
            ..Default::default()
        };
        let router = ABTestRouter::new(config);

        let _rng = rand::thread_rng();

        // Simulate 200 predictions (~100 per group with 50/50 split)
        for i in 0..200 {
            let user_id = format!("user{}", i);
            let group = router.get_or_assign_group(&user_id).await;

            // Treatment has better performance (deterministic with seeded values)
            let (correct, pnl, return_pct) = match group {
                ABGroup::Control => {
                    let correct = (i % 2) == 0; // 50% win rate
                    let return_pct = 0.001; // Low return
                    let pnl = return_pct * 10000.0;
                    (correct, pnl, return_pct)
                },
                ABGroup::Treatment => {
                    let correct = (i % 3) != 0; // 66% win rate (better)
                    let return_pct = 0.003; // Higher return
                    let pnl = return_pct * 10000.0;
                    (correct, pnl, return_pct)
                },
            };

            router
                .record_outcome(group, correct, pnl, return_pct, 50)
                .await;
        }

        // Get results
        let results = router.get_results().await.unwrap();

        assert!(results.control_group.predictions >= 90);
        assert!(results.treatment_group.predictions >= 90);

        // Treatment should have better metrics (deterministic)
        assert!(
            results.treatment_group.win_rate() >= results.control_group.win_rate(),
            "Treatment win rate {} should be >= control {}",
            results.treatment_group.win_rate(),
            results.control_group.win_rate()
        );
    }
}