foxhunt/ml/examples/ab_test_demonstration.rs

//! A/B Testing Framework Demonstration
//!
//! This example demonstrates the complete A/B testing workflow for comparing
//! an ensemble model against a single-model baseline.
//!
//! Usage:
//! ```bash
//! cargo run -p ml --example ab_test_demonstration --release
//! ```

use ml::ensemble::{
    ABTestConfig, ABTestRouter, ABGroup, Recommendation,
};
use rand::Rng;

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    println!("=== A/B Testing Framework Demonstration ===\n");

    // Step 1: Configure A/B test
    println!("Step 1: Configure A/B Test");
    let config = ABTestConfig {
        test_id: "ensemble_vs_dqn_demo".to_string(),
        control_model: "DQN-epoch30".to_string(),
        treatment_model: "6-Model-Ensemble".to_string(),
        traffic_split: 0.5, // 50/50 split
        min_sample_size: 1000, // Minimum 1000 predictions per group
        significance_level: 0.05, // 95% confidence
        max_duration_hours: 168, // 1 week
        start_time: chrono::Utc::now().timestamp(),
    };

    println!("  Test ID: {}", config.test_id);
    println!("  Control: {}", config.control_model);
    println!("  Treatment: {}", config.treatment_model);
    println!("  Traffic Split: {}% treatment", config.traffic_split * 100.0);
    println!("  Min Sample Size: {} per group", config.min_sample_size);
    println!();

    // Step 2: Initialize A/B router
    println!("Step 2: Initialize A/B Test Router");
    let router = ABTestRouter::new(config);
    println!("  ✓ Router initialized with stratified randomization\n");

    // Step 3: Simulate trading predictions
    println!("Step 3: Simulate Trading Predictions (2000 predictions)");
    let mut rng = rand::thread_rng();

    for i in 0..2000 {
        let user_id = format!("trader_{}", i);
        let group = router.get_or_assign_group(&user_id).await;

        // Simulate model predictions
        // Treatment (Ensemble) has 10% better Sharpe ratio
        let (correct, pnl, return_pct, latency_us) = match group {
            ABGroup::Control => {
                // Control: DQN only
                // Win rate: 53%, Sharpe: ~1.5
                let correct = rng.gen::<f64>() < 0.53;
                let return_pct = rng.gen::<f64>() * 0.04 - 0.019; // Mean ~0.1%
                let pnl = return_pct * 10000.0;
                (correct, pnl, return_pct, 45)
            }
            ABGroup::Treatment => {
                // Treatment: 6-model ensemble
                // Win rate: 58% (5% better), Sharpe: ~1.65 (10% better)
                let correct = rng.gen::<f64>() < 0.58;
                let return_pct = rng.gen::<f64>() * 0.04 - 0.017; // Mean ~0.15%
                let pnl = return_pct * 10000.0;
                (correct, pnl, return_pct, 48)
            }
        };

        router.record_outcome(group, correct, pnl, return_pct, latency_us).await;

        // Progress updates
        if (i + 1) % 500 == 0 {
            println!("  Progress: {} predictions recorded", i + 1);
        }
    }
    println!("  ✓ Simulation complete\n");

    // Step 4: Compute statistical significance
    println!("Step 4: Compute Statistical Significance");
    let results = router.get_results().await?;

    println!("\n--- Control Group (DQN) ---");
    println!("  Predictions: {}", results.control_group.predictions);
    println!("  Win Rate: {:.2}%", results.control_group.win_rate() * 100.0);
    println!("  Sharpe Ratio: {:.3}", results.control_group.sharpe_ratio());
    println!("  Total P&L: ${:.2}", results.control_group.total_pnl);
    println!("  Avg Latency: {:.1}μs", results.control_group.avg_latency_us);

    println!("\n--- Treatment Group (Ensemble) ---");
    println!("  Predictions: {}", results.treatment_group.predictions);
    println!("  Win Rate: {:.2}% ({:+.2}%)",
             results.treatment_group.win_rate() * 100.0,
             results.win_rate_diff * 100.0);
    println!("  Sharpe Ratio: {:.3} ({:+.3})",
             results.treatment_group.sharpe_ratio(),
             results.sharpe_diff);
    println!("  Total P&L: ${:.2} ({:+.2})",
             results.treatment_group.total_pnl,
             results.pnl_diff);
    println!("  Avg Latency: {:.1}μs", results.treatment_group.avg_latency_us);

    // Step 5: Statistical Test Results
    println!("\n--- Statistical Test Results ---");
    println!("  Sharpe Ratio Difference: {:+.3}", results.sharpe_diff);
    println!("    Test Statistic: {:.3}", results.sharpe_test.test_statistic);
    println!("    P-value: {:.6}", results.sharpe_test.p_value);
    println!("    Significant: {}", if results.sharpe_test.is_significant { "YES ✓" } else { "NO ✗" });
    println!("    95% CI: [{:.3}, {:.3}]",
             results.sharpe_test.confidence_interval.0,
             results.sharpe_test.confidence_interval.1);

    println!("\n  Win Rate Difference: {:+.2}%", results.win_rate_diff * 100.0);
    println!("    Test Statistic: {:.3}", results.win_rate_test.test_statistic);
    println!("    P-value: {:.6}", results.win_rate_test.p_value);
    println!("    Significant: {}", if results.win_rate_test.is_significant { "YES ✓" } else { "NO ✗" });

    println!("\n  P&L Difference: ${:.2}", results.pnl_diff);
    println!("    Test Statistic: {:.3}", results.pnl_test.test_statistic);
    println!("    P-value: {:.6}", results.pnl_test.p_value);
    println!("    Significant: {}", if results.pnl_test.is_significant { "YES ✓" } else { "NO ✗" });

    // Step 6: Recommendation
    println!("\n--- RECOMMENDATION ---");
    match &results.recommendation {
        Recommendation::RolloutTreatment(msg) => {
            println!("  ✓ ROLL OUT ENSEMBLE TO 100%");
            println!("  {}", msg);
        }
        Recommendation::RevertToControl(msg) => {
            println!("  ✗ REVERT TO CONTROL");
            println!("  {}", msg);
        }
        Recommendation::Neutral(msg) => {
            println!("  → NO MEANINGFUL DIFFERENCE");
            println!("  {}", msg);
        }
        Recommendation::Inconclusive(msg) => {
            println!("  ⚠ INCONCLUSIVE - CONTINUE TESTING");
            println!("  {}", msg);
        }
    }

    // Step 7: Power Analysis
    println!("\n--- Power Analysis ---");
    let effect_size = 0.2; // Detect 20% Sharpe difference
    let min_n = ml::ensemble::ABMetricsTracker::calculate_min_sample_size(
        effect_size,
        0.8, // 80% power
        0.05, // 5% alpha
    );
    println!("  To detect 20% Sharpe improvement with 80% power:");
    println!("  Minimum sample size per group: {} predictions", min_n);
    println!("  Current sample size: {} (control), {} (treatment)",
             results.control_group.predictions,
             results.treatment_group.predictions);

    if results.control_group.predictions >= min_n as u64 &&
       results.treatment_group.predictions >= min_n as u64 {
        println!("  ✓ Sufficient sample size achieved");
    } else {
        println!("  ⚠ Sample size below threshold, continue testing");
    }

    println!("\n=== A/B Test Demonstration Complete ===");

    Ok(())
}