foxhunt/crates/ml/examples/ab_test_demonstration.rs

//! A/B Testing Framework Demonstration
//!
//! This example demonstrates the complete A/B testing workflow for comparing
//! an ensemble model against a single-model baseline.
//!
//! Usage:
//! ```bash
//! cargo run -p ml --example ab_test_demonstration --release
//! ```

use ml::ensemble::{ABGroup, ABTestConfig, ABTestRouter, Recommendation};
use rand::Rng;

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    println!("=== A/B Testing Framework Demonstration ===\n");

    // Step 1: Configure A/B test
    println!("Step 1: Configure A/B Test");
    let config = ABTestConfig {
        test_id: "ensemble_vs_dqn_demo".to_string(),
        control_model: "DQN-epoch30".to_string(),
        treatment_model: "6-Model-Ensemble".to_string(),
        traffic_split: 0.5,       // 50/50 split
        min_sample_size: 1000,    // Minimum 1000 predictions per group
        significance_level: 0.05, // 95% confidence
        max_duration_hours: 168,  // 1 week
        start_time: chrono::Utc::now().timestamp(),
    };

    println!("  Test ID: {}", config.test_id);
    println!("  Control: {}", config.control_model);
    println!("  Treatment: {}", config.treatment_model);
    println!(
        "  Traffic Split: {}% treatment",
        config.traffic_split * 100.0
    );
    println!("  Min Sample Size: {} per group", config.min_sample_size);
    println!();

    // Step 2: Initialize A/B router
    println!("Step 2: Initialize A/B Test Router");
    let router = ABTestRouter::new(config);
    println!("  ✓ Router initialized with stratified randomization\n");

    // Step 3: Simulate trading predictions
    println!("Step 3: Simulate Trading Predictions (2000 predictions)");
    let mut rng = rand::thread_rng();

    for i in 0..2000 {
        let user_id = format!("trader_{}", i);
        let group = router.get_or_assign_group(&user_id).await;

        // Simulate model predictions
        // Treatment (Ensemble) has 10% better Sharpe ratio
        let (correct, pnl, return_pct, latency_us) = match group {
            ABGroup::Control => {
                // Control: DQN only
                // Win rate: 53%, Sharpe: ~1.5
                let correct = rng.gen::<f64>() < 0.53;
                let return_pct = rng.gen::<f64>() * 0.04 - 0.019; // Mean ~0.1%
                let pnl = return_pct * 10000.0;
                (correct, pnl, return_pct, 45)
            },
            ABGroup::Treatment => {
                // Treatment: 6-model ensemble
                // Win rate: 58% (5% better), Sharpe: ~1.65 (10% better)
                let correct = rng.gen::<f64>() < 0.58;
                let return_pct = rng.gen::<f64>() * 0.04 - 0.017; // Mean ~0.15%
                let pnl = return_pct * 10000.0;
                (correct, pnl, return_pct, 48)
            },
        };

        router
            .record_outcome(group, correct, pnl, return_pct, latency_us)
            .await;

        // Progress updates
        if (i + 1) % 500 == 0 {
            println!("  Progress: {} predictions recorded", i + 1);
        }
    }
    println!("  ✓ Simulation complete\n");

    // Step 4: Compute statistical significance
    println!("Step 4: Compute Statistical Significance");
    let results = router.get_results().await?;

    println!("\n--- Control Group (DQN) ---");
    println!("  Predictions: {}", results.control_group.predictions);
    println!(
        "  Win Rate: {:.2}%",
        results.control_group.win_rate() * 100.0
    );
    println!(
        "  Sharpe Ratio: {:.3}",
        results.control_group.sharpe_ratio()
    );
    println!("  Total P&L: ${:.2}", results.control_group.total_pnl);
    println!(
        "  Avg Latency: {:.1}μs",
        results.control_group.avg_latency_us
    );

    println!("\n--- Treatment Group (Ensemble) ---");
    println!("  Predictions: {}", results.treatment_group.predictions);
    println!(
        "  Win Rate: {:.2}% ({:+.2}%)",
        results.treatment_group.win_rate() * 100.0,
        results.win_rate_diff * 100.0
    );
    println!(
        "  Sharpe Ratio: {:.3} ({:+.3})",
        results.treatment_group.sharpe_ratio(),
        results.sharpe_diff
    );
    println!(
        "  Total P&L: ${:.2} ({:+.2})",
        results.treatment_group.total_pnl, results.pnl_diff
    );
    println!(
        "  Avg Latency: {:.1}μs",
        results.treatment_group.avg_latency_us
    );

    // Step 5: Statistical Test Results
    println!("\n--- Statistical Test Results ---");
    println!("  Sharpe Ratio Difference: {:+.3}", results.sharpe_diff);
    println!(
        "    Test Statistic: {:.3}",
        results.sharpe_test.test_statistic
    );
    println!("    P-value: {:.6}", results.sharpe_test.p_value);
    println!(
        "    Significant: {}",
        if results.sharpe_test.is_significant {
            "YES ✓"
        } else {
            "NO ✗"
        }
    );
    println!(
        "    95% CI: [{:.3}, {:.3}]",
        results.sharpe_test.confidence_interval.0, results.sharpe_test.confidence_interval.1
    );

    println!(
        "\n  Win Rate Difference: {:+.2}%",
        results.win_rate_diff * 100.0
    );
    println!(
        "    Test Statistic: {:.3}",
        results.win_rate_test.test_statistic
    );
    println!("    P-value: {:.6}", results.win_rate_test.p_value);
    println!(
        "    Significant: {}",
        if results.win_rate_test.is_significant {
            "YES ✓"
        } else {
            "NO ✗"
        }
    );

    println!("\n  P&L Difference: ${:.2}", results.pnl_diff);
    println!("    Test Statistic: {:.3}", results.pnl_test.test_statistic);
    println!("    P-value: {:.6}", results.pnl_test.p_value);
    println!(
        "    Significant: {}",
        if results.pnl_test.is_significant {
            "YES ✓"
        } else {
            "NO ✗"
        }
    );

    // Step 6: Recommendation
    println!("\n--- RECOMMENDATION ---");
    match &results.recommendation {
        Recommendation::RolloutTreatment(msg) => {
            println!("  ✓ ROLL OUT ENSEMBLE TO 100%");
            println!("  {}", msg);
        },
        Recommendation::RevertToControl(msg) => {
            println!("  ✗ REVERT TO CONTROL");
            println!("  {}", msg);
        },
        Recommendation::Neutral(msg) => {
            println!("  → NO MEANINGFUL DIFFERENCE");
            println!("  {}", msg);
        },
        Recommendation::Inconclusive(msg) => {
            println!("  ⚠ INCONCLUSIVE - CONTINUE TESTING");
            println!("  {}", msg);
        },
    }

    // Step 7: Power Analysis
    println!("\n--- Power Analysis ---");
    let effect_size = 0.2; // Detect 20% Sharpe difference
    let min_n = ml::ensemble::ABMetricsTracker::calculate_min_sample_size(
        effect_size,
        0.8,  // 80% power
        0.05, // 5% alpha
    );
    println!("  To detect 20% Sharpe improvement with 80% power:");
    println!("  Minimum sample size per group: {} predictions", min_n);
    println!(
        "  Current sample size: {} (control), {} (treatment)",
        results.control_group.predictions, results.treatment_group.predictions
    );

    if results.control_group.predictions >= min_n as u64
        && results.treatment_group.predictions >= min_n as u64
    {
        println!("  ✓ Sufficient sample size achieved");
    } else {
        println!("  ⚠ Sample size below threshold, continue testing");
    }

    println!("\n=== A/B Test Demonstration Complete ===");

    Ok(())
}