//! A/B Testing Framework Demonstration //! //! This example demonstrates the complete A/B testing workflow for comparing //! an ensemble model against a single-model baseline. //! //! Usage: //! ```bash //! cargo run -p ml --example ab_test_demonstration --release //! ``` use ml::ensemble::{ ABTestConfig, ABTestRouter, ABGroup, Recommendation, }; use rand::Rng; #[tokio::main] async fn main() -> Result<(), Box> { println!("=== A/B Testing Framework Demonstration ===\n"); // Step 1: Configure A/B test println!("Step 1: Configure A/B Test"); let config = ABTestConfig { test_id: "ensemble_vs_dqn_demo".to_string(), control_model: "DQN-epoch30".to_string(), treatment_model: "6-Model-Ensemble".to_string(), traffic_split: 0.5, // 50/50 split min_sample_size: 1000, // Minimum 1000 predictions per group significance_level: 0.05, // 95% confidence max_duration_hours: 168, // 1 week start_time: chrono::Utc::now().timestamp(), }; println!(" Test ID: {}", config.test_id); println!(" Control: {}", config.control_model); println!(" Treatment: {}", config.treatment_model); println!(" Traffic Split: {}% treatment", config.traffic_split * 100.0); println!(" Min Sample Size: {} per group", config.min_sample_size); println!(); // Step 2: Initialize A/B router println!("Step 2: Initialize A/B Test Router"); let router = ABTestRouter::new(config); println!(" ✓ Router initialized with stratified randomization\n"); // Step 3: Simulate trading predictions println!("Step 3: Simulate Trading Predictions (2000 predictions)"); let mut rng = rand::thread_rng(); for i in 0..2000 { let user_id = format!("trader_{}", i); let group = router.get_or_assign_group(&user_id).await; // Simulate model predictions // Treatment (Ensemble) has 10% better Sharpe ratio let (correct, pnl, return_pct, latency_us) = match group { ABGroup::Control => { // Control: DQN only // Win rate: 53%, Sharpe: ~1.5 let correct = rng.gen::() < 0.53; let return_pct = rng.gen::() * 0.04 - 0.019; // Mean ~0.1% let pnl = return_pct * 10000.0; (correct, pnl, return_pct, 45) } ABGroup::Treatment => { // Treatment: 6-model ensemble // Win rate: 58% (5% better), Sharpe: ~1.65 (10% better) let correct = rng.gen::() < 0.58; let return_pct = rng.gen::() * 0.04 - 0.017; // Mean ~0.15% let pnl = return_pct * 10000.0; (correct, pnl, return_pct, 48) } }; router.record_outcome(group, correct, pnl, return_pct, latency_us).await; // Progress updates if (i + 1) % 500 == 0 { println!(" Progress: {} predictions recorded", i + 1); } } println!(" ✓ Simulation complete\n"); // Step 4: Compute statistical significance println!("Step 4: Compute Statistical Significance"); let results = router.get_results().await?; println!("\n--- Control Group (DQN) ---"); println!(" Predictions: {}", results.control_group.predictions); println!(" Win Rate: {:.2}%", results.control_group.win_rate() * 100.0); println!(" Sharpe Ratio: {:.3}", results.control_group.sharpe_ratio()); println!(" Total P&L: ${:.2}", results.control_group.total_pnl); println!(" Avg Latency: {:.1}μs", results.control_group.avg_latency_us); println!("\n--- Treatment Group (Ensemble) ---"); println!(" Predictions: {}", results.treatment_group.predictions); println!(" Win Rate: {:.2}% ({:+.2}%)", results.treatment_group.win_rate() * 100.0, results.win_rate_diff * 100.0); println!(" Sharpe Ratio: {:.3} ({:+.3})", results.treatment_group.sharpe_ratio(), results.sharpe_diff); println!(" Total P&L: ${:.2} ({:+.2})", results.treatment_group.total_pnl, results.pnl_diff); println!(" Avg Latency: {:.1}μs", results.treatment_group.avg_latency_us); // Step 5: Statistical Test Results println!("\n--- Statistical Test Results ---"); println!(" Sharpe Ratio Difference: {:+.3}", results.sharpe_diff); println!(" Test Statistic: {:.3}", results.sharpe_test.test_statistic); println!(" P-value: {:.6}", results.sharpe_test.p_value); println!(" Significant: {}", if results.sharpe_test.is_significant { "YES ✓" } else { "NO ✗" }); println!(" 95% CI: [{:.3}, {:.3}]", results.sharpe_test.confidence_interval.0, results.sharpe_test.confidence_interval.1); println!("\n Win Rate Difference: {:+.2}%", results.win_rate_diff * 100.0); println!(" Test Statistic: {:.3}", results.win_rate_test.test_statistic); println!(" P-value: {:.6}", results.win_rate_test.p_value); println!(" Significant: {}", if results.win_rate_test.is_significant { "YES ✓" } else { "NO ✗" }); println!("\n P&L Difference: ${:.2}", results.pnl_diff); println!(" Test Statistic: {:.3}", results.pnl_test.test_statistic); println!(" P-value: {:.6}", results.pnl_test.p_value); println!(" Significant: {}", if results.pnl_test.is_significant { "YES ✓" } else { "NO ✗" }); // Step 6: Recommendation println!("\n--- RECOMMENDATION ---"); match &results.recommendation { Recommendation::RolloutTreatment(msg) => { println!(" ✓ ROLL OUT ENSEMBLE TO 100%"); println!(" {}", msg); } Recommendation::RevertToControl(msg) => { println!(" ✗ REVERT TO CONTROL"); println!(" {}", msg); } Recommendation::Neutral(msg) => { println!(" → NO MEANINGFUL DIFFERENCE"); println!(" {}", msg); } Recommendation::Inconclusive(msg) => { println!(" ⚠ INCONCLUSIVE - CONTINUE TESTING"); println!(" {}", msg); } } // Step 7: Power Analysis println!("\n--- Power Analysis ---"); let effect_size = 0.2; // Detect 20% Sharpe difference let min_n = ml::ensemble::ABMetricsTracker::calculate_min_sample_size( effect_size, 0.8, // 80% power 0.05, // 5% alpha ); println!(" To detect 20% Sharpe improvement with 80% power:"); println!(" Minimum sample size per group: {} predictions", min_n); println!(" Current sample size: {} (control), {} (treatment)", results.control_group.predictions, results.treatment_group.predictions); if results.control_group.predictions >= min_n as u64 && results.treatment_group.predictions >= min_n as u64 { println!(" ✓ Sufficient sample size achieved"); } else { println!(" ⚠ Sample size below threshold, continue testing"); } println!("\n=== A/B Test Demonstration Complete ==="); Ok(()) }