//! Production Validation and Model Comparison Utilities //! //! Provides validation logic for determining if a model is production-ready //! and comparison functions for evaluating model improvements. use crate::strategy_tester::StrategyResult; use rust_decimal::Decimal; use rust_decimal_macros::dec; use serde::{Deserialize, Serialize}; /// Production readiness thresholds #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ProductionCriteria { /// Minimum total return (e.g., 0.0 for profitable) pub min_total_return: Decimal, /// Minimum Sharpe ratio (e.g., 1.5) pub min_sharpe_ratio: Decimal, /// Maximum drawdown (e.g., 0.20 for 20%) pub max_drawdown: Decimal, /// Minimum win rate (e.g., 0.45 for 45%) pub min_win_rate: Decimal, /// Minimum number of trades (e.g., 10) pub min_trades: u64, } impl Default for ProductionCriteria { fn default() -> Self { Self { min_total_return: Decimal::ZERO, // Profitable min_sharpe_ratio: dec!(1.5), // Good risk-adjusted returns max_drawdown: dec!(0.20), // 20% max drawdown min_win_rate: dec!(0.45), // 45% win rate min_trades: 10, // Sufficient sample size } } } impl ProductionCriteria { /// Conservative criteria for production deployment pub fn conservative() -> Self { Self { min_total_return: dec!(0.05), // 5% minimum return min_sharpe_ratio: dec!(2.0), // High Sharpe max_drawdown: dec!(0.15), // 15% max drawdown min_win_rate: dec!(0.50), // 50% win rate min_trades: 50, // More trades for confidence } } /// Aggressive criteria for high-risk strategies pub fn aggressive() -> Self { Self { min_total_return: Decimal::ZERO, // Just profitable min_sharpe_ratio: dec!(1.0), // Lower Sharpe acceptable max_drawdown: dec!(0.30), // 30% max drawdown min_win_rate: dec!(0.40), // 40% win rate min_trades: 5, // Fewer trades needed } } /// Check if a strategy result meets these criteria pub fn is_production_ready(&self, result: &StrategyResult) -> bool { result.total_return > self.min_total_return && result.sharpe_ratio > self.min_sharpe_ratio && result.max_drawdown < self.max_drawdown && result.win_rate > self.min_win_rate && result.total_trades >= self.min_trades } /// Generate detailed validation report pub fn validate(&self, result: &StrategyResult) -> ValidationReport { let mut passed_checks = Vec::new(); let mut failed_checks = Vec::new(); // Check each criterion if result.total_return > self.min_total_return { passed_checks.push(format!( "Total return: {:.2}% > {:.2}%", result.total_return * dec!(100), self.min_total_return * dec!(100) )); } else { failed_checks.push(format!( "Total return: {:.2}% <= {:.2}% (FAIL)", result.total_return * dec!(100), self.min_total_return * dec!(100) )); } if result.sharpe_ratio > self.min_sharpe_ratio { passed_checks.push(format!( "Sharpe ratio: {:.2} > {:.2}", result.sharpe_ratio, self.min_sharpe_ratio )); } else { failed_checks.push(format!( "Sharpe ratio: {:.2} <= {:.2} (FAIL)", result.sharpe_ratio, self.min_sharpe_ratio )); } if result.max_drawdown < self.max_drawdown { passed_checks.push(format!( "Max drawdown: {:.2}% < {:.2}%", result.max_drawdown * dec!(100), self.max_drawdown * dec!(100) )); } else { failed_checks.push(format!( "Max drawdown: {:.2}% >= {:.2}% (FAIL)", result.max_drawdown * dec!(100), self.max_drawdown * dec!(100) )); } if result.win_rate > self.min_win_rate { passed_checks.push(format!( "Win rate: {:.2}% > {:.2}%", result.win_rate * dec!(100), self.min_win_rate * dec!(100) )); } else { failed_checks.push(format!( "Win rate: {:.2}% <= {:.2}% (FAIL)", result.win_rate * dec!(100), self.min_win_rate * dec!(100) )); } if result.total_trades >= self.min_trades { passed_checks.push(format!( "Total trades: {} >= {}", result.total_trades, self.min_trades )); } else { failed_checks.push(format!( "Total trades: {} < {} (FAIL)", result.total_trades, self.min_trades )); } let production_ready = failed_checks.is_empty(); ValidationReport { production_ready, passed_checks, failed_checks, strategy_name: result.strategy_name.clone(), total_return: result.total_return, sharpe_ratio: result.sharpe_ratio, max_drawdown: result.max_drawdown, win_rate: result.win_rate, total_trades: result.total_trades, } } } /// Validation report for a strategy #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ValidationReport { /// Whether the strategy is production-ready pub production_ready: bool, /// List of checks that passed pub passed_checks: Vec, /// List of checks that failed pub failed_checks: Vec, /// Strategy name pub strategy_name: String, /// Key metrics pub total_return: Decimal, pub sharpe_ratio: Decimal, pub max_drawdown: Decimal, pub win_rate: Decimal, pub total_trades: u64, } impl ValidationReport { /// Print formatted validation report pub fn print_report(&self) { println!("\n=== VALIDATION REPORT ==="); println!("Strategy: {}", self.strategy_name); println!("Status: {}", if self.production_ready { "✅ PRODUCTION READY" } else { "❌ NOT READY" }); println!("\nKey Metrics:"); println!(" Total Return: {:.2}%", self.total_return * dec!(100)); println!(" Sharpe Ratio: {:.2}", self.sharpe_ratio); println!(" Max Drawdown: {:.2}%", self.max_drawdown * dec!(100)); println!(" Win Rate: {:.2}%", self.win_rate * dec!(100)); println!(" Total Trades: {}", self.total_trades); if !self.passed_checks.is_empty() { println!("\n✅ Passed Checks ({}):", self.passed_checks.len()); for check in &self.passed_checks { println!(" • {}", check); } } if !self.failed_checks.is_empty() { println!("\n❌ Failed Checks ({}):", self.failed_checks.len()); for check in &self.failed_checks { println!(" • {}", check); } } println!("========================\n"); } } /// Model comparison result #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ModelComparison { /// Sharpe ratio improvement (percentage) pub sharpe_improvement: Decimal, /// Total return improvement (absolute) pub return_improvement: Decimal, /// Drawdown improvement (absolute, positive = better) pub drawdown_improvement: Decimal, /// Win rate improvement (absolute) pub win_rate_improvement: Decimal, /// Whether new model is better overall pub is_better: bool, /// Whether new model shows regression (>10% worse returns) pub is_regression: bool, /// Recommendation text pub recommendation: String, /// Baseline model name pub baseline_name: String, /// New model name pub new_model_name: String, } /// Compare two models pub fn compare_models(baseline: &StrategyResult, new_model: &StrategyResult) -> ModelComparison { let sharpe_improvement = if baseline.sharpe_ratio > Decimal::ZERO { (new_model.sharpe_ratio - baseline.sharpe_ratio) / baseline.sharpe_ratio } else { Decimal::ZERO }; let return_improvement = new_model.total_return - baseline.total_return; let drawdown_improvement = baseline.max_drawdown - new_model.max_drawdown; // Positive = better let win_rate_improvement = new_model.win_rate - baseline.win_rate; let is_better = new_model.sharpe_ratio > baseline.sharpe_ratio && new_model.total_return > baseline.total_return && new_model.max_drawdown < baseline.max_drawdown; let is_regression = new_model.total_return < baseline.total_return * dec!(0.9); let recommendation = if is_regression { "REJECT - Regression detected (returns dropped >10%)".to_string() } else if is_better { "APPROVE - Improvement confirmed across all metrics".to_string() } else if return_improvement > Decimal::ZERO && sharpe_improvement > Decimal::ZERO { "APPROVE - Returns and risk-adjusted performance improved".to_string() } else if return_improvement < Decimal::ZERO { "REVIEW - Lower returns despite other improvements".to_string() } else { "REVIEW - Mixed results, manual review recommended".to_string() }; ModelComparison { sharpe_improvement, return_improvement, drawdown_improvement, win_rate_improvement, is_better, is_regression, recommendation, baseline_name: baseline.strategy_name.clone(), new_model_name: new_model.strategy_name.clone(), } } impl ModelComparison { /// Print formatted comparison report pub fn print_report(&self) { println!("\n=== MODEL COMPARISON REPORT ==="); println!("Baseline: {}", self.baseline_name); println!("New Model: {}", self.new_model_name); println!("\nImprovements:"); println!(" Return: {:+.2}%", self.return_improvement * dec!(100)); println!(" Sharpe: {:+.2}%", self.sharpe_improvement * dec!(100)); println!(" Drawdown: {:+.2}% (positive = better)", self.drawdown_improvement * dec!(100)); println!(" Win Rate: {:+.2}%", self.win_rate_improvement * dec!(100)); println!("\nStatus:"); if self.is_regression { println!(" 🔴 REGRESSION DETECTED"); } else if self.is_better { println!(" ✅ OVERALL IMPROVEMENT"); } else { println!(" ⚠️ MIXED RESULTS"); } println!("\nRecommendation:"); println!(" {}", self.recommendation); println!("==============================\n"); } } #[cfg(test)] mod tests { use super::*; #[test] fn test_production_criteria_default() { let criteria = ProductionCriteria::default(); let passing = StrategyResult { strategy_name: "test".to_string(), total_return: dec!(0.08), annualized_return: dec!(0.32), max_drawdown: dec!(0.15), sharpe_ratio: dec!(2.13), total_trades: 100, win_rate: dec!(0.55), avg_trade_return: dec!(0.0008), final_value: dec!(108000), trades: vec![], performance_timeline: vec![], }; assert!(criteria.is_production_ready(&passing)); let report = criteria.validate(&passing); assert!(report.production_ready); assert_eq!(report.failed_checks.len(), 0); assert_eq!(report.passed_checks.len(), 5); } #[test] fn test_comparison_approve() { let baseline = StrategyResult { strategy_name: "baseline".to_string(), total_return: dec!(0.08), annualized_return: dec!(0.32), max_drawdown: dec!(0.15), sharpe_ratio: dec!(2.13), total_trades: 100, win_rate: dec!(0.55), avg_trade_return: dec!(0.0008), final_value: dec!(108000), trades: vec![], performance_timeline: vec![], }; let improved = StrategyResult { strategy_name: "improved".to_string(), total_return: dec!(0.12), annualized_return: dec!(0.48), max_drawdown: dec!(0.10), sharpe_ratio: dec!(4.8), total_trades: 110, win_rate: dec!(0.62), avg_trade_return: dec!(0.00109), final_value: dec!(112000), trades: vec![], performance_timeline: vec![], }; let comparison = compare_models(&baseline, &improved); assert!(comparison.is_better); assert!(!comparison.is_regression); assert!(comparison.recommendation.contains("APPROVE")); } #[test] fn test_comparison_reject_regression() { let baseline = StrategyResult { strategy_name: "baseline".to_string(), total_return: dec!(0.10), annualized_return: dec!(0.40), max_drawdown: dec!(0.15), sharpe_ratio: dec!(2.67), total_trades: 100, win_rate: dec!(0.58), avg_trade_return: dec!(0.001), final_value: dec!(110000), trades: vec![], performance_timeline: vec![], }; let regression = StrategyResult { strategy_name: "regression".to_string(), total_return: dec!(0.03), // 70% worse annualized_return: dec!(0.12), max_drawdown: dec!(0.18), sharpe_ratio: dec!(0.67), total_trades: 90, win_rate: dec!(0.48), avg_trade_return: dec!(0.000333), final_value: dec!(103000), trades: vec![], performance_timeline: vec![], }; let comparison = compare_models(&baseline, ®ression); assert!(!comparison.is_better); assert!(comparison.is_regression); assert!(comparison.recommendation.contains("REJECT")); } }