foxhunt/backtesting/src/validation.rs

//! Production Validation and Model Comparison Utilities
//!
//! Provides validation logic for determining if a model is production-ready
//! and comparison functions for evaluating model improvements.

use crate::strategy_tester::StrategyResult;
use rust_decimal::Decimal;
use rust_decimal_macros::dec;
use serde::{Deserialize, Serialize};

/// Production readiness thresholds
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ProductionCriteria {
    /// Minimum total return (e.g., 0.0 for profitable)
    pub min_total_return: Decimal,
    /// Minimum Sharpe ratio (e.g., 1.5)
    pub min_sharpe_ratio: Decimal,
    /// Maximum drawdown (e.g., 0.20 for 20%)
    pub max_drawdown: Decimal,
    /// Minimum win rate (e.g., 0.45 for 45%)
    pub min_win_rate: Decimal,
    /// Minimum number of trades (e.g., 10)
    pub min_trades: u64,
}

impl Default for ProductionCriteria {
    fn default() -> Self {
        Self {
            min_total_return: Decimal::ZERO, // Profitable
            min_sharpe_ratio: dec!(1.5),     // Good risk-adjusted returns
            max_drawdown: dec!(0.20),        // 20% max drawdown
            min_win_rate: dec!(0.45),        // 45% win rate
            min_trades: 10,                  // Sufficient sample size
        }
    }
}

impl ProductionCriteria {
    /// Conservative criteria for production deployment
    pub fn conservative() -> Self {
        Self {
            min_total_return: dec!(0.05), // 5% minimum return
            min_sharpe_ratio: dec!(2.0),  // High Sharpe
            max_drawdown: dec!(0.15),     // 15% max drawdown
            min_win_rate: dec!(0.50),     // 50% win rate
            min_trades: 50,               // More trades for confidence
        }
    }

    /// Aggressive criteria for high-risk strategies
    pub fn aggressive() -> Self {
        Self {
            min_total_return: Decimal::ZERO, // Just profitable
            min_sharpe_ratio: dec!(1.0),     // Lower Sharpe acceptable
            max_drawdown: dec!(0.30),        // 30% max drawdown
            min_win_rate: dec!(0.40),        // 40% win rate
            min_trades: 5,                   // Fewer trades needed
        }
    }

    /// Check if a strategy result meets these criteria
    pub fn is_production_ready(&self, result: &StrategyResult) -> bool {
        result.total_return > self.min_total_return
            && result.sharpe_ratio > self.min_sharpe_ratio
            && result.max_drawdown < self.max_drawdown
            && result.win_rate > self.min_win_rate
            && result.total_trades >= self.min_trades
    }

    /// Generate detailed validation report
    pub fn validate(&self, result: &StrategyResult) -> ValidationReport {
        let mut passed_checks = Vec::new();
        let mut failed_checks = Vec::new();

        // Check each criterion
        if result.total_return > self.min_total_return {
            passed_checks.push(format!(
                "Total return: {:.2}% > {:.2}%",
                result.total_return * dec!(100),
                self.min_total_return * dec!(100)
            ));
        } else {
            failed_checks.push(format!(
                "Total return: {:.2}% <= {:.2}% (FAIL)",
                result.total_return * dec!(100),
                self.min_total_return * dec!(100)
            ));
        }

        if result.sharpe_ratio > self.min_sharpe_ratio {
            passed_checks.push(format!(
                "Sharpe ratio: {:.2} > {:.2}",
                result.sharpe_ratio, self.min_sharpe_ratio
            ));
        } else {
            failed_checks.push(format!(
                "Sharpe ratio: {:.2} <= {:.2} (FAIL)",
                result.sharpe_ratio, self.min_sharpe_ratio
            ));
        }

        if result.max_drawdown < self.max_drawdown {
            passed_checks.push(format!(
                "Max drawdown: {:.2}% < {:.2}%",
                result.max_drawdown * dec!(100),
                self.max_drawdown * dec!(100)
            ));
        } else {
            failed_checks.push(format!(
                "Max drawdown: {:.2}% >= {:.2}% (FAIL)",
                result.max_drawdown * dec!(100),
                self.max_drawdown * dec!(100)
            ));
        }

        if result.win_rate > self.min_win_rate {
            passed_checks.push(format!(
                "Win rate: {:.2}% > {:.2}%",
                result.win_rate * dec!(100),
                self.min_win_rate * dec!(100)
            ));
        } else {
            failed_checks.push(format!(
                "Win rate: {:.2}% <= {:.2}% (FAIL)",
                result.win_rate * dec!(100),
                self.min_win_rate * dec!(100)
            ));
        }

        if result.total_trades >= self.min_trades {
            passed_checks.push(format!(
                "Total trades: {} >= {}",
                result.total_trades, self.min_trades
            ));
        } else {
            failed_checks.push(format!(
                "Total trades: {} < {} (FAIL)",
                result.total_trades, self.min_trades
            ));
        }

        let production_ready = failed_checks.is_empty();

        ValidationReport {
            production_ready,
            passed_checks,
            failed_checks,
            strategy_name: result.strategy_name.clone(),
            total_return: result.total_return,
            sharpe_ratio: result.sharpe_ratio,
            max_drawdown: result.max_drawdown,
            win_rate: result.win_rate,
            total_trades: result.total_trades,
        }
    }
}

/// Validation report for a strategy
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ValidationReport {
    /// Whether the strategy is production-ready
    pub production_ready: bool,
    /// List of checks that passed
    pub passed_checks: Vec<String>,
    /// List of checks that failed
    pub failed_checks: Vec<String>,
    /// Strategy name
    pub strategy_name: String,
    /// Key metrics
    pub total_return: Decimal,
    pub sharpe_ratio: Decimal,
    pub max_drawdown: Decimal,
    pub win_rate: Decimal,
    pub total_trades: u64,
}

impl ValidationReport {
    /// Print formatted validation report
    pub fn print_report(&self) {
        println!("\n=== VALIDATION REPORT ===");
        println!("Strategy: {}", self.strategy_name);
        println!("Status: {}", if self.production_ready { "✅ PRODUCTION READY" } else { "❌ NOT READY" });
        println!("\nKey Metrics:");
        println!("  Total Return:  {:.2}%", self.total_return * dec!(100));
        println!("  Sharpe Ratio:  {:.2}", self.sharpe_ratio);
        println!("  Max Drawdown:  {:.2}%", self.max_drawdown * dec!(100));
        println!("  Win Rate:      {:.2}%", self.win_rate * dec!(100));
        println!("  Total Trades:  {}", self.total_trades);

        if !self.passed_checks.is_empty() {
            println!("\n✅ Passed Checks ({}):", self.passed_checks.len());
            for check in &self.passed_checks {
                println!("  • {}", check);
            }
        }

        if !self.failed_checks.is_empty() {
            println!("\n❌ Failed Checks ({}):", self.failed_checks.len());
            for check in &self.failed_checks {
                println!("  • {}", check);
            }
        }

        println!("========================\n");
    }
}

/// Model comparison result
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ModelComparison {
    /// Sharpe ratio improvement (percentage)
    pub sharpe_improvement: Decimal,
    /// Total return improvement (absolute)
    pub return_improvement: Decimal,
    /// Drawdown improvement (absolute, positive = better)
    pub drawdown_improvement: Decimal,
    /// Win rate improvement (absolute)
    pub win_rate_improvement: Decimal,
    /// Whether new model is better overall
    pub is_better: bool,
    /// Whether new model shows regression (>10% worse returns)
    pub is_regression: bool,
    /// Recommendation text
    pub recommendation: String,
    /// Baseline model name
    pub baseline_name: String,
    /// New model name
    pub new_model_name: String,
}

/// Compare two models
pub fn compare_models(baseline: &StrategyResult, new_model: &StrategyResult) -> ModelComparison {
    let sharpe_improvement = if baseline.sharpe_ratio > Decimal::ZERO {
        (new_model.sharpe_ratio - baseline.sharpe_ratio) / baseline.sharpe_ratio
    } else {
        Decimal::ZERO
    };

    let return_improvement = new_model.total_return - baseline.total_return;
    let drawdown_improvement = baseline.max_drawdown - new_model.max_drawdown; // Positive = better
    let win_rate_improvement = new_model.win_rate - baseline.win_rate;

    let is_better = new_model.sharpe_ratio > baseline.sharpe_ratio
        && new_model.total_return > baseline.total_return
        && new_model.max_drawdown < baseline.max_drawdown;

    let is_regression = new_model.total_return < baseline.total_return * dec!(0.9);

    let recommendation = if is_regression {
        "REJECT - Regression detected (returns dropped >10%)".to_string()
    } else if is_better {
        "APPROVE - Improvement confirmed across all metrics".to_string()
    } else if return_improvement > Decimal::ZERO && sharpe_improvement > Decimal::ZERO {
        "APPROVE - Returns and risk-adjusted performance improved".to_string()
    } else if return_improvement < Decimal::ZERO {
        "REVIEW - Lower returns despite other improvements".to_string()
    } else {
        "REVIEW - Mixed results, manual review recommended".to_string()
    };

    ModelComparison {
        sharpe_improvement,
        return_improvement,
        drawdown_improvement,
        win_rate_improvement,
        is_better,
        is_regression,
        recommendation,
        baseline_name: baseline.strategy_name.clone(),
        new_model_name: new_model.strategy_name.clone(),
    }
}

impl ModelComparison {
    /// Print formatted comparison report
    pub fn print_report(&self) {
        println!("\n=== MODEL COMPARISON REPORT ===");
        println!("Baseline:  {}", self.baseline_name);
        println!("New Model: {}", self.new_model_name);
        println!("\nImprovements:");
        println!("  Return:    {:+.2}%", self.return_improvement * dec!(100));
        println!("  Sharpe:    {:+.2}%", self.sharpe_improvement * dec!(100));
        println!("  Drawdown:  {:+.2}% (positive = better)", self.drawdown_improvement * dec!(100));
        println!("  Win Rate:  {:+.2}%", self.win_rate_improvement * dec!(100));
        println!("\nStatus:");
        if self.is_regression {
            println!("  🔴 REGRESSION DETECTED");
        } else if self.is_better {
            println!("  ✅ OVERALL IMPROVEMENT");
        } else {
            println!("  ⚠️  MIXED RESULTS");
        }
        println!("\nRecommendation:");
        println!("  {}", self.recommendation);
        println!("==============================\n");
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_production_criteria_default() {
        let criteria = ProductionCriteria::default();
        
        let passing = StrategyResult {
            strategy_name: "test".to_string(),
            total_return: dec!(0.08),
            annualized_return: dec!(0.32),
            max_drawdown: dec!(0.15),
            sharpe_ratio: dec!(2.13),
            total_trades: 100,
            win_rate: dec!(0.55),
            avg_trade_return: dec!(0.0008),
            final_value: dec!(108000),
            trades: vec![],
            performance_timeline: vec![],
        };

        assert!(criteria.is_production_ready(&passing));

        let report = criteria.validate(&passing);
        assert!(report.production_ready);
        assert_eq!(report.failed_checks.len(), 0);
        assert_eq!(report.passed_checks.len(), 5);
    }

    #[test]
    fn test_comparison_approve() {
        let baseline = StrategyResult {
            strategy_name: "baseline".to_string(),
            total_return: dec!(0.08),
            annualized_return: dec!(0.32),
            max_drawdown: dec!(0.15),
            sharpe_ratio: dec!(2.13),
            total_trades: 100,
            win_rate: dec!(0.55),
            avg_trade_return: dec!(0.0008),
            final_value: dec!(108000),
            trades: vec![],
            performance_timeline: vec![],
        };

        let improved = StrategyResult {
            strategy_name: "improved".to_string(),
            total_return: dec!(0.12),
            annualized_return: dec!(0.48),
            max_drawdown: dec!(0.10),
            sharpe_ratio: dec!(4.8),
            total_trades: 110,
            win_rate: dec!(0.62),
            avg_trade_return: dec!(0.00109),
            final_value: dec!(112000),
            trades: vec![],
            performance_timeline: vec![],
        };

        let comparison = compare_models(&baseline, &improved);
        assert!(comparison.is_better);
        assert!(!comparison.is_regression);
        assert!(comparison.recommendation.contains("APPROVE"));
    }

    #[test]
    fn test_comparison_reject_regression() {
        let baseline = StrategyResult {
            strategy_name: "baseline".to_string(),
            total_return: dec!(0.10),
            annualized_return: dec!(0.40),
            max_drawdown: dec!(0.15),
            sharpe_ratio: dec!(2.67),
            total_trades: 100,
            win_rate: dec!(0.58),
            avg_trade_return: dec!(0.001),
            final_value: dec!(110000),
            trades: vec![],
            performance_timeline: vec![],
        };

        let regression = StrategyResult {
            strategy_name: "regression".to_string(),
            total_return: dec!(0.03), // 70% worse
            annualized_return: dec!(0.12),
            max_drawdown: dec!(0.18),
            sharpe_ratio: dec!(0.67),
            total_trades: 90,
            win_rate: dec!(0.48),
            avg_trade_return: dec!(0.000333),
            final_value: dec!(103000),
            trades: vec![],
            performance_timeline: vec![],
        };

        let comparison = compare_models(&baseline, &regression);
        assert!(!comparison.is_better);
        assert!(comparison.is_regression);
        assert!(comparison.recommendation.contains("REJECT"));
    }
}