foxhunt/crates/ml/examples/generate_backtest_report.rs

//! Backtesting Report Generator
//!
//! Generates comprehensive markdown reports comparing DQN model performance against baseline.
//! Provides automated deployment recommendations based on production criteria.
//!
//! # Features
//!
//! - **Baseline Comparison**: Compare new model vs Trial #35 (or custom baseline)
//! - **Production Criteria**: Automated validation of 5 key metrics
//! - **Deployment Recommendation**: APPROVE/REJECT/REVIEW with reasoning
//! - **Markdown Export**: Professional formatted reports for documentation
//!
//! # Usage
//!
//! ```bash
//! # Generate report with default Trial #35 baseline
//! cargo run -p ml --example generate_backtest_report --release
//!
//! # Generate report with custom baseline (from JSON)
//! cargo run -p ml --example generate_backtest_report --release -- \
//!   --new-model "DQN-Wave3-Entropy" \
//!   --baseline-model "DQN-Trial35-Baseline" \
//!   --output-file dqn_comparison_report.md
//!
//! # Example with specific performance metrics (manual entry)
//! cargo run -p ml --example generate_backtest_report --release -- \
//!   --new-model "DQN-Wave4-Test" \
//!   --total-return 18.5 \
//!   --sharpe 2.3 \
//!   --drawdown 12.5 \
//!   --win-rate 0.58 \
//!   --alpha 3.2
//! ```
//!
//! # Production Criteria
//!
//! A model is **APPROVED** if it passes ≥4 of these criteria:
//! - Total Return > 0%
//! - Sharpe Ratio > 1.5
//! - Max Drawdown < 20%
//! - Win Rate > 50%
//! - Alpha vs B&H > 0%
//!
//! # Output
//!
//! - Markdown report file (default: `backtest_comparison_report.md`)
//! - Console summary with deployment recommendation
//! - JSON export option for CI/CD integration

use anyhow::Result;
use clap::Parser;
use ml::backtesting::report::{BacktestReport, PerformanceMetrics};
use std::path::PathBuf;
use tracing::info;
use tracing_subscriber;

/// CLI arguments for report generation
#[derive(Parser, Debug)]
#[command(
    name = "generate_backtest_report",
    about = "Generate comprehensive DQN backtesting comparison report",
    long_about = "Creates markdown reports comparing new DQN models against baseline (Trial #35) with automated deployment recommendations."
)]
struct Args {
    /// Name of the new model being evaluated
    #[arg(long, default_value = "DQN-New-Model")]
    new_model: String,

    /// Name of the baseline model for comparison
    #[arg(long, default_value = "DQN-Trial35-Baseline")]
    baseline_model: String,

    /// Output markdown file path
    #[arg(long, default_value = "backtest_comparison_report.md")]
    output_file: PathBuf,

    /// Use Trial #35 baseline metrics (default)
    #[arg(long, default_value_t = true)]
    use_trial35_baseline: bool,

    /// Total return percentage (e.g., 15.2 = 15.2%)
    #[arg(long)]
    total_return: Option<f64>,

    /// Sharpe ratio
    #[arg(long)]
    sharpe: Option<f64>,

    /// Maximum drawdown percentage (e.g., 12.5 = 12.5%)
    #[arg(long)]
    drawdown: Option<f64>,

    /// Win rate (0.0-1.0, e.g., 0.58 = 58%)
    #[arg(long)]
    win_rate: Option<f64>,

    /// Alpha vs buy-and-hold (percentage)
    #[arg(long)]
    alpha: Option<f64>,

    /// Total number of trades
    #[arg(long)]
    total_trades: Option<usize>,

    /// Average trade return percentage
    #[arg(long)]
    avg_trade_return: Option<f64>,

    /// Verbose logging
    #[arg(short, long)]
    verbose: bool,
}

/// Trial #35 baseline metrics (reference model from hyperopt)
///
/// These metrics represent the baseline DQN model from hyperopt Trial #35.
/// Update these values based on actual backtesting results from Trial #35.
fn get_trial35_baseline() -> PerformanceMetrics {
    // NOTE: These are placeholder values. Replace with actual Trial #35 metrics.
    // Expected source: /tmp/dqn_trial35_backtest_results.json or similar.
    PerformanceMetrics {
        total_return_pct: 12.1,
        sharpe_ratio: 1.8,
        max_drawdown_pct: 18.3,
        win_rate: 0.52,
        alpha: 1.5,
        total_trades: 138,
        avg_trade_return_pct: 0.088,
    }
}

/// Example strong model metrics (for demonstration)
fn get_example_strong_model() -> PerformanceMetrics {
    PerformanceMetrics {
        total_return_pct: 18.5,
        sharpe_ratio: 2.5,
        max_drawdown_pct: 10.2,
        win_rate: 0.62,
        alpha: 4.5,
        total_trades: 150,
        avg_trade_return_pct: 0.123,
    }
}

/// Example marginal model metrics (for demonstration)
fn get_example_marginal_model() -> PerformanceMetrics {
    PerformanceMetrics {
        total_return_pct: 5.3,
        sharpe_ratio: 1.2,
        max_drawdown_pct: 22.8,
        win_rate: 0.48,
        alpha: 0.8,
        total_trades: 125,
        avg_trade_return_pct: 0.042,
    }
}

/// Example weak model metrics (for demonstration)
fn get_example_weak_model() -> PerformanceMetrics {
    PerformanceMetrics {
        total_return_pct: -3.2,
        sharpe_ratio: 0.6,
        max_drawdown_pct: 35.4,
        win_rate: 0.38,
        alpha: -2.1,
        total_trades: 110,
        avg_trade_return_pct: -0.029,
    }
}

fn main() -> Result<()> {
    let args = Args::parse();

    // Initialize logging
    if args.verbose {
        tracing_subscriber::fmt()
            .with_max_level(tracing::Level::DEBUG)
            .init();
    } else {
        tracing_subscriber::fmt()
            .with_max_level(tracing::Level::INFO)
            .init();
    }

    info!("=== DQN Backtesting Report Generator ===");
    info!("New Model: {}", args.new_model);
    info!("Baseline Model: {}", args.baseline_model);
    info!("Output File: {}", args.output_file.display());

    // Determine new model metrics
    let new_results = if let Some(total_return) = args.total_return {
        // Use CLI arguments if provided
        PerformanceMetrics {
            total_return_pct: total_return,
            sharpe_ratio: args.sharpe.unwrap_or(1.0),
            max_drawdown_pct: args.drawdown.unwrap_or(15.0),
            win_rate: args.win_rate.unwrap_or(0.50),
            alpha: args.alpha.unwrap_or(0.0),
            total_trades: args.total_trades.unwrap_or(100),
            avg_trade_return_pct: args.avg_trade_return.unwrap_or(0.05),
        }
    } else {
        // Use example strong model for demonstration
        info!("No metrics provided via CLI, using example strong model");
        get_example_strong_model()
    };

    // Determine baseline metrics
    let baseline_results = if args.use_trial35_baseline {
        Some(get_trial35_baseline())
    } else {
        None
    };

    // Create report
    let report = BacktestReport {
        model_name: args.new_model.clone(),
        baseline_name: args.baseline_model.clone(),
        new_results,
        baseline_results,
    };

    // Generate markdown
    let markdown = report.generate_markdown();

    // Write to file
    std::fs::write(&args.output_file, &markdown)?;
    info!("✅ Report generated: {}", args.output_file.display());

    // Print summary to console
    println!("\n{}", "=".repeat(80));
    println!("REPORT SUMMARY");
    println!("{}", "=".repeat(80));
    println!("\nModel: {}", report.model_name);
    println!("Baseline: {}", report.baseline_name);
    println!("\nPerformance:");
    println!(
        "  Total Return: {:.2}%",
        report.new_results.total_return_pct
    );
    println!("  Sharpe Ratio: {:.2}", report.new_results.sharpe_ratio);
    println!(
        "  Max Drawdown: {:.2}%",
        report.new_results.max_drawdown_pct
    );
    println!("  Win Rate: {:.1}%", report.new_results.win_rate * 100.0);
    println!("  Alpha: {:.2}%", report.new_results.alpha);
    println!("  Total Trades: {}", report.new_results.total_trades);

    if let Some(baseline) = &report.baseline_results {
        println!("\nComparison vs Baseline:");
        let return_diff = report.new_results.total_return_pct - baseline.total_return_pct;
        let sharpe_diff = report.new_results.sharpe_ratio - baseline.sharpe_ratio;
        let dd_diff = report.new_results.max_drawdown_pct - baseline.max_drawdown_pct;
        let wr_diff = (report.new_results.win_rate - baseline.win_rate) * 100.0;

        println!("  Return: {:+.2}%", return_diff);
        println!("  Sharpe: {:+.2}", sharpe_diff);
        println!("  Drawdown: {:+.2}%", dd_diff);
        println!("  Win Rate: {:+.1}%", wr_diff);
    }

    // Print recommendation
    let recommendation = report.get_recommendation();
    println!("\nDeployment Recommendation:");
    println!("  {}", recommendation.status);
    println!("\n{}", "=".repeat(80));

    // Generate additional example reports for demonstration
    if args.verbose {
        info!("\n\nGenerating additional example reports for comparison...");

        // Marginal model example
        let marginal_report = BacktestReport {
            model_name: "DQN-Marginal-Example".to_string(),
            baseline_name: args.baseline_model.clone(),
            new_results: get_example_marginal_model(),
            baseline_results: Some(get_trial35_baseline()),
        };
        let marginal_path = args
            .output_file
            .with_file_name("backtest_marginal_example.md");
        std::fs::write(&marginal_path, marginal_report.generate_markdown())?;
        info!("  Generated marginal example: {}", marginal_path.display());

        // Weak model example
        let weak_report = BacktestReport {
            model_name: "DQN-Weak-Example".to_string(),
            baseline_name: args.baseline_model.clone(),
            new_results: get_example_weak_model(),
            baseline_results: Some(get_trial35_baseline()),
        };
        let weak_path = args.output_file.with_file_name("backtest_weak_example.md");
        std::fs::write(&weak_path, weak_report.generate_markdown())?;
        info!("  Generated weak example: {}", weak_path.display());
    }

    Ok(())
}