foxhunt/ml/examples/evaluate_production_checkpoint.rs

//! Evaluate production DQN checkpoint
//!
//! This script loads the trained checkpoint and runs backtest evaluation
//! to extract Sharpe ratio and other performance metrics.

use anyhow::{Context, Result};
use candle_core::{Device, Tensor};
use ml::data_loaders::load_parquet_data;
use ml::dqn::dqn::WorkingDQN;
use ml::evaluation::engine::{Action, EvaluationEngine};
use ml::evaluation::metrics::{OHLCVBarF32, PerformanceMetrics};
use std::path::Path;
use tracing::info;

#[tokio::main]
async fn main() -> Result<()> {
    // Initialize logging
    tracing_subscriber::fmt()
        .with_max_level(tracing::Level::INFO)
        .init();

    let checkpoint_path = "/home/jgrusewski/Work/foxhunt/ml/trained_models/dqn_epoch_100.safetensors";
    let data_path = "/home/jgrusewski/Work/foxhunt/test_data/ES_FUT_180d.parquet";
    let initial_capital = 100000.0;

    info!("=== Production DQN Checkpoint Evaluation ===");
    info!("Checkpoint: {}", checkpoint_path);
    info!("Data: {}", data_path);
    info!("Capital: ${}", initial_capital);

    // Step 1: Load parquet data (returns Vec<[f64; 128]>)
    info!("Loading validation data...");
    let data = load_parquet_data(Path::new(data_path), 50)
        .context("Failed to load parquet data")?;

    info!("Loaded {} samples from parquet", data.len());

    // Step 2: Load DQN model from checkpoint
    info!("Loading DQN model from checkpoint...");
    let device = Device::cuda_if_available(0)?;
    info!("Device: {:?}", device);

    // Create DQN with default config (architecture must match saved model)
    use ml::dqn::dqn::WorkingDQNConfig;
    let config = WorkingDQNConfig {
        state_dim: 54,
        num_actions: 45,
        hidden_dims: vec![256, 128, 64],
        learning_rate: 0.000037,
        gamma: 0.974,
        epsilon_start: 0.1,
        epsilon_end: 0.01,
        epsilon_decay: 0.995,
        replay_buffer_capacity: 94016,
        batch_size: 114,
        target_update_frequency: 10,
        tau: 0.001,
        warmup_steps: 0,
        save_interval: 10,
        device: device.clone(),
        use_double_dqn: true,
        use_dueling: true,
        use_noisy_nets: true,
        use_per: true,
        per_alpha: 0.504,
        per_beta_start: 0.239,
        per_beta_end: 1.0,
        per_beta_frames: 100000,
        categorical_v_min: -1.6,
        categorical_v_max: 2.1,
        categorical_num_atoms: 200,
        n_step: 2,
        gradient_clip: 10.0,
        eval_interval: 10,
        eval_episodes: 5,
    };

    let mut dqn = WorkingDQN::new(config)?;

    // Load weights from checkpoint
    dqn.load_from_safetensors(checkpoint_path)?;
    info!("Model loaded successfully");

    // Step 3: Run backtest evaluation
    info!("Running backtest evaluation...");
    let mut engine = EvaluationEngine::new(initial_capital);
    let mut ohlcv_bars = Vec::with_capacity(data.len());

    for (bar_idx, feature_array) in data.iter().enumerate() {
        // Extract close price from features (index 3 in the 128-dim feature array)
        let close_price = feature_array[3] as f32;

        // Convert feature array to Tensor (54-dim state expected by DQN)
        // We need to pad/truncate the 128-dim array to 54-dim
        let mut state_vec = vec![0.0f32; 54];
        for (i, &val) in feature_array.iter().take(128).enumerate() {
            state_vec[i] = val as f32;
        }

        let state = Tensor::from_vec(state_vec, (1, 54), &device)
            .context("Failed to create state tensor")?;

        // Get action from DQN (greedy, no exploration)
        let action_idx = dqn.select_action(&state, 0.0)?; // epsilon=0.0 for greedy
        let action = Action::from(action_idx);

        // Create OHLCV bar
        let bar = OHLCVBarF32 {
            timestamp: bar_idx as i64,
            open: close_price,
            high: close_price,
            low: close_price,
            close: close_price,
            volume: 0.0,
        };

        // Process bar through evaluation engine
        engine.process_bar(bar_idx, &bar, action);
        ohlcv_bars.push(bar);

        // Log progress every 1000 bars
        if (bar_idx + 1) % 1000 == 0 {
            info!("Processed {}/{} bars", bar_idx + 1, data.len());
        }
    }

    // Close any open position
    if let Some(last_bar) = ohlcv_bars.last() {
        engine.close_position(ohlcv_bars.len() - 1, last_bar);
    }

    // Step 4: Calculate performance metrics
    info!("Calculating performance metrics...");
    let metrics = PerformanceMetrics::from_trades(&engine.trades, initial_capital, &ohlcv_bars);
    let action_dist = engine.get_action_distribution();

    // Step 5: Print comprehensive report
    println!("\n{}", "=".repeat(70));
    println!("PRODUCTION DQN MODEL EVALUATION REPORT");
    println!("{}", "=".repeat(70));
    println!();

    println!("MODEL DETAILS:");
    println!("  Checkpoint: {}", checkpoint_path);
    println!("  Training: 100 epochs, 65.4 minutes, $0.27 cost");
    println!("  Final Loss: -0.088, Q-value: 0.888");
    println!();

    println!("VALIDATION TARGET (Trial #2):");
    println!("  Sharpe Ratio: 1.2464");
    println!("  Win Rate: 53.59%");
    println!("  Max Drawdown: 0.13%");
    println!("  Total Return: 1.71%");
    println!();

    println!("BACKTEST PERFORMANCE:");
    println!("  Total Return: {:.2}%", metrics.total_return_pct);
    println!("  Sharpe Ratio: {:.4}", metrics.sharpe_ratio);
    println!("  Sortino Ratio: {:.4}", metrics.sortino_ratio);
    println!("  Calmar Ratio: {:.4}", metrics.calmar_ratio);
    println!("  Omega Ratio: {:.4}", metrics.omega_ratio);
    println!("  Max Drawdown: {:.2}%", metrics.max_drawdown_pct);
    println!("  Win Rate: {:.2}%", metrics.win_rate * 100.0);
    println!("  Total Trades: {}", metrics.total_trades);
    println!("  Avg Trade P&L: ${:.2}", metrics.avg_trade_pnl);
    println!("  Final Equity: ${:.2}", metrics.final_equity);
    println!();

    println!("ACTION DISTRIBUTION:");
    println!("  BUY:  {:>6} ({:>5.2}%)", action_dist.buy_count, action_dist.buy_pct);
    println!("  SELL: {:>6} ({:>5.2}%)", action_dist.sell_count, action_dist.sell_pct);
    println!("  HOLD: {:>6} ({:>5.2}%)", action_dist.hold_count, action_dist.hold_pct);
    println!();

    // Compare to validation target
    println!("COMPARISON TO VALIDATION TARGET:");
    let sharpe_diff = ((metrics.sharpe_ratio - 1.2464) / 1.2464 * 100.0).abs();
    let win_rate_diff = ((metrics.win_rate * 100.0 - 53.59) / 53.59 * 100.0).abs();
    let dd_diff = ((metrics.max_drawdown_pct - 0.13) / 0.13 * 100.0).abs();

    println!("  Sharpe: {:.4} vs 1.2464 ({:+.1}%)", metrics.sharpe_ratio,
             (metrics.sharpe_ratio - 1.2464) / 1.2464 * 100.0);
    println!("  Win Rate: {:.2}% vs 53.59% ({:+.1}%)", metrics.win_rate * 100.0,
             (metrics.win_rate * 100.0 - 53.59) / 53.59 * 100.0);
    println!("  Drawdown: {:.2}% vs 0.13% ({:+.1}%)", metrics.max_drawdown_pct,
             (metrics.max_drawdown_pct - 0.13) / 0.13 * 100.0);
    println!();

    // Determine success/failure
    let sharpe_ok = metrics.sharpe_ratio >= 1.05; // ±15% tolerance
    let sharpe_close = metrics.sharpe_ratio >= 0.85; // ±30% tolerance

    println!("PRODUCTION READINESS:");
    if sharpe_ok {
        println!("  ✅ SUCCESS: Sharpe {:.4} >= 1.05 (within ±15% of 1.2464)", metrics.sharpe_ratio);
        println!("  ✅ Hyperopt validated");
        println!("  ✅ Ready for production deployment");
    } else if sharpe_close {
        println!("  ⚠️  PARTIAL SUCCESS: Sharpe {:.4} in range [0.85, 1.05]", metrics.sharpe_ratio);
        println!("  ⚠️  Acceptable variance from target (±30%)");
        println!("  ⚠️  Monitor closely in production");
    } else {
        println!("  ❌ FAILURE: Sharpe {:.4} < 0.85 (>30% below target)", metrics.sharpe_ratio);
        println!("  ❌ Significant mismatch from validation");
        println!("  ❌ Do NOT deploy to production");
    }

    println!("{}", "=".repeat(70));
    println!();

    Ok(())
}