diff --git a/ml/examples/evaluate_baseline.rs b/ml/examples/evaluate_baseline.rs new file mode 100644 index 000000000..ea4e4a167 --- /dev/null +++ b/ml/examples/evaluate_baseline.rs @@ -0,0 +1,894 @@ +//! Walk-forward evaluation binary for DQN and PPO baseline models. +//! +//! Loads trained model checkpoints, runs inference on walk-forward test data, +//! computes financial metrics (Sharpe, drawdown, win rate, profit factor), +//! and generates a JSON report. +//! +//! # Usage +//! +//! ```bash +//! SQLX_OFFLINE=true cargo run -p ml --example evaluate_baseline -- \ +//! --model both --models-dir ml/trained_models \ +//! --data-dir data/cache/futures-baseline \ +//! --output ml/trained_models/evaluation_report.json +//! ``` + +#![allow(unused_crate_dependencies)] +#![deny( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::indexing_slicing +)] + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use chrono::{DateTime, TimeZone, Utc}; +use clap::Parser; +use serde::Serialize; +use tracing::{error, info, warn}; + +use dbn::decode::DecodeRecord; +use ml::dqn::{DQNConfig, DQN}; +use ml::features::extraction::extract_ml_features; +use ml::ppo::ppo::{PPOConfig, PPO}; +use ml::types::OHLCVBar; +use ml::walk_forward::{generate_walk_forward_windows, NormStats, WalkForwardConfig}; + +// --------------------------------------------------------------------------- +// CLI Arguments +// --------------------------------------------------------------------------- + +/// Walk-forward evaluation binary for DQN/PPO baseline models. +#[derive(Parser, Debug)] +#[command( + name = "evaluate_baseline", + about = "Evaluate trained DQN/PPO checkpoints with walk-forward test data" +)] +struct Args { + /// Directory containing trained model checkpoints + #[arg(long, default_value = "ml/trained_models")] + models_dir: PathBuf, + + /// Path to directory containing .dbn.zst files + #[arg(long, default_value = "data/cache/futures-baseline")] + data_dir: PathBuf, + + /// Output path for evaluation report JSON + #[arg(long, default_value = "ml/trained_models/evaluation_report.json")] + output: PathBuf, + + /// Which model(s) to evaluate: "dqn", "ppo", or "both" + #[arg(long, default_value = "both")] + model: String, + + /// Feature dimension (must match extract_ml_features output) + #[arg(long, default_value_t = 51)] + feature_dim: usize, + + /// Number of actions for the DQN/PPO action space + #[arg(long, default_value_t = 3)] + num_actions: usize, +} + +// --------------------------------------------------------------------------- +// Report Data Types +// --------------------------------------------------------------------------- + +/// Metrics for a single fold/model combination. +#[derive(Debug, Serialize)] +struct FoldMetrics { + fold: usize, + model: String, + sharpe_ratio: f64, + max_drawdown_pct: f64, + win_rate_pct: f64, + profit_factor: f64, + total_return_pct: f64, + num_trades: usize, + test_start: String, + test_end: String, +} + +/// Aggregate metrics across all folds for both models. +#[derive(Debug, Serialize)] +struct AggregateMetrics { + dqn_avg_sharpe: f64, + dqn_avg_drawdown: f64, + dqn_avg_win_rate: f64, + ppo_avg_sharpe: f64, + ppo_avg_drawdown: f64, + ppo_avg_win_rate: f64, +} + +/// Sanity checks to flag obviously broken models. +#[derive(Debug, Serialize)] +struct SanityChecks { + /// True if any model has Sharpe > 0 + beats_random: bool, + /// True if all 3 actions (buy/sell/hold) were used + action_diversity: bool, + /// True if Sharpe std < 2x |mean Sharpe| + fold_consistency: bool, +} + +/// Full evaluation report written to JSON. +#[derive(Debug, Serialize)] +struct EvaluationReport { + folds: Vec, + aggregate: AggregateMetrics, + sanity_checks: SanityChecks, +} + +// --------------------------------------------------------------------------- +// DBN Loading (reused from train_baseline) +// --------------------------------------------------------------------------- + +/// Recursively discover .dbn.zst files under `dir`. +fn find_dbn_files(dir: &Path) -> Result> { + let mut files = Vec::new(); + if !dir.exists() { + anyhow::bail!("Data directory does not exist: {}", dir.display()); + } + collect_dbn_files_recursive(dir, &mut files)?; + files.sort(); + Ok(files) +} + +/// Recursive helper that walks the directory tree without `walkdir`. +fn collect_dbn_files_recursive(dir: &Path, out: &mut Vec) -> Result<()> { + let entries = std::fs::read_dir(dir) + .with_context(|| format!("Cannot read directory: {}", dir.display()))?; + for entry in entries { + let entry = entry.with_context(|| "Failed to read dir entry")?; + let path = entry.path(); + if path.is_dir() { + collect_dbn_files_recursive(&path, out)?; + } else if path + .file_name() + .and_then(|n| n.to_str()) + .map(|n| n.ends_with(".dbn.zst")) + .unwrap_or(false) + { + out.push(path); + } + } + Ok(()) +} + +/// Load OHLCV bars from a single .dbn.zst file. +fn load_bars_from_dbn(path: &Path) -> Result> { + use dbn::decode::dbn::Decoder; + use dbn::OhlcvMsg; + + let file = std::fs::File::open(path) + .with_context(|| format!("Cannot open DBN file: {}", path.display()))?; + let buf = std::io::BufReader::new(file); + let mut decoder = Decoder::new(buf) + .with_context(|| format!("Failed to create DBN decoder for {}", path.display()))?; + + let mut bars = Vec::new(); + + while let Some(record) = decoder + .decode_record::() + .with_context(|| format!("Error decoding records from {}", path.display()))? + { + let ts_nanos = record.hd.ts_event; + let timestamp = nanos_to_datetime(ts_nanos); + let price_scale = 1e-9_f64; + bars.push(OHLCVBar { + timestamp, + open: record.open as f64 * price_scale, + high: record.high as f64 * price_scale, + low: record.low as f64 * price_scale, + close: record.close as f64 * price_scale, + volume: record.volume as f64, + }); + } + + Ok(bars) +} + +/// Convert nanosecond UNIX timestamp to chrono DateTime. +fn nanos_to_datetime(nanos: u64) -> DateTime { + let secs = (nanos / 1_000_000_000) as i64; + let subsec_nanos = (nanos % 1_000_000_000) as u32; + Utc.timestamp_opt(secs, subsec_nanos) + .single() + .unwrap_or_else(Utc::now) +} + +/// Load all OHLCV bars from a directory of .dbn.zst files, sorted chronologically. +fn load_all_bars(data_dir: &Path) -> Result> { + let dbn_files = find_dbn_files(data_dir)?; + if dbn_files.is_empty() { + anyhow::bail!( + "No .dbn.zst files found in {}. Run download_baseline first.", + data_dir.display() + ); + } + info!( + "Found {} .dbn.zst files in {}", + dbn_files.len(), + data_dir.display() + ); + + let mut all_bars = Vec::new(); + for path in &dbn_files { + match load_bars_from_dbn(path) { + Ok(bars) => { + info!(" {} -> {} bars", path.display(), bars.len()); + all_bars.extend(bars); + } + Err(e) => { + warn!(" Skipping {} — {}", path.display(), e); + } + } + } + + all_bars.sort_by_key(|b| b.timestamp); + info!("Total bars loaded: {}", all_bars.len()); + Ok(all_bars) +} + +// --------------------------------------------------------------------------- +// Financial Metrics +// --------------------------------------------------------------------------- + +/// Container for computed financial metrics from a sequence of trade returns. +struct ComputedMetrics { + sharpe_ratio: f64, + max_drawdown_pct: f64, + win_rate_pct: f64, + profit_factor: f64, + total_return_pct: f64, + num_trades: usize, +} + +/// Compute financial metrics from a sequence of per-bar trade returns. +/// +/// - Sharpe: annualized (mean / std * sqrt(252)) +/// - Max drawdown: largest peak-to-trough drop on cumulative equity curve (%) +/// - Win rate: percentage of returns > 0 +/// - Profit factor: gross_profit / gross_loss (inf if no losses) +/// - Total return: sum of returns * 100 (as percentage) +/// - Num trades: count of non-zero returns (BUY or SELL actions) +fn compute_metrics(returns: &[f64]) -> ComputedMetrics { + let n = returns.len(); + if n == 0 { + return ComputedMetrics { + sharpe_ratio: 0.0, + max_drawdown_pct: 0.0, + win_rate_pct: 0.0, + profit_factor: 0.0, + total_return_pct: 0.0, + num_trades: 0, + }; + } + + // Count actual trades (non-zero returns, i.e. BUY or SELL actions) + let num_trades = returns.iter().filter(|&&r| r.abs() > 1e-12).count(); + + // Mean and std of returns + let sum: f64 = returns.iter().sum(); + let mean = sum / n as f64; + + let variance: f64 = returns.iter().map(|&r| (r - mean).powi(2)).sum::() / n as f64; + let std = variance.sqrt(); + + // Annualized Sharpe ratio + let sharpe_ratio = if std > 1e-12 { + (mean / std) * 252.0_f64.sqrt() + } else { + 0.0 + }; + + // Max drawdown on cumulative equity curve + let mut equity = 1.0_f64; + let mut peak = 1.0_f64; + let mut max_drawdown = 0.0_f64; + + for &ret in returns { + equity += ret; + if equity > peak { + peak = equity; + } + let drawdown = if peak > 1e-12 { + (peak - equity) / peak + } else { + 0.0 + }; + if drawdown > max_drawdown { + max_drawdown = drawdown; + } + } + let max_drawdown_pct = max_drawdown * 100.0; + + // Win rate + let wins = returns.iter().filter(|&&r| r > 0.0).count(); + let win_rate_pct = if num_trades > 0 { + (wins as f64 / num_trades as f64) * 100.0 + } else { + 0.0 + }; + + // Profit factor + let gross_profit: f64 = returns.iter().filter(|&&r| r > 0.0).sum(); + let gross_loss: f64 = returns.iter().filter(|&&r| r < 0.0).map(|&r| r.abs()).sum(); + let profit_factor = if gross_loss > 1e-12 { + gross_profit / gross_loss + } else if gross_profit > 0.0 { + f64::INFINITY + } else { + 0.0 + }; + + // Total return + let total_return_pct = sum * 100.0; + + ComputedMetrics { + sharpe_ratio, + max_drawdown_pct, + win_rate_pct, + profit_factor, + total_return_pct, + num_trades, + } +} + +// --------------------------------------------------------------------------- +// DQN Evaluation +// --------------------------------------------------------------------------- + +/// Run DQN inference on test features and return per-bar trade returns and +/// action counts (buy, sell, hold). +fn evaluate_dqn_fold( + fold: usize, + test_features: &[[f64; 51]], + test_bars: &[OHLCVBar], + models_dir: &Path, + args: &Args, +) -> Result<(Vec, [usize; 3])> { + let ckpt_path = models_dir.join(format!("dqn_fold{}_best.safetensors", fold)); + if !ckpt_path.exists() { + anyhow::bail!( + "DQN checkpoint not found: {}", + ckpt_path.display() + ); + } + + // Create DQN with same config as training + let config = DQNConfig { + state_dim: args.feature_dim, + num_actions: args.num_actions, + hidden_dims: vec![128, 64], + learning_rate: 1e-4, + gamma: 0.99, + epsilon_start: 0.0, // No exploration during evaluation + epsilon_end: 0.0, + epsilon_decay: 1.0, + replay_buffer_capacity: 100, // Minimal buffer, not used for eval + batch_size: 64, + min_replay_size: 64, + target_update_freq: 500, + warmup_steps: 0, + use_double_dqn: true, + use_huber_loss: true, + use_per: false, + use_dueling: false, + use_distributional: false, + use_noisy_nets: false, + use_cql: false, + use_iqn: false, + use_cvar_action_selection: false, + ..DQNConfig::default() + }; + + let mut dqn = DQN::new(config).context("Failed to create DQN model")?; + + // Set epsilon to zero for pure greedy evaluation + dqn.set_epsilon(0.0); + + // Load trained weights + dqn.load_from_safetensors(&ckpt_path.to_string_lossy()) + .with_context(|| format!("Failed to load DQN checkpoint: {}", ckpt_path.display()))?; + + info!( + " [DQN] Loaded checkpoint: {} (eps={})", + ckpt_path.display(), + dqn.get_epsilon() + ); + + // Run inference + let n = test_features.len(); + let mut returns = Vec::with_capacity(n); + let mut action_counts = [0_usize; 3]; // [buy, sell, hold] + + for i in 0..n.saturating_sub(1) { + let feat = match test_features.get(i) { + Some(f) => f, + None => continue, + }; + + let state: Vec = feat.iter().map(|&v| v as f32).collect(); + + // Select greedy action + let action_idx = match dqn.select_action(&state) { + Ok(fa) => { + let idx = fa.to_index().min(args.num_actions.saturating_sub(1)); + idx as u8 + } + Err(e) => { + warn!(" [DQN] select_action error at step {}: {}", i, e); + 2 // Default to HOLD on error + } + }; + + // Map action index to trade direction + let mapped_action = action_idx.min(2); + + // Track action diversity + if let Some(count) = action_counts.get_mut(mapped_action as usize) { + *count += 1; + } + + // Compute return + let close_cur = test_bars.get(i).map(|b| b.close).unwrap_or(0.0); + let close_next = test_bars.get(i + 1).map(|b| b.close).unwrap_or(close_cur); + let price_change = close_next - close_cur; + + let ret = match mapped_action { + 0 => price_change, // BUY + 1 => -price_change, // SELL + _ => 0.0, // HOLD + }; + returns.push(ret); + } + + Ok((returns, action_counts)) +} + +// --------------------------------------------------------------------------- +// PPO Evaluation +// --------------------------------------------------------------------------- + +/// Run PPO inference on test features and return per-bar trade returns and +/// action counts (buy, sell, hold). +fn evaluate_ppo_fold( + fold: usize, + test_features: &[[f64; 51]], + test_bars: &[OHLCVBar], + models_dir: &Path, + args: &Args, +) -> Result<(Vec, [usize; 3])> { + let actor_path = models_dir.join(format!("ppo_fold{}_actor.safetensors", fold)); + let critic_path = models_dir.join(format!("ppo_fold{}_critic.safetensors", fold)); + + if !actor_path.exists() { + anyhow::bail!( + "PPO actor checkpoint not found: {}", + actor_path.display() + ); + } + if !critic_path.exists() { + anyhow::bail!( + "PPO critic checkpoint not found: {}", + critic_path.display() + ); + } + + // Create PPO config matching training + let config = PPOConfig { + state_dim: args.feature_dim, + num_actions: args.num_actions, + policy_hidden_dims: vec![128, 64], + value_hidden_dims: vec![128, 64], + policy_learning_rate: 3e-4, + value_learning_rate: 1e-3, + clip_epsilon: 0.2, + value_loss_coeff: 0.5, + entropy_coeff: 0.01, + batch_size: 64, + mini_batch_size: 64, + num_epochs: 4, + max_grad_norm: 0.5, + use_lstm: false, + ..PPOConfig::default() + }; + + // Load PPO from checkpoint + let ppo = PPO::load_checkpoint( + &actor_path.to_string_lossy(), + &critic_path.to_string_lossy(), + config, + candle_core::Device::Cpu, + ) + .with_context(|| { + format!( + "Failed to load PPO checkpoint: actor={}, critic={}", + actor_path.display(), + critic_path.display() + ) + })?; + + info!( + " [PPO] Loaded checkpoint: {}", + actor_path.display() + ); + + // Run inference + let n = test_features.len(); + let mut returns = Vec::with_capacity(n); + let mut action_counts = [0_usize; 3]; // [buy, sell, hold] + + for i in 0..n.saturating_sub(1) { + let feat = match test_features.get(i) { + Some(f) => f, + None => continue, + }; + + let state: Vec = feat.iter().map(|&v| v as f32).collect(); + + // Get action from policy + let action_idx = match ppo.act(&state) { + Ok((action, _value)) => action.to_int().min(2), + Err(e) => { + warn!(" [PPO] act error at step {}: {}", i, e); + 2 // Default to HOLD on error + } + }; + + // Track action diversity + if let Some(count) = action_counts.get_mut(action_idx as usize) { + *count += 1; + } + + // Compute return + let close_cur = test_bars.get(i).map(|b| b.close).unwrap_or(0.0); + let close_next = test_bars.get(i + 1).map(|b| b.close).unwrap_or(close_cur); + let price_change = close_next - close_cur; + + let ret = match action_idx { + 0 => price_change, // BUY + 1 => -price_change, // SELL + _ => 0.0, // HOLD + }; + returns.push(ret); + } + + Ok((returns, action_counts)) +} + +// --------------------------------------------------------------------------- +// Aggregate & Sanity Checks +// --------------------------------------------------------------------------- + +/// Compute average metrics for a specific model across all folds. +fn compute_aggregate(folds: &[FoldMetrics], model_name: &str) -> (f64, f64, f64) { + let model_folds: Vec<&FoldMetrics> = folds.iter().filter(|f| f.model == model_name).collect(); + if model_folds.is_empty() { + return (0.0, 0.0, 0.0); + } + let n = model_folds.len() as f64; + let avg_sharpe = model_folds.iter().map(|f| f.sharpe_ratio).sum::() / n; + let avg_dd = model_folds.iter().map(|f| f.max_drawdown_pct).sum::() / n; + let avg_wr = model_folds.iter().map(|f| f.win_rate_pct).sum::() / n; + (avg_sharpe, avg_dd, avg_wr) +} + +/// Run sanity checks across all fold metrics. +fn run_sanity_checks( + folds: &[FoldMetrics], + all_action_counts: &[[usize; 3]], +) -> SanityChecks { + // beats_random: any model Sharpe > 0? + let beats_random = folds.iter().any(|f| f.sharpe_ratio > 0.0); + + // action_diversity: all 3 actions used across all evaluations? + let mut total_actions = [0_usize; 3]; + for counts in all_action_counts { + for (total, &count) in total_actions.iter_mut().zip(counts.iter()) { + *total += count; + } + } + let action_diversity = total_actions.iter().all(|&c| c > 0); + + // fold_consistency: std(Sharpe) < 2 * |mean(Sharpe)| across all folds + let sharpe_values: Vec = folds.iter().map(|f| f.sharpe_ratio).collect(); + let fold_consistency = if sharpe_values.is_empty() { + false + } else { + let n = sharpe_values.len() as f64; + let mean_sharpe = sharpe_values.iter().sum::() / n; + let var = sharpe_values + .iter() + .map(|&s| (s - mean_sharpe).powi(2)) + .sum::() + / n; + let std_sharpe = var.sqrt(); + std_sharpe < 2.0 * mean_sharpe.abs() + }; + + SanityChecks { + beats_random, + action_diversity, + fold_consistency, + } +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +fn main() -> Result<()> { + // Initialize tracing + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .init(); + + let args = Args::parse(); + + let eval_dqn = args.model == "dqn" || args.model == "both"; + let eval_ppo = args.model == "ppo" || args.model == "both"; + + info!("=== Walk-Forward Baseline Evaluation ==="); + info!(" Model(s): {}", args.model); + info!(" Models dir: {}", args.models_dir.display()); + info!(" Data dir: {}", args.data_dir.display()); + info!(" Output: {}", args.output.display()); + info!(" Feature dim: {}", args.feature_dim); + info!(" Num actions: {}", args.num_actions); + + // 1. Load all OHLCV bars from DBN files + info!("Step 1/5: Loading OHLCV bars from DBN files..."); + let bars = load_all_bars(&args.data_dir)?; + if bars.is_empty() { + anyhow::bail!("No bars loaded from {}", args.data_dir.display()); + } + info!( + " Loaded {} bars ({} to {})", + bars.len(), + bars.first().map(|b| b.timestamp.to_string()).unwrap_or_default(), + bars.last().map(|b| b.timestamp.to_string()).unwrap_or_default(), + ); + + // 2. Generate walk-forward windows (same config as training) + info!("Step 2/5: Generating walk-forward windows..."); + let wf_config = WalkForwardConfig::default(); + let windows = generate_walk_forward_windows(&bars, &wf_config); + if windows.is_empty() { + anyhow::bail!( + "No walk-forward windows generated. Need at least {} months of data.", + wf_config.initial_train_months + wf_config.val_months + wf_config.test_months + ); + } + info!(" Generated {} walk-forward folds", windows.len()); + + // 3. Evaluate each fold + info!("Step 3/5: Evaluating models on test data..."); + let mut all_fold_metrics: Vec = Vec::new(); + let mut all_action_counts: Vec<[usize; 3]> = Vec::new(); + + for window in &windows { + info!( + "--- Fold {} --- Test: {} bars ({} to {})", + window.fold, + window.test.len(), + window.test + .first() + .map(|b| b.timestamp.to_string()) + .unwrap_or_default(), + window.test + .last() + .map(|b| b.timestamp.to_string()) + .unwrap_or_default(), + ); + + // Load NormStats from training + let norm_path = args + .models_dir + .join(format!("norm_stats_fold{}.json", window.fold)); + let norm_stats: NormStats = if norm_path.exists() { + let norm_json = std::fs::read_to_string(&norm_path) + .with_context(|| format!("Failed to read {}", norm_path.display()))?; + serde_json::from_str(&norm_json) + .with_context(|| format!("Failed to parse {}", norm_path.display()))? + } else { + warn!( + " NormStats not found at {}, computing from test data (degraded)", + norm_path.display() + ); + // Fallback: compute from test data (not ideal, but allows evaluation) + let test_feat = extract_ml_features(&window.test) + .context("Feature extraction failed for test bars")?; + NormStats::from_features(&test_feat) + }; + + // Extract features from test bars + let test_features = match extract_ml_features(&window.test) { + Ok(f) => f, + Err(e) => { + warn!( + " Fold {} — test feature extraction failed: {}", + window.fold, e + ); + continue; + } + }; + + if test_features.is_empty() { + warn!(" Fold {} — empty test features, skipping", window.fold); + continue; + } + + // Normalize test features + let test_norm = norm_stats.normalize_batch(&test_features); + + // Align bars to features (features skip warmup period) + let warmup_offset = window.test.len().saturating_sub(test_norm.len()); + let test_bars_aligned = if warmup_offset < window.test.len() { + &window.test[warmup_offset..] + } else { + &window.test + }; + + // Test period date range for the report + let test_start = test_bars_aligned + .first() + .map(|b| b.timestamp.format("%Y-%m-%d").to_string()) + .unwrap_or_default(); + let test_end = test_bars_aligned + .last() + .map(|b| b.timestamp.format("%Y-%m-%d").to_string()) + .unwrap_or_default(); + + // Evaluate DQN + if eval_dqn { + match evaluate_dqn_fold( + window.fold, + &test_norm, + test_bars_aligned, + &args.models_dir, + &args, + ) { + Ok((returns, action_counts)) => { + let metrics = compute_metrics(&returns); + info!( + " [DQN] Fold {} — Sharpe={:.4} MaxDD={:.2}% WinRate={:.1}% PF={:.2} Return={:.4}% Trades={}", + window.fold, + metrics.sharpe_ratio, + metrics.max_drawdown_pct, + metrics.win_rate_pct, + metrics.profit_factor, + metrics.total_return_pct, + metrics.num_trades, + ); + info!( + " [DQN] Actions — BUY={} SELL={} HOLD={}", + action_counts.first().copied().unwrap_or(0), + action_counts.get(1).copied().unwrap_or(0), + action_counts.get(2).copied().unwrap_or(0), + ); + all_fold_metrics.push(FoldMetrics { + fold: window.fold, + model: "dqn".to_string(), + sharpe_ratio: metrics.sharpe_ratio, + max_drawdown_pct: metrics.max_drawdown_pct, + win_rate_pct: metrics.win_rate_pct, + profit_factor: metrics.profit_factor, + total_return_pct: metrics.total_return_pct, + num_trades: metrics.num_trades, + test_start: test_start.clone(), + test_end: test_end.clone(), + }); + all_action_counts.push(action_counts); + } + Err(e) => { + error!(" [DQN] Fold {} evaluation failed: {}", window.fold, e); + } + } + } + + // Evaluate PPO + if eval_ppo { + match evaluate_ppo_fold( + window.fold, + &test_norm, + test_bars_aligned, + &args.models_dir, + &args, + ) { + Ok((returns, action_counts)) => { + let metrics = compute_metrics(&returns); + info!( + " [PPO] Fold {} — Sharpe={:.4} MaxDD={:.2}% WinRate={:.1}% PF={:.2} Return={:.4}% Trades={}", + window.fold, + metrics.sharpe_ratio, + metrics.max_drawdown_pct, + metrics.win_rate_pct, + metrics.profit_factor, + metrics.total_return_pct, + metrics.num_trades, + ); + info!( + " [PPO] Actions — BUY={} SELL={} HOLD={}", + action_counts.first().copied().unwrap_or(0), + action_counts.get(1).copied().unwrap_or(0), + action_counts.get(2).copied().unwrap_or(0), + ); + all_fold_metrics.push(FoldMetrics { + fold: window.fold, + model: "ppo".to_string(), + sharpe_ratio: metrics.sharpe_ratio, + max_drawdown_pct: metrics.max_drawdown_pct, + win_rate_pct: metrics.win_rate_pct, + profit_factor: metrics.profit_factor, + total_return_pct: metrics.total_return_pct, + num_trades: metrics.num_trades, + test_start: test_start.clone(), + test_end: test_end.clone(), + }); + all_action_counts.push(action_counts); + } + Err(e) => { + error!(" [PPO] Fold {} evaluation failed: {}", window.fold, e); + } + } + } + } + + // 4. Compute aggregate metrics + info!("Step 4/5: Computing aggregate metrics..."); + let (dqn_avg_sharpe, dqn_avg_drawdown, dqn_avg_win_rate) = + compute_aggregate(&all_fold_metrics, "dqn"); + let (ppo_avg_sharpe, ppo_avg_drawdown, ppo_avg_win_rate) = + compute_aggregate(&all_fold_metrics, "ppo"); + + let aggregate = AggregateMetrics { + dqn_avg_sharpe, + dqn_avg_drawdown, + dqn_avg_win_rate, + ppo_avg_sharpe, + ppo_avg_drawdown, + ppo_avg_win_rate, + }; + + info!(" DQN — avg Sharpe={:.4} avg MaxDD={:.2}% avg WinRate={:.1}%", + dqn_avg_sharpe, dqn_avg_drawdown, dqn_avg_win_rate); + info!(" PPO — avg Sharpe={:.4} avg MaxDD={:.2}% avg WinRate={:.1}%", + ppo_avg_sharpe, ppo_avg_drawdown, ppo_avg_win_rate); + + // 5. Sanity checks & report + info!("Step 5/5: Running sanity checks and saving report..."); + let sanity_checks = run_sanity_checks(&all_fold_metrics, &all_action_counts); + + info!(" Beats random: {}", sanity_checks.beats_random); + info!(" Action diversity: {}", sanity_checks.action_diversity); + info!(" Fold consistency: {}", sanity_checks.fold_consistency); + + let report = EvaluationReport { + folds: all_fold_metrics, + aggregate, + sanity_checks, + }; + + // Save report + if let Some(parent) = args.output.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("Failed to create output dir: {}", parent.display()))?; + } + let report_json = serde_json::to_string_pretty(&report) + .context("Failed to serialize evaluation report")?; + std::fs::write(&args.output, &report_json) + .with_context(|| format!("Failed to write report to {}", args.output.display()))?; + + info!("=== Evaluation Complete ==="); + info!(" Report saved to: {}", args.output.display()); + info!(" Total fold evaluations: {}", report.folds.len()); + + Ok(()) +}