//! Walk-forward evaluation binary for DQN and PPO baseline models. //! //! Loads trained model checkpoints, runs inference on walk-forward test data, //! computes financial metrics (Sharpe, drawdown, win rate, profit factor), //! and generates a JSON report. //! //! # Usage //! //! ```bash //! SQLX_OFFLINE=true cargo run -p ml --example evaluate_baseline -- \ //! --model both --models-dir ml/trained_models \ //! --data-dir data/cache/futures-baseline \ //! --output ml/trained_models/evaluation_report.json //! ``` #![allow(unused_crate_dependencies)] #![deny( clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::indexing_slicing )] use std::path::{Path, PathBuf}; use anyhow::{Context, Result}; use chrono::{DateTime, TimeZone, Utc}; use clap::Parser; use serde::Serialize; use tracing::{error, info, warn}; use dbn::decode::DecodeRecord; use ml::dqn::{DQNConfig, DQN}; use ml::features::extraction::extract_ml_features; use ml::ppo::ppo::{PPOConfig, PPO}; use ml::types::OHLCVBar; use ml::walk_forward::{generate_walk_forward_windows, NormStats, WalkForwardConfig}; // --------------------------------------------------------------------------- // CLI Arguments // --------------------------------------------------------------------------- /// Walk-forward evaluation binary for DQN/PPO baseline models. #[derive(Parser, Debug)] #[command( name = "evaluate_baseline", about = "Evaluate trained DQN/PPO checkpoints with walk-forward test data" )] struct Args { /// Directory containing trained model checkpoints #[arg(long, default_value = "ml/trained_models")] models_dir: PathBuf, /// Path to directory containing .dbn.zst files #[arg(long, default_value = "data/cache/futures-baseline")] data_dir: PathBuf, /// Output path for evaluation report JSON #[arg(long, default_value = "ml/trained_models/evaluation_report.json")] output: PathBuf, /// Which model(s) to evaluate: "dqn", "ppo", or "both" #[arg(long, default_value = "both")] model: String, /// Feature dimension (must match extract_ml_features output) #[arg(long, default_value_t = 51)] feature_dim: usize, /// Number of actions for the DQN/PPO action space #[arg(long, default_value_t = 3)] num_actions: usize, } // --------------------------------------------------------------------------- // Report Data Types // --------------------------------------------------------------------------- /// Metrics for a single fold/model combination. #[derive(Debug, Serialize)] struct FoldMetrics { fold: usize, model: String, sharpe_ratio: f64, max_drawdown_pct: f64, win_rate_pct: f64, profit_factor: f64, total_return_pct: f64, num_trades: usize, test_start: String, test_end: String, } /// Aggregate metrics across all folds for both models. #[derive(Debug, Serialize)] struct AggregateMetrics { dqn_avg_sharpe: f64, dqn_avg_drawdown: f64, dqn_avg_win_rate: f64, ppo_avg_sharpe: f64, ppo_avg_drawdown: f64, ppo_avg_win_rate: f64, } /// Sanity checks to flag obviously broken models. #[derive(Debug, Serialize)] struct SanityChecks { /// True if any model has Sharpe > 0 beats_random: bool, /// True if all 3 actions (buy/sell/hold) were used action_diversity: bool, /// True if Sharpe std < 2x |mean Sharpe| fold_consistency: bool, } /// Full evaluation report written to JSON. #[derive(Debug, Serialize)] struct EvaluationReport { folds: Vec, aggregate: AggregateMetrics, sanity_checks: SanityChecks, } // --------------------------------------------------------------------------- // DBN Loading (reused from train_baseline) // --------------------------------------------------------------------------- /// Recursively discover .dbn.zst files under `dir`. fn find_dbn_files(dir: &Path) -> Result> { let mut files = Vec::new(); if !dir.exists() { anyhow::bail!("Data directory does not exist: {}", dir.display()); } collect_dbn_files_recursive(dir, &mut files)?; files.sort(); Ok(files) } /// Recursive helper that walks the directory tree without `walkdir`. fn collect_dbn_files_recursive(dir: &Path, out: &mut Vec) -> Result<()> { let entries = std::fs::read_dir(dir) .with_context(|| format!("Cannot read directory: {}", dir.display()))?; for entry in entries { let entry = entry.with_context(|| "Failed to read dir entry")?; let path = entry.path(); if path.is_dir() { collect_dbn_files_recursive(&path, out)?; } else if path .file_name() .and_then(|n| n.to_str()) .map(|n| n.ends_with(".dbn.zst")) .unwrap_or(false) { out.push(path); } } Ok(()) } /// Load OHLCV bars from a single .dbn.zst file. fn load_bars_from_dbn(path: &Path) -> Result> { use dbn::decode::dbn::Decoder; use dbn::OhlcvMsg; let file = std::fs::File::open(path) .with_context(|| format!("Cannot open DBN file: {}", path.display()))?; let buf = std::io::BufReader::new(file); let mut decoder = Decoder::new(buf) .with_context(|| format!("Failed to create DBN decoder for {}", path.display()))?; let mut bars = Vec::new(); while let Some(record) = decoder .decode_record::() .with_context(|| format!("Error decoding records from {}", path.display()))? { let ts_nanos = record.hd.ts_event; let timestamp = nanos_to_datetime(ts_nanos); let price_scale = 1e-9_f64; bars.push(OHLCVBar { timestamp, open: record.open as f64 * price_scale, high: record.high as f64 * price_scale, low: record.low as f64 * price_scale, close: record.close as f64 * price_scale, volume: record.volume as f64, }); } Ok(bars) } /// Convert nanosecond UNIX timestamp to chrono DateTime. fn nanos_to_datetime(nanos: u64) -> DateTime { let secs = (nanos / 1_000_000_000) as i64; let subsec_nanos = (nanos % 1_000_000_000) as u32; Utc.timestamp_opt(secs, subsec_nanos) .single() .unwrap_or_else(Utc::now) } /// Load all OHLCV bars from a directory of .dbn.zst files, sorted chronologically. fn load_all_bars(data_dir: &Path) -> Result> { let dbn_files = find_dbn_files(data_dir)?; if dbn_files.is_empty() { anyhow::bail!( "No .dbn.zst files found in {}. Run download_baseline first.", data_dir.display() ); } info!( "Found {} .dbn.zst files in {}", dbn_files.len(), data_dir.display() ); let mut all_bars = Vec::new(); for path in &dbn_files { match load_bars_from_dbn(path) { Ok(bars) => { info!(" {} -> {} bars", path.display(), bars.len()); all_bars.extend(bars); } Err(e) => { warn!(" Skipping {} — {}", path.display(), e); } } } all_bars.sort_by_key(|b| b.timestamp); info!("Total bars loaded: {}", all_bars.len()); Ok(all_bars) } // --------------------------------------------------------------------------- // Financial Metrics // --------------------------------------------------------------------------- /// Container for computed financial metrics from a sequence of trade returns. struct ComputedMetrics { sharpe_ratio: f64, max_drawdown_pct: f64, win_rate_pct: f64, profit_factor: f64, total_return_pct: f64, num_trades: usize, } /// Compute financial metrics from a sequence of per-bar trade returns. /// /// - Sharpe: annualized (mean / std * sqrt(252)) /// - Max drawdown: largest peak-to-trough drop on cumulative equity curve (%) /// - Win rate: percentage of returns > 0 /// - Profit factor: gross_profit / gross_loss (inf if no losses) /// - Total return: sum of returns * 100 (as percentage) /// - Num trades: count of non-zero returns (BUY or SELL actions) fn compute_metrics(returns: &[f64]) -> ComputedMetrics { let n = returns.len(); if n == 0 { return ComputedMetrics { sharpe_ratio: 0.0, max_drawdown_pct: 0.0, win_rate_pct: 0.0, profit_factor: 0.0, total_return_pct: 0.0, num_trades: 0, }; } // Count actual trades (non-zero returns, i.e. BUY or SELL actions) let num_trades = returns.iter().filter(|&&r| r.abs() > 1e-12).count(); // Mean and std of returns let sum: f64 = returns.iter().sum(); let mean = sum / n as f64; let variance: f64 = returns.iter().map(|&r| (r - mean).powi(2)).sum::() / n as f64; let std = variance.sqrt(); // Annualized Sharpe ratio let sharpe_ratio = if std > 1e-12 { (mean / std) * 252.0_f64.sqrt() } else { 0.0 }; // Max drawdown on cumulative equity curve let mut equity = 1.0_f64; let mut peak = 1.0_f64; let mut max_drawdown = 0.0_f64; for &ret in returns { equity += ret; if equity > peak { peak = equity; } let drawdown = if peak > 1e-12 { (peak - equity) / peak } else { 0.0 }; if drawdown > max_drawdown { max_drawdown = drawdown; } } let max_drawdown_pct = max_drawdown * 100.0; // Win rate let wins = returns.iter().filter(|&&r| r > 0.0).count(); let win_rate_pct = if num_trades > 0 { (wins as f64 / num_trades as f64) * 100.0 } else { 0.0 }; // Profit factor let gross_profit: f64 = returns.iter().filter(|&&r| r > 0.0).sum(); let gross_loss: f64 = returns.iter().filter(|&&r| r < 0.0).map(|&r| r.abs()).sum(); let profit_factor = if gross_loss > 1e-12 { gross_profit / gross_loss } else if gross_profit > 0.0 { f64::INFINITY } else { 0.0 }; // Total return let total_return_pct = sum * 100.0; ComputedMetrics { sharpe_ratio, max_drawdown_pct, win_rate_pct, profit_factor, total_return_pct, num_trades, } } // --------------------------------------------------------------------------- // DQN Evaluation // --------------------------------------------------------------------------- /// Run DQN inference on test features and return per-bar trade returns and /// action counts (buy, sell, hold). fn evaluate_dqn_fold( fold: usize, test_features: &[[f64; 51]], test_bars: &[OHLCVBar], models_dir: &Path, args: &Args, ) -> Result<(Vec, [usize; 3])> { let ckpt_path = models_dir.join(format!("dqn_fold{}_best.safetensors", fold)); if !ckpt_path.exists() { anyhow::bail!( "DQN checkpoint not found: {}", ckpt_path.display() ); } // Create DQN with same config as training let config = DQNConfig { state_dim: args.feature_dim, num_actions: args.num_actions, hidden_dims: vec![128, 64], learning_rate: 1e-4, gamma: 0.99, epsilon_start: 0.0, // No exploration during evaluation epsilon_end: 0.0, epsilon_decay: 1.0, replay_buffer_capacity: 100, // Minimal buffer, not used for eval batch_size: 64, min_replay_size: 64, target_update_freq: 500, warmup_steps: 0, use_double_dqn: true, use_huber_loss: true, use_per: false, use_dueling: false, use_distributional: false, use_noisy_nets: false, use_cql: false, use_iqn: false, use_cvar_action_selection: false, ..DQNConfig::default() }; let mut dqn = DQN::new(config).context("Failed to create DQN model")?; // Set epsilon to zero for pure greedy evaluation dqn.set_epsilon(0.0); // Load trained weights dqn.load_from_safetensors(&ckpt_path.to_string_lossy()) .with_context(|| format!("Failed to load DQN checkpoint: {}", ckpt_path.display()))?; info!( " [DQN] Loaded checkpoint: {} (eps={})", ckpt_path.display(), dqn.get_epsilon() ); // Run inference let n = test_features.len(); let mut returns = Vec::with_capacity(n); let mut action_counts = [0_usize; 3]; // [buy, sell, hold] for i in 0..n.saturating_sub(1) { let feat = match test_features.get(i) { Some(f) => f, None => continue, }; let state: Vec = feat.iter().map(|&v| v as f32).collect(); // Select greedy action let action_idx = match dqn.select_action(&state) { Ok(fa) => { let idx = fa.to_index().min(args.num_actions.saturating_sub(1)); idx as u8 } Err(e) => { warn!(" [DQN] select_action error at step {}: {}", i, e); 2 // Default to HOLD on error } }; // Map action index to trade direction let mapped_action = action_idx.min(2); // Track action diversity if let Some(count) = action_counts.get_mut(mapped_action as usize) { *count += 1; } // Compute return let close_cur = test_bars.get(i).map(|b| b.close).unwrap_or(0.0); let close_next = test_bars.get(i + 1).map(|b| b.close).unwrap_or(close_cur); let price_change = close_next - close_cur; let ret = match mapped_action { 0 => price_change, // BUY 1 => -price_change, // SELL _ => 0.0, // HOLD }; returns.push(ret); } Ok((returns, action_counts)) } // --------------------------------------------------------------------------- // PPO Evaluation // --------------------------------------------------------------------------- /// Run PPO inference on test features and return per-bar trade returns and /// action counts (buy, sell, hold). fn evaluate_ppo_fold( fold: usize, test_features: &[[f64; 51]], test_bars: &[OHLCVBar], models_dir: &Path, args: &Args, ) -> Result<(Vec, [usize; 3])> { let actor_path = models_dir.join(format!("ppo_fold{}_actor.safetensors", fold)); let critic_path = models_dir.join(format!("ppo_fold{}_critic.safetensors", fold)); if !actor_path.exists() { anyhow::bail!( "PPO actor checkpoint not found: {}", actor_path.display() ); } if !critic_path.exists() { anyhow::bail!( "PPO critic checkpoint not found: {}", critic_path.display() ); } // Create PPO config matching training let config = PPOConfig { state_dim: args.feature_dim, num_actions: args.num_actions, policy_hidden_dims: vec![128, 64], value_hidden_dims: vec![128, 64], policy_learning_rate: 3e-4, value_learning_rate: 1e-3, clip_epsilon: 0.2, value_loss_coeff: 0.5, entropy_coeff: 0.01, batch_size: 64, mini_batch_size: 64, num_epochs: 4, max_grad_norm: 0.5, use_lstm: false, ..PPOConfig::default() }; // Load PPO from checkpoint let ppo = PPO::load_checkpoint( &actor_path.to_string_lossy(), &critic_path.to_string_lossy(), config, candle_core::Device::Cpu, ) .with_context(|| { format!( "Failed to load PPO checkpoint: actor={}, critic={}", actor_path.display(), critic_path.display() ) })?; info!( " [PPO] Loaded checkpoint: {}", actor_path.display() ); // Run inference let n = test_features.len(); let mut returns = Vec::with_capacity(n); let mut action_counts = [0_usize; 3]; // [buy, sell, hold] for i in 0..n.saturating_sub(1) { let feat = match test_features.get(i) { Some(f) => f, None => continue, }; let state: Vec = feat.iter().map(|&v| v as f32).collect(); // Get action from policy let action_idx = match ppo.act(&state) { Ok((action, _value)) => action.to_int().min(2), Err(e) => { warn!(" [PPO] act error at step {}: {}", i, e); 2 // Default to HOLD on error } }; // Track action diversity if let Some(count) = action_counts.get_mut(action_idx as usize) { *count += 1; } // Compute return let close_cur = test_bars.get(i).map(|b| b.close).unwrap_or(0.0); let close_next = test_bars.get(i + 1).map(|b| b.close).unwrap_or(close_cur); let price_change = close_next - close_cur; let ret = match action_idx { 0 => price_change, // BUY 1 => -price_change, // SELL _ => 0.0, // HOLD }; returns.push(ret); } Ok((returns, action_counts)) } // --------------------------------------------------------------------------- // Aggregate & Sanity Checks // --------------------------------------------------------------------------- /// Compute average metrics for a specific model across all folds. fn compute_aggregate(folds: &[FoldMetrics], model_name: &str) -> (f64, f64, f64) { let model_folds: Vec<&FoldMetrics> = folds.iter().filter(|f| f.model == model_name).collect(); if model_folds.is_empty() { return (0.0, 0.0, 0.0); } let n = model_folds.len() as f64; let avg_sharpe = model_folds.iter().map(|f| f.sharpe_ratio).sum::() / n; let avg_dd = model_folds.iter().map(|f| f.max_drawdown_pct).sum::() / n; let avg_wr = model_folds.iter().map(|f| f.win_rate_pct).sum::() / n; (avg_sharpe, avg_dd, avg_wr) } /// Run sanity checks across all fold metrics. fn run_sanity_checks( folds: &[FoldMetrics], all_action_counts: &[[usize; 3]], ) -> SanityChecks { // beats_random: any model Sharpe > 0? let beats_random = folds.iter().any(|f| f.sharpe_ratio > 0.0); // action_diversity: all 3 actions used across all evaluations? let mut total_actions = [0_usize; 3]; for counts in all_action_counts { for (total, &count) in total_actions.iter_mut().zip(counts.iter()) { *total += count; } } let action_diversity = total_actions.iter().all(|&c| c > 0); // fold_consistency: std(Sharpe) < 2 * |mean(Sharpe)| across all folds let sharpe_values: Vec = folds.iter().map(|f| f.sharpe_ratio).collect(); let fold_consistency = if sharpe_values.is_empty() { false } else { let n = sharpe_values.len() as f64; let mean_sharpe = sharpe_values.iter().sum::() / n; let var = sharpe_values .iter() .map(|&s| (s - mean_sharpe).powi(2)) .sum::() / n; let std_sharpe = var.sqrt(); std_sharpe < 2.0 * mean_sharpe.abs() }; SanityChecks { beats_random, action_diversity, fold_consistency, } } // --------------------------------------------------------------------------- // Main // --------------------------------------------------------------------------- fn main() -> Result<()> { // Initialize tracing tracing_subscriber::fmt() .with_env_filter( tracing_subscriber::EnvFilter::try_from_default_env() .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), ) .init(); let args = Args::parse(); let eval_dqn = args.model == "dqn" || args.model == "both"; let eval_ppo = args.model == "ppo" || args.model == "both"; info!("=== Walk-Forward Baseline Evaluation ==="); info!(" Model(s): {}", args.model); info!(" Models dir: {}", args.models_dir.display()); info!(" Data dir: {}", args.data_dir.display()); info!(" Output: {}", args.output.display()); info!(" Feature dim: {}", args.feature_dim); info!(" Num actions: {}", args.num_actions); // 1. Load all OHLCV bars from DBN files info!("Step 1/5: Loading OHLCV bars from DBN files..."); let bars = load_all_bars(&args.data_dir)?; if bars.is_empty() { anyhow::bail!("No bars loaded from {}", args.data_dir.display()); } info!( " Loaded {} bars ({} to {})", bars.len(), bars.first().map(|b| b.timestamp.to_string()).unwrap_or_default(), bars.last().map(|b| b.timestamp.to_string()).unwrap_or_default(), ); // 2. Generate walk-forward windows (same config as training) info!("Step 2/5: Generating walk-forward windows..."); let wf_config = WalkForwardConfig::default(); let windows = generate_walk_forward_windows(&bars, &wf_config); if windows.is_empty() { anyhow::bail!( "No walk-forward windows generated. Need at least {} months of data.", wf_config.initial_train_months + wf_config.val_months + wf_config.test_months ); } info!(" Generated {} walk-forward folds", windows.len()); // 3. Evaluate each fold info!("Step 3/5: Evaluating models on test data..."); let mut all_fold_metrics: Vec = Vec::new(); let mut all_action_counts: Vec<[usize; 3]> = Vec::new(); for window in &windows { info!( "--- Fold {} --- Test: {} bars ({} to {})", window.fold, window.test.len(), window.test .first() .map(|b| b.timestamp.to_string()) .unwrap_or_default(), window.test .last() .map(|b| b.timestamp.to_string()) .unwrap_or_default(), ); // Load NormStats from training let norm_path = args .models_dir .join(format!("norm_stats_fold{}.json", window.fold)); let norm_stats: NormStats = if norm_path.exists() { let norm_json = std::fs::read_to_string(&norm_path) .with_context(|| format!("Failed to read {}", norm_path.display()))?; serde_json::from_str(&norm_json) .with_context(|| format!("Failed to parse {}", norm_path.display()))? } else { warn!( " NormStats not found at {}, computing from test data (degraded)", norm_path.display() ); // Fallback: compute from test data (not ideal, but allows evaluation) let test_feat = extract_ml_features(&window.test) .context("Feature extraction failed for test bars")?; NormStats::from_features(&test_feat) }; // Extract features from test bars let test_features = match extract_ml_features(&window.test) { Ok(f) => f, Err(e) => { warn!( " Fold {} — test feature extraction failed: {}", window.fold, e ); continue; } }; if test_features.is_empty() { warn!(" Fold {} — empty test features, skipping", window.fold); continue; } // Normalize test features let test_norm = norm_stats.normalize_batch(&test_features); // Align bars to features (features skip warmup period) let warmup_offset = window.test.len().saturating_sub(test_norm.len()); let test_bars_aligned = if warmup_offset < window.test.len() { &window.test[warmup_offset..] } else { &window.test }; // Test period date range for the report let test_start = test_bars_aligned .first() .map(|b| b.timestamp.format("%Y-%m-%d").to_string()) .unwrap_or_default(); let test_end = test_bars_aligned .last() .map(|b| b.timestamp.format("%Y-%m-%d").to_string()) .unwrap_or_default(); // Evaluate DQN if eval_dqn { match evaluate_dqn_fold( window.fold, &test_norm, test_bars_aligned, &args.models_dir, &args, ) { Ok((returns, action_counts)) => { let metrics = compute_metrics(&returns); info!( " [DQN] Fold {} — Sharpe={:.4} MaxDD={:.2}% WinRate={:.1}% PF={:.2} Return={:.4}% Trades={}", window.fold, metrics.sharpe_ratio, metrics.max_drawdown_pct, metrics.win_rate_pct, metrics.profit_factor, metrics.total_return_pct, metrics.num_trades, ); info!( " [DQN] Actions — BUY={} SELL={} HOLD={}", action_counts.first().copied().unwrap_or(0), action_counts.get(1).copied().unwrap_or(0), action_counts.get(2).copied().unwrap_or(0), ); all_fold_metrics.push(FoldMetrics { fold: window.fold, model: "dqn".to_string(), sharpe_ratio: metrics.sharpe_ratio, max_drawdown_pct: metrics.max_drawdown_pct, win_rate_pct: metrics.win_rate_pct, profit_factor: metrics.profit_factor, total_return_pct: metrics.total_return_pct, num_trades: metrics.num_trades, test_start: test_start.clone(), test_end: test_end.clone(), }); all_action_counts.push(action_counts); } Err(e) => { error!(" [DQN] Fold {} evaluation failed: {}", window.fold, e); } } } // Evaluate PPO if eval_ppo { match evaluate_ppo_fold( window.fold, &test_norm, test_bars_aligned, &args.models_dir, &args, ) { Ok((returns, action_counts)) => { let metrics = compute_metrics(&returns); info!( " [PPO] Fold {} — Sharpe={:.4} MaxDD={:.2}% WinRate={:.1}% PF={:.2} Return={:.4}% Trades={}", window.fold, metrics.sharpe_ratio, metrics.max_drawdown_pct, metrics.win_rate_pct, metrics.profit_factor, metrics.total_return_pct, metrics.num_trades, ); info!( " [PPO] Actions — BUY={} SELL={} HOLD={}", action_counts.first().copied().unwrap_or(0), action_counts.get(1).copied().unwrap_or(0), action_counts.get(2).copied().unwrap_or(0), ); all_fold_metrics.push(FoldMetrics { fold: window.fold, model: "ppo".to_string(), sharpe_ratio: metrics.sharpe_ratio, max_drawdown_pct: metrics.max_drawdown_pct, win_rate_pct: metrics.win_rate_pct, profit_factor: metrics.profit_factor, total_return_pct: metrics.total_return_pct, num_trades: metrics.num_trades, test_start: test_start.clone(), test_end: test_end.clone(), }); all_action_counts.push(action_counts); } Err(e) => { error!(" [PPO] Fold {} evaluation failed: {}", window.fold, e); } } } } // 4. Compute aggregate metrics info!("Step 4/5: Computing aggregate metrics..."); let (dqn_avg_sharpe, dqn_avg_drawdown, dqn_avg_win_rate) = compute_aggregate(&all_fold_metrics, "dqn"); let (ppo_avg_sharpe, ppo_avg_drawdown, ppo_avg_win_rate) = compute_aggregate(&all_fold_metrics, "ppo"); let aggregate = AggregateMetrics { dqn_avg_sharpe, dqn_avg_drawdown, dqn_avg_win_rate, ppo_avg_sharpe, ppo_avg_drawdown, ppo_avg_win_rate, }; info!(" DQN — avg Sharpe={:.4} avg MaxDD={:.2}% avg WinRate={:.1}%", dqn_avg_sharpe, dqn_avg_drawdown, dqn_avg_win_rate); info!(" PPO — avg Sharpe={:.4} avg MaxDD={:.2}% avg WinRate={:.1}%", ppo_avg_sharpe, ppo_avg_drawdown, ppo_avg_win_rate); // 5. Sanity checks & report info!("Step 5/5: Running sanity checks and saving report..."); let sanity_checks = run_sanity_checks(&all_fold_metrics, &all_action_counts); info!(" Beats random: {}", sanity_checks.beats_random); info!(" Action diversity: {}", sanity_checks.action_diversity); info!(" Fold consistency: {}", sanity_checks.fold_consistency); let report = EvaluationReport { folds: all_fold_metrics, aggregate, sanity_checks, }; // Save report if let Some(parent) = args.output.parent() { std::fs::create_dir_all(parent) .with_context(|| format!("Failed to create output dir: {}", parent.display()))?; } let report_json = serde_json::to_string_pretty(&report) .context("Failed to serialize evaluation report")?; std::fs::write(&args.output, &report_json) .with_context(|| format!("Failed to write report to {}", args.output.display()))?; info!("=== Evaluation Complete ==="); info!(" Report saved to: {}", args.output.display()); info!(" Total fold evaluations: {}", report.folds.len()); Ok(()) }