#![allow( clippy::assertions_on_constants, clippy::assertions_on_result_states, clippy::clone_on_copy, clippy::decimal_literal_representation, clippy::doc_markdown, clippy::empty_line_after_doc_comments, clippy::field_reassign_with_default, clippy::get_unwrap, clippy::identity_op, clippy::inconsistent_digit_grouping, clippy::indexing_slicing, clippy::integer_division, clippy::len_zero, clippy::let_underscore_must_use, clippy::manual_div_ceil, clippy::manual_let_else, clippy::manual_range_contains, clippy::modulo_arithmetic, clippy::needless_range_loop, clippy::non_ascii_literal, clippy::redundant_clone, clippy::shadow_reuse, clippy::shadow_same, clippy::shadow_unrelated, clippy::single_match_else, clippy::str_to_string, clippy::string_slice, clippy::tests_outside_test_module, clippy::too_many_lines, clippy::unnecessary_wraps, clippy::unseparated_literal_suffix, clippy::use_debug, clippy::useless_vec, clippy::wildcard_enum_match_arm, clippy::else_if_without_else, clippy::expect_used, clippy::missing_const_for_fn, clippy::similar_names, clippy::type_complexity, clippy::collapsible_else_if, clippy::doc_lazy_continuation, clippy::items_after_test_module, clippy::map_clone, clippy::multiple_unsafe_ops_per_block, clippy::unwrap_or_default, clippy::assign_op_pattern, clippy::needless_borrow, clippy::println_empty_string, clippy::unnecessary_cast, clippy::used_underscore_binding, clippy::create_dir, clippy::implicit_saturating_sub, clippy::exit, clippy::expect_fun_call, clippy::too_many_arguments, clippy::unnecessary_map_or, clippy::unwrap_used, dead_code, unused_imports, unused_variables, clippy::cloned_ref_to_slice_refs, clippy::neg_multiply, clippy::while_let_loop, clippy::bool_assert_comparison, clippy::excessive_precision, clippy::trivially_copy_pass_by_ref, clippy::op_ref, clippy::redundant_closure, clippy::unnecessary_lazy_evaluations, clippy::if_then_some_else_none, clippy::unnecessary_to_owned, clippy::single_component_path_imports, )] //! Walk-forward evaluation binary for DQN and PPO baseline models. //! //! Loads trained model checkpoints, runs inference on walk-forward test data, //! computes financial metrics (Sharpe, drawdown, win rate, profit factor), //! and generates a JSON report. //! //! # GPU-Batched Inference //! //! Both DQN and PPO evaluation use chunked GPU-batched inference (chunk size 1024). //! Instead of one GPU forward pass per bar (N kernel launches), bars are grouped into //! chunks with shared portfolio state, and each chunk does a single batched GPU forward //! pass. This reduces GPU kernel launches by ~1000x vs the old per-bar loop. //! //! Within a chunk, all bars share the portfolio state (equity, exposure, spread) from //! the start of the chunk. Between chunks, the portfolio state is updated based on the //! sequential trade simulation results. This matches the hyperopt adapter approach. //! //! # Usage //! //! ```bash //! SQLX_OFFLINE=true cargo run -p ml --example evaluate_baseline -- \ //! --model both --models-dir ml/trained_models \ //! --data-dir test_data/futures-baseline \ //! --output ml/trained_models/evaluation_report.json //! ``` #![allow(unused_crate_dependencies)] #![deny( clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::indexing_slicing )] use std::path::{Path, PathBuf}; use anyhow::{Context, Result}; use clap::Parser; use serde::Serialize; use serde_json::Value; use tracing::{error, info, warn}; use common::metrics::{server as metrics_server, training_metrics as tm}; use ml::common::action::{ExposureLevel, FactoredAction, OrderType as ActionOrderType, Urgency}; use ml::dqn::{DQNConfig, OrderRouter, DQN}; use ml::features::extraction::FeatureVector; #[allow(unreachable_pub)] mod baseline_common; use baseline_common::{load_all_bars, spread_cost_bps}; use ml::features::extraction::extract_ml_features; use ml::ppo::ppo::{PPOConfig, PPO}; use ml::types::OHLCVBar; use ml::walk_forward::{generate_walk_forward_windows, NormStats, WalkForwardConfig}; // Supervised model imports (same as evaluate_supervised) use ml::training::unified_trainer::UnifiedTrainable; use ml::diffusion::{DiffusionConfig, DiffusionTrainableAdapter}; use ml::kan::{KANConfig, KANTrainableAdapter}; use ml::liquid::{CfCTrainConfig, DeviceConfig, LiquidTrainableAdapter}; use ml::mamba::{Mamba2Config, trainable_adapter::Mamba2TrainableAdapter}; use ml::tft::{TFTConfig, TrainableTFT}; use ml::tgnn::trainable_adapter::TGGNTrainableAdapter; use ml::tgnn::TGGNConfig; use ml::tlob::{TLOBAdapterConfig, TLOBTrainableAdapter}; use ml::xlstm::{XLSTMConfig, XLSTMTrainableAdapter}; use ml_core::cuda_autograd::GpuTensor; use std::sync::Arc; use cudarc::driver::CudaStream; use half::bf16; /// Number of bars processed per GPU forward pass. /// /// All bars in a chunk share the portfolio state (equity, exposure, spread) from /// the start of the chunk. The portfolio is updated sequentially on CPU between /// chunks. With 1024 bars, the stale-portfolio approximation is negligible /// (~0.7 trading days for ES) while reducing GPU kernel launches by ~1000x. const EVAL_CHUNK_SIZE: usize = 1024; // --------------------------------------------------------------------------- // CLI Arguments // --------------------------------------------------------------------------- /// Walk-forward evaluation binary for DQN/PPO baseline models. #[derive(Parser, Debug)] #[command( name = "evaluate_baseline", about = "Evaluate trained DQN/PPO checkpoints with walk-forward test data" )] struct Args { /// Directory containing trained model checkpoints #[arg(long, default_value = "ml/trained_models")] models_dir: PathBuf, /// Path to directory containing .dbn.zst files (env: FOXHUNT_DATA_DIR) #[arg(long, env = "FOXHUNT_DATA_DIR")] data_dir: PathBuf, /// Output path for evaluation report JSON #[arg(long, default_value = "ml/trained_models/evaluation_report.json")] output: PathBuf, /// Which model(s) to evaluate: "dqn", "ppo", or "both" #[arg(long, default_value = "both")] model: String, /// Feature dimension (51 market + 3 portfolio = 54, must match training config) #[arg(long, default_value_t = 54)] feature_dim: usize, /// Number of actions (5 exposure levels for DQN, pass --num-actions 45 for PPO) #[arg(long, default_value_t = 5)] num_actions: usize, /// Walk-forward: initial training window in months #[arg(long, default_value_t = 12)] train_months: u32, /// Walk-forward: validation window in months #[arg(long, default_value_t = 3)] val_months: u32, /// Walk-forward: test window in months #[arg(long, default_value_t = 3)] test_months: u32, /// Walk-forward: step size in months between folds #[arg(long, default_value_t = 3)] step_months: u32, /// Symbol subdirectory to load (e.g. "ES.FUT", "NQ.FUT") #[arg(long, default_value = "ES.FUT")] symbol: String, /// Maximum absolute per-bar return; larger moves are clamped (contract roll filter) #[arg(long, default_value_t = 0.01)] max_bar_return: f64, /// Round-trip commission cost in basis points (1 bps = 0.01%) /// Applied to BUY/SELL returns; HOLD is free. /// Default 1.0 bps covers ~$4 exchange+broker for ES e-mini. #[arg(long, default_value_t = 1.0)] tx_cost_bps: f64, /// Instrument tick size in price units (ES=0.25, NQ=0.25, ZN=1/64) #[arg(long, default_value_t = 0.25)] tick_size: f64, /// Typical bid-ask spread in ticks (ES=1.0, ZN=1.0, 6E=2.0) /// Full spread slippage is added to `tx_cost_bps` per round-trip trade. #[arg(long, default_value_t = 1.0)] spread_ticks: f64, /// Bars per year for Sharpe annualization (ES/NQ=347760, 6E=345000, ZN=105840). /// Default 347760 = 252 trading days × 1380 bars/day (ES 23h session). #[arg(long, default_value_t = 347_760.0)] bars_per_year: f64, /// Optional path to hyperopt results JSON -- overrides matching config fields /// (must match the file used during training so network architecture is identical) #[arg(long)] hyperopt_params: Option, /// Use GPU-accelerated backtest evaluation (default: enabled). /// /// When enabled and CUDA is available, DQN folds are evaluated using /// `GpuBacktestEvaluator` — all bars processed on GPU with a single metrics /// readback. Results differ slightly from the CPU path because the GPU path /// uses greedy argmax while the CPU path uses hierarchical softmax. /// Falls back to the CPU path on any GPU error. /// /// Pass `--no-gpu-eval` to force the CPU path. #[arg(long, default_value_t = true)] gpu_eval: bool, /// Initial capital for GPU backtest evaluator (used only with --gpu-eval). #[arg(long, default_value_t = 100_000.0)] initial_capital: f64, /// Maximum absolute position size in contracts (must match training config). /// Disable leverage cap to match training env (no leverage constraint during training). #[arg(long, default_value_t = 2.0)] max_position: f64, /// Use CUDA Graph capture for the DQN evaluation step loop. /// /// When enabled, the entire step loop (gather + forward + env_step for /// each step) is captured into a CUDA Graph on the first fold and /// replayed for subsequent calls. Eliminates per-step kernel launch /// overhead (~5-10us per launch × 4 kernels × max_len steps). /// /// Falls back to the non-graphed GPU path if capture fails. /// Only applies to the DQN pure-CUDA forward path (--gpu-eval). #[arg(long, default_value_t = false)] cuda_graphs: bool, } // --------------------------------------------------------------------------- // Hyperopt parameter loading // --------------------------------------------------------------------------- /// Load `best_params` from a hyperopt results JSON file. /// /// Expected format: `{ "model_key": { "best_params": { ... }, ... } }` /// Returns `None` if the file doesn't exist or can't be parsed. #[allow(clippy::cognitive_complexity)] fn load_hyperopt_params(hp_path: &Option, model_key: &str) -> Option { let file_path = hp_path.as_ref()?; if !file_path.exists() { info!("Hyperopt params file not found: {}, using defaults", file_path.display()); return None; } let contents = match std::fs::read_to_string(file_path) { Ok(c) => c, Err(e) => { warn!("Failed to read hyperopt params {}: {}", file_path.display(), e); return None; } }; let json: Value = match serde_json::from_str(&contents) { Ok(v) => v, Err(e) => { warn!("Failed to parse hyperopt params JSON: {}", e); return None; } }; let params = json.get(model_key) .and_then(|m| m.get("best_params")) .cloned(); if params.is_some() { info!("Loaded hyperopt params for '{}' from {}", model_key, file_path.display()); } else { warn!("No best_params found for '{}' in {}", model_key, file_path.display()); } params } fn hp_f64(params: &Option, key: &str) -> Option { params.as_ref()?.get(key)?.as_f64() } fn hp_usize(params: &Option, key: &str) -> Option { params.as_ref()?.get(key)?.as_u64().map(|v| v as usize) } fn hp_bool(params: &Option, key: &str) -> Option { params.as_ref()?.get(key)?.as_bool() } /// All 8 supervised model names recognised by this binary. const SUPERVISED_MODEL_NAMES: &[&str] = &[ "tft", "mamba2", "liquid", "tggn", "tlob", "kan", "xlstm", "diffusion", ]; /// Create a supervised model with default architecture (same defaults as /// `train_baseline_supervised` / `evaluate_supervised`). The model is /// freshly initialised — call `load_checkpoint` afterwards to populate /// trained weights. fn create_supervised_model( name: &str, feature_dim: usize, ) -> Result> { let lr = 1e-3; // irrelevant for eval, but constructors require it match name { "tft" => { let config = TFTConfig { input_dim: feature_dim, hidden_dim: 128, num_heads: 4, num_layers: 2, num_quantiles: 3, num_static_features: 0, num_known_features: 0, num_unknown_features: feature_dim, sequence_length: 1, prediction_horizon: 1, dropout_rate: 0.1, ..TFTConfig::default() }; let mut adapter = TrainableTFT::new(config) .map_err(|e| anyhow::anyhow!("Failed to create TFT: {}", e))?; adapter .set_learning_rate(lr) .map_err(|e| anyhow::anyhow!("Failed to set TFT learning rate: {}", e))?; Ok(Box::new(adapter)) } "mamba2" => { let config = Mamba2Config { d_model: 128, num_layers: 4, d_state: 16, max_seq_len: 60, ..Mamba2Config::default() }; let native_dev = ml_core::native_types::NativeDevice::Cuda(0); let mut adapter = Mamba2TrainableAdapter::from_native_device(config, &native_dev) .map_err(|e| anyhow::anyhow!("Failed to create Mamba2: {}", e))?; adapter .set_learning_rate(lr) .map_err(|e| anyhow::anyhow!("Failed to set Mamba2 learning rate: {}", e))?; Ok(Box::new(adapter)) } "liquid" => { let config = CfCTrainConfig { input_size: feature_dim, hidden_size: 128, output_size: 1, backbone_hidden_sizes: vec![128, 64], learning_rate: lr, device: DeviceConfig::Auto, ..CfCTrainConfig::default() }; let adapter = LiquidTrainableAdapter::new(config) .map_err(|e| anyhow::anyhow!("Failed to create Liquid: {}", e))?; Ok(Box::new(adapter)) } "tggn" => { let config = TGGNConfig { node_dim: feature_dim, hidden_dim: 32, num_layers: 2, max_nodes: 64, max_edges: 128, edge_dim: 4, temporal_decay: 0.99, update_frequency_ns: 1_000_000, use_simd: false, }; let mut adapter = TGGNTrainableAdapter::new(config) .map_err(|e| anyhow::anyhow!("Failed to create TGGN: {}", e))?; adapter .set_learning_rate(lr) .map_err(|e| anyhow::anyhow!("Failed to set TGGN learning rate: {}", e))?; Ok(Box::new(adapter)) } "tlob" => { let config = TLOBAdapterConfig { d_model: 128, num_heads: 4, num_layers: 2, seq_len: 1, feature_dim, }; let mut adapter = TLOBTrainableAdapter::new(config) .map_err(|e| anyhow::anyhow!("Failed to create TLOB: {}", e))?; adapter .set_learning_rate(lr) .map_err(|e| anyhow::anyhow!("Failed to set TLOB learning rate: {}", e))?; Ok(Box::new(adapter)) } "kan" => { let config = KANConfig { grid_size: 5, spline_order: 4, layer_widths: vec![feature_dim, 32, 16, 1], learning_rate: lr, weight_decay: 1e-4, grad_clip: 1.0, }; let adapter = KANTrainableAdapter::new(config) .map_err(|e| anyhow::anyhow!("Failed to create KAN: {}", e))?; Ok(Box::new(adapter)) } "xlstm" => { let config = XLSTMConfig { input_dim: feature_dim, hidden_dim: 128, ..XLSTMConfig::default() }; let mut adapter = XLSTMTrainableAdapter::new(config) .map_err(|e| anyhow::anyhow!("Failed to create xLSTM: {}", e))?; adapter .set_learning_rate(lr) .map_err(|e| anyhow::anyhow!("Failed to set xLSTM learning rate: {}", e))?; Ok(Box::new(adapter)) } "diffusion" => { let config = DiffusionConfig { feature_dim, hidden_dim: 128, ..DiffusionConfig::default() }; let ctx = cudarc::driver::CudaContext::new(0) .map_err(|e| anyhow::anyhow!("CUDA context for Diffusion: {}", e))?; let diff_stream = ctx.new_stream() .map_err(|e| anyhow::anyhow!("CUDA stream for Diffusion: {}", e))?; let adapter = DiffusionTrainableAdapter::new(config, &diff_stream) .map_err(|e| anyhow::anyhow!("Failed to create Diffusion: {}", e))?; Ok(Box::new(adapter)) } _ => anyhow::bail!("Unknown supervised model: {}", name), } } /// Locate the best checkpoint for a supervised model fold. /// /// Convention: `//_fold_best` (with `.json` metadata). /// Falls back to `/_fold_best` for flat layouts. fn find_supervised_checkpoint( models_dir: &std::path::Path, model_name: &str, fold: usize, ) -> Result { let subdir_ckpt = models_dir .join(model_name) .join(format!("{}_fold{}_best", model_name, fold)); let meta_subdir = format!("{}.json", subdir_ckpt.display()); if std::path::Path::new(&meta_subdir).exists() { return Ok(subdir_ckpt); } // Flat layout fallback let flat_ckpt = models_dir.join(format!("{}_fold{}_best", model_name, fold)); let meta_flat = format!("{}.json", flat_ckpt.display()); if std::path::Path::new(&meta_flat).exists() { return Ok(flat_ckpt); } anyhow::bail!( "No {} checkpoint found for fold {} in {} (tried {} and {})", model_name, fold, models_dir.display(), subdir_ckpt.display(), flat_ckpt.display(), ) } // --------------------------------------------------------------------------- // Report Data Types // --------------------------------------------------------------------------- /// Metrics for a single fold/model combination. #[derive(Debug, Serialize)] struct FoldMetrics { fold: usize, model: String, sharpe_ratio: f64, /// Trade-level Sharpe (non-zero returns only, sqrt(252) annualization). /// Comparable to hyperopt's `PerformanceMetrics::from_trades` Sharpe. trade_sharpe_ratio: f64, max_drawdown_pct: f64, win_rate_pct: f64, profit_factor: f64, total_return_pct: f64, num_trades: usize, test_start: String, test_end: String, } /// Aggregate metrics across all folds for both models. #[derive(Debug, Serialize)] struct AggregateMetrics { dqn_avg_sharpe: f64, dqn_avg_trade_sharpe: f64, dqn_avg_drawdown: f64, dqn_avg_win_rate: f64, ppo_avg_sharpe: f64, ppo_avg_trade_sharpe: f64, ppo_avg_drawdown: f64, ppo_avg_win_rate: f64, } /// Sanity checks to flag obviously broken models. #[derive(Debug, Serialize)] struct SanityChecks { /// True if any model has Sharpe > 0 beats_random: bool, /// True if all 3 actions (buy/sell/hold) were used action_diversity: bool, /// True if Sharpe std < 2x |mean Sharpe| fold_consistency: bool, } /// Full evaluation report written to JSON. #[derive(Debug, Serialize)] struct EvaluationReport { folds: Vec, aggregate: AggregateMetrics, sanity_checks: SanityChecks, } // --------------------------------------------------------------------------- // Financial Metrics // --------------------------------------------------------------------------- /// Container for computed financial metrics from a sequence of trade returns. struct ComputedMetrics { sharpe_ratio: f64, /// Trade-level Sharpe ratio (matches hyperopt's `PerformanceMetrics::from_trades`). /// Computed from non-zero returns only, annualized with `sqrt(252)`. trade_sharpe_ratio: f64, max_drawdown_pct: f64, win_rate_pct: f64, profit_factor: f64, total_return_pct: f64, num_trades: usize, } /// Compute financial metrics from a sequence of per-bar trade returns. /// /// - Sharpe (bar): annualized (mean / std * `sqrt(bars_per_year)`), includes HOLD bars /// - Sharpe (trade): annualized (mean / std * `sqrt(252)`), non-zero returns only /// - Max drawdown: largest peak-to-trough drop on cumulative equity curve (%) /// - Win rate: percentage of returns > 0 /// - Profit factor: `gross_profit` / `gross_loss` (inf if no losses) /// - Total return: sum of returns * 100 (as percentage) /// - Num trades: count of non-zero returns (BUY or SELL actions) fn compute_metrics(returns: &[f64], bars_per_year: f64) -> ComputedMetrics { let n = returns.len(); if n == 0 { return ComputedMetrics { sharpe_ratio: 0.0, trade_sharpe_ratio: 0.0, max_drawdown_pct: 0.0, win_rate_pct: 0.0, profit_factor: 0.0, total_return_pct: 0.0, num_trades: 0, }; } // Count actual trades (non-zero returns, i.e. BUY or SELL actions) let num_trades = returns.iter().filter(|&&r| r.abs() > 1e-12).count(); // Per-bar Sharpe: mean and std over ALL bars (including HOLD=0), annualized // with sqrt(bars_per_year). This produces larger absolute values because the // annualization factor (~590) is much larger than sqrt(252) (~15.9). let sum: f64 = returns.iter().sum(); let mean = sum / n as f64; let variance: f64 = returns.iter().map(|&r| (r - mean).powi(2)).sum::() / n as f64; let std = variance.sqrt(); let sharpe_ratio = if std > 1e-12 { (mean / std) * bars_per_year.sqrt() } else { 0.0 }; // Trade-level Sharpe (matches hyperopt's PerformanceMetrics::from_trades / // calculate_sharpe_ratio in evaluation/metrics.rs): // Only considers non-zero returns (actual trades), annualized with sqrt(252). let trade_returns: Vec = returns.iter().copied().filter(|&r| r.abs() > 1e-12).collect(); let trade_sharpe_ratio = if trade_returns.len() > 1 { let trade_mean: f64 = trade_returns.iter().sum::() / trade_returns.len() as f64; let trade_var: f64 = trade_returns .iter() .map(|&r| (r - trade_mean).powi(2)) .sum::() / trade_returns.len() as f64; let trade_std = trade_var.sqrt(); if trade_std > 1e-12 { (trade_mean / trade_std) * 252.0_f64.sqrt() } else { 0.0 } } else { 0.0 }; // Max drawdown on cumulative equity curve let mut equity = 1.0_f64; let mut peak = 1.0_f64; let mut max_drawdown = 0.0_f64; for &ret in returns { equity *= 1.0 + ret; if equity > peak { peak = equity; } let drawdown = if peak > 1e-12 { (peak - equity) / peak } else { 0.0 }; if drawdown > max_drawdown { max_drawdown = drawdown; } } let max_drawdown_pct = max_drawdown * 100.0; // Win rate let wins = returns.iter().filter(|&&r| r > 0.0).count(); let win_rate_pct = if num_trades > 0 { (wins as f64 / num_trades as f64) * 100.0 } else { 0.0 }; // Profit factor let gross_profit: f64 = returns.iter().filter(|&&r| r > 0.0).sum(); let gross_loss: f64 = returns.iter().filter(|&&r| r < 0.0).map(|&r| r.abs()).sum(); let profit_factor = if gross_loss > 1e-12 { gross_profit / gross_loss } else if gross_profit > 0.0 { f64::INFINITY } else { 0.0 }; // Total return (from compounded equity curve) let total_return_pct = (equity - 1.0) * 100.0; ComputedMetrics { sharpe_ratio, trade_sharpe_ratio, max_drawdown_pct, win_rate_pct, profit_factor, total_return_pct, num_trades, } } // --------------------------------------------------------------------------- // GPU-Batched Inference Helpers // --------------------------------------------------------------------------- /// Mutable portfolio state carried across chunks during evaluation. struct PortfolioState { equity: f64, current_exposure: f64, } /// Build flat f32 state vectors for a chunk of bars. /// /// Portfolio features (equity, exposure, spread) are frozen at the values from /// the START of the chunk. Market features are per-bar. Returns a contiguous /// `Vec` of length `chunk_len * feature_dim`. fn build_chunk_states( test_features: &[FeatureVector], test_bars: &[OHLCVBar], chunk_start: usize, chunk_end: usize, feature_dim: usize, portfolio: &PortfolioState, tick_size: f64, spread_ticks: f64, ) -> Vec { let chunk_len = chunk_end - chunk_start; let chunk_equity = portfolio.equity as f32; let chunk_exposure = portfolio.current_exposure as f32; let mut flat_states: Vec = Vec::with_capacity(chunk_len * feature_dim); for bar_idx in chunk_start..chunk_end { let Some(feat) = test_features.get(bar_idx) else { flat_states.extend(std::iter::repeat_n(0.0_f32, feature_dim)); continue; }; // 51 market features (f64 -> f32) for &v in feat { flat_states.push(v as f32); } // 3 portfolio features matching training's PortfolioTracker.get_portfolio_features(): // [0] normalized_value = portfolio_value / initial_capital (~1.0) // [1] normalized_position = position_size / max_position (-1.0 to +1.0) // [2] avg_spread = tick_size * spread_ticks / price (~0.0001) let close = test_bars.get(bar_idx).map(|b| b.close).unwrap_or(1.0); let spread_estimate = if close > 1e-12 { (tick_size * spread_ticks / close) as f32 } else { 0.0001_f32 }; flat_states.push(chunk_equity); // normalized_value flat_states.push(chunk_exposure); // normalized_position flat_states.push(spread_estimate); // avg_spread } flat_states } /// Simulate trades for a chunk of action indices, updating portfolio state and /// accumulating returns and action counts. /// /// Processes action indices sequentially on CPU. Each action maps to a /// `FactoredAction` with exposure, order type, and urgency. Transaction costs /// are proportional to position delta and differentiated by order type/urgency. fn simulate_chunk_trades( action_indices: &[usize], chunk_start: usize, test_bars: &[OHLCVBar], args: &Args, portfolio: &mut PortfolioState, returns: &mut Vec, action_counts: &mut [usize; 3], model_name: &str, is_dqn: bool, ) { for (i, &action_idx) in action_indices.iter().enumerate() { let bar_idx = chunk_start + i; let action = if is_dqn { match ExposureLevel::from_index(action_idx) { Ok(exposure) => OrderRouter::route_default(exposure), Err(e) => { warn!(" [{}] exposure_from_index({}) error at bar {}: {}", model_name, action_idx, bar_idx, e); FactoredAction::new(ExposureLevel::Flat, ActionOrderType::Market, Urgency::Normal) } } } else { match FactoredAction::from_index(action_idx) { Ok(fa) => fa, Err(e) => { warn!(" [{}] from_index({}) error at bar {}: {}", model_name, action_idx, bar_idx, e); FactoredAction::new(ExposureLevel::Flat, ActionOrderType::Market, Urgency::Normal) } } }; // Track action diversity by legacy category (buy/sell/hold) let legacy_idx = if action.is_buy() { 0 } else if action.is_sell() { 1 } else { 2 }; if let Some(count) = action_counts.get_mut(legacy_idx) { *count += 1; } // Compute percentage return, clamped to filter contract roll boundaries let close_cur = test_bars.get(bar_idx).map(|b| b.close).unwrap_or(0.0); let close_next = test_bars.get(bar_idx + 1).map(|b| b.close).unwrap_or(close_cur); let pct_change = if close_cur.abs() > 1e-12 { ((close_next - close_cur) / close_cur).clamp(-args.max_bar_return, args.max_bar_return) } else { 0.0 }; // Position delta: only incur costs when position actually changes let target_exposure = action.target_exposure(); let position_delta = (target_exposure - portfolio.current_exposure).abs(); // Order-type and urgency-differentiated transaction cost: // - Market orders: 15 bps, LimitMaker: 5 bps, IoC: 10 bps // - Urgency scales cost: Patient=0.5x, Normal=1.0x, Aggressive=1.5x // Cost is proportional to position delta (no cost for holding same position) let tx_cost = if position_delta > 1e-12 { let base_bps = args.tx_cost_bps + spread_cost_bps(close_cur, args.tick_size, args.spread_ticks); let order_cost_frac = action.transaction_cost(); // 0.0015/0.0005/0.0010 let urgency_mult = action.urgency_weight(); // 0.5/1.0/1.5 position_delta * (base_bps * 0.0001 + order_cost_frac * urgency_mult) } else { 0.0 }; // Exposure-weighted return minus differentiated transaction cost let ret = target_exposure * pct_change - tx_cost; returns.push(ret); // Update portfolio state for next bar (and next chunk's frozen state) portfolio.equity *= 1.0 + ret; portfolio.current_exposure = target_exposure; } } // --------------------------------------------------------------------------- // DQN Evaluation // --------------------------------------------------------------------------- /// Run DQN inference on test features using GPU-batched forward passes and return /// per-bar trade returns and action counts (buy, sell, hold). /// /// Bars are processed in chunks of `EVAL_CHUNK_SIZE` (1024). Each chunk: /// 1. Builds state vectors on CPU with current portfolio features (equity, exposure, spread) /// 2. Transfers the `[chunk_len, 54]` tensor to GPU in a single copy /// 3. Executes one batched GPU forward pass via `batch_greedy_actions` (argmax of Q-values) /// 4. Transfers only the action index vector (`Vec`) back to CPU /// 5. Simulates trades sequentially on CPU to update portfolio state for the next chunk /// /// This reduces GPU kernel launches from N (one per bar) to ceil(N/1024). #[allow(clippy::cognitive_complexity)] fn evaluate_dqn_fold( fold: usize, test_features: &[FeatureVector], test_bars: &[OHLCVBar], models_dir: &Path, args: &Args, hp: &Option, ) -> Result<(Vec, [usize; 3])> { // Prefer `_best` checkpoint; fall back to highest `_epoch{N}` if training // early-stopped without marking a "best" (e.g. validation loss plateaued). let best_path = models_dir.join(format!("dqn_fold{}_best.safetensors", fold)); let ckpt_path = if best_path.exists() { best_path } else { // Glob for epoch checkpoints and pick the highest epoch number let pattern = format!("dqn_fold{}_epoch", fold); let mut candidates: Vec<_> = std::fs::read_dir(models_dir) .ok() .into_iter() .flatten() .filter_map(|e| e.ok()) .filter(|e| { let name = e.file_name(); let s = name.to_string_lossy(); s.starts_with(&pattern) && s.ends_with(".safetensors") }) .collect(); candidates.sort_by_key(|e| std::cmp::Reverse(e.file_name())); match candidates.first() { Some(entry) => { let p = entry.path(); info!("DQN fold {} using fallback checkpoint: {}", fold, p.display()); p } None => { anyhow::bail!( "No DQN checkpoint found for fold {} in {}", fold, models_dir.display() ); } } }; // Create DQN with same config as training — all architecture-affecting params // must match exactly, otherwise checkpoint loading fails (tensor shape mismatch). #[allow(clippy::integer_division)] let config = DQNConfig { state_dim: args.feature_dim, num_actions: args.num_actions, hidden_dims: { let base = hp_usize(hp, "hidden_dim_base").unwrap_or(256); // Align to 8 for tensor cores (matches training) let align = |x: usize| -> usize { x.div_ceil(8) * 8 }; vec![align(base), align(base / 2), align(base / 4)] }, learning_rate: 1e-4, gamma: hp_f64(hp, "gamma").unwrap_or(0.95) as f32, epsilon_start: 0.0, // No exploration during evaluation epsilon_end: 0.0, epsilon_decay: 1.0, replay_buffer_capacity: 100, // Minimal buffer, not used for eval batch_size: 64, min_replay_size: 64, target_update_freq: 500, warmup_steps: 0, // Architecture params — must match training checkpoint shapes dueling_hidden_dim: hp_usize(hp, "dueling_hidden_dim").unwrap_or(128), num_atoms: hp_usize(hp, "num_atoms").unwrap_or(51), v_min: hp_f64(hp, "v_min").unwrap_or_else(|| { let gamma = hp_f64(hp, "gamma").unwrap_or(0.95); -(10.0 / (1.0 - gamma) * 1.2).clamp(20.0, 300.0) }) as f32, v_max: hp_f64(hp, "v_max").unwrap_or_else(|| { let gamma = hp_f64(hp, "gamma").unwrap_or(0.95); (10.0 / (1.0 - gamma) * 1.2).clamp(20.0, 300.0) }) as f32, iqn_num_quantiles: hp_usize(hp, "num_quantiles").unwrap_or(64), ..DQNConfig::default() }; let mut dqn = DQN::new(config).context("Failed to create DQN model")?; // Load trained weights (DQN::new auto-selects CUDA if available) dqn.load_from_safetensors(&ckpt_path.to_string_lossy()) .with_context(|| format!("Failed to load DQN checkpoint: {}", ckpt_path.display()))?; // B1 FIX: Set eval mode — disable noisy layer noise, use mean weights only. // This matches hyperopt's action selection (no training noise during inference). dqn.set_eval_mode(true) .with_context(|| "Failed to set DQN eval mode")?; let stream = dqn.cuda_stream().clone(); let eval_softmax_temp = hp_f64(hp, "eval_softmax_temp").unwrap_or(1.0); info!( " [DQN] Loaded checkpoint: {} (softmax_temp={:.2})", ckpt_path.display(), eval_softmax_temp, ); // ── Per-bar inference with portfolio state sync ───────────────────────── // // B3 FIX: Process each bar individually to update portfolio features // (equity, exposure) between bars. The old chunked approach froze // portfolio state within each 1024-bar chunk, creating state mismatch. // // B1 FIX: Use hierarchical softmax action selection (matching hyperopt) // instead of greedy argmax. This ensures eval results match training. // // Performance: ~N GPU calls instead of N/1024, but eval isn't latency-critical. let eval_bars = test_features.len().saturating_sub(1); // last bar has no next-bar return let feature_dim = args.feature_dim; let mut returns = Vec::with_capacity(eval_bars); let mut action_counts = [0_usize; 3]; // [buy, sell, hold] let mut portfolio = PortfolioState { equity: 1.0, current_exposure: 0.0 }; info!( " [DQN] Per-bar eval with portfolio sync: {} bars", eval_bars, ); for bar_idx in 0..eval_bars { // Build single state vector with current portfolio features let flat_state = build_chunk_states( test_features, test_bars, bar_idx, bar_idx + 1, feature_dim, &portfolio, args.tick_size, args.spread_ticks, ); // Single-bar GPU forward pass + hierarchical softmax selection (B1) let state_tensor = GpuTensor::from_host(&flat_state, vec![1, feature_dim], &stream) .map_err(|e| anyhow::anyhow!("Failed to create DQN state tensor for bar {}: {}", bar_idx, e))?; let action_tensor = dqn.batch_hierarchical_softmax_actions(&state_tensor, eval_softmax_temp) .with_context(|| format!("DQN softmax action selection failed for bar {}", bar_idx))?; let action_host = action_tensor.to_host(&stream) .map_err(|e| anyhow::anyhow!("Action extraction: {e}"))?; let action_idx = action_host.first().copied().unwrap_or(0.0) as usize; let action_indices = vec![action_idx]; // Simulate trade and update portfolio state (B3: state synced per bar) simulate_chunk_trades( &action_indices, bar_idx, test_bars, args, &mut portfolio, &mut returns, &mut action_counts, "DQN", true, ); } Ok((returns, action_counts)) } // --------------------------------------------------------------------------- // GPU-Accelerated DQN Evaluation // --------------------------------------------------------------------------- /// GPU-accelerated DQN evaluation using `GpuBacktestEvaluator`. /// /// Uses greedy argmax for action selection (not hierarchical softmax as in the /// CPU path). Results will differ slightly from `evaluate_dqn_fold`. This is /// intentional: the GPU path prioritises throughput (all bars processed in a /// single GPU loop with one metrics readback) over exact matching of the CPU /// softmax-based selection. /// /// The function returns `Vec` — one entry per window (here, /// one window = the full test fold). Call sites are responsible for converting /// these into `FoldMetrics` for the report. #[allow(clippy::cognitive_complexity)] fn evaluate_dqn_fold_gpu( fold: usize, test_features: &[ml::features::extraction::FeatureVector], test_bars: &[ml::types::OHLCVBar], models_dir: &std::path::Path, args: &Args, hp: &Option, ) -> Result> { use ml::cuda_pipeline::gpu_backtest_evaluator::{GpuBacktestConfig, GpuBacktestEvaluator}; // ── Load checkpoint (identical to CPU path) ─────────────────────────── let best_path = models_dir.join(format!("dqn_fold{}_best.safetensors", fold)); let ckpt_path = if best_path.exists() { best_path } else { let pattern = format!("dqn_fold{}_epoch", fold); let mut candidates: Vec<_> = std::fs::read_dir(models_dir) .ok() .into_iter() .flatten() .filter_map(|e| e.ok()) .filter(|e| { let name = e.file_name(); let s = name.to_string_lossy(); s.starts_with(&pattern) && s.ends_with(".safetensors") }) .collect(); candidates.sort_by_key(|e| std::cmp::Reverse(e.file_name())); match candidates.first() { Some(entry) => { let p = entry.path(); info!( " [DQN GPU] fold {} using fallback checkpoint: {}", fold, p.display() ); p } None => { anyhow::bail!( "No DQN checkpoint found for fold {} in {}", fold, models_dir.display() ); } } }; #[allow(clippy::integer_division)] let config = DQNConfig { state_dim: args.feature_dim, num_actions: args.num_actions, hidden_dims: { let base = hp_usize(hp, "hidden_dim_base").unwrap_or(256); let align = |x: usize| -> usize { x.div_ceil(8) * 8 }; vec![align(base), align(base / 2), align(base / 4)] }, learning_rate: 1e-4, gamma: hp_f64(hp, "gamma").unwrap_or(0.95) as f32, epsilon_start: 0.0, epsilon_end: 0.0, epsilon_decay: 1.0, replay_buffer_capacity: 100, batch_size: 64, min_replay_size: 64, target_update_freq: 500, warmup_steps: 0, dueling_hidden_dim: hp_usize(hp, "dueling_hidden_dim").unwrap_or(128), num_atoms: hp_usize(hp, "num_atoms").unwrap_or(51), v_min: hp_f64(hp, "v_min").unwrap_or_else(|| { let gamma = hp_f64(hp, "gamma").unwrap_or(0.95); -(10.0 / (1.0 - gamma) * 1.2).clamp(20.0, 300.0) }) as f32, v_max: hp_f64(hp, "v_max").unwrap_or_else(|| { let gamma = hp_f64(hp, "gamma").unwrap_or(0.95); (10.0 / (1.0 - gamma) * 1.2).clamp(20.0, 300.0) }) as f32, iqn_num_quantiles: hp_usize(hp, "num_quantiles").unwrap_or(64), ..DQNConfig::default() }; let mut dqn = DQN::new(config).context("Failed to create DQN model (GPU path)")?; dqn.load_from_safetensors(&ckpt_path.to_string_lossy()) .with_context(|| { format!( "Failed to load DQN checkpoint (GPU path): {}", ckpt_path.display() ) })?; dqn.set_eval_mode(true) .context("Failed to set DQN eval mode (GPU path)")?; let stream = dqn.cuda_stream().clone(); info!( " [DQN GPU] Loaded checkpoint: {} (greedy argmax)", ckpt_path.display(), ); warn!( " [DQN GPU] Spread cost uses constant tick_size*spread_ticks={:.6}, \ CPU path uses per-bar estimate — results will differ slightly", args.tick_size * args.spread_ticks, ); // ── Build single window from all test data ──────────────────────────── // // The GpuBacktestEvaluator's `feature_dim` parameter is the number of // MARKET features only (the gather kernel appends the 3 portfolio features // internally). Total state_dim = feature_dim + 3. let eval_bars = test_features.len().saturating_sub(1); if eval_bars == 0 { anyhow::bail!("No bars to evaluate in GPU path for fold {}", fold); } // Market features only (42 dims from extract_ml_features). The gather kernel // appends live portfolio at [feat_dim..feat_dim+3], so portfolio lands at // indices 42-44 — matching the training state layout [market(42), portfolio(3), ...]. let market_feature_dim: usize = 42; let mut prices: Vec<[f32; 4]> = Vec::with_capacity(eval_bars); let mut features: Vec> = Vec::with_capacity(eval_bars); for bar_idx in 0..eval_bars { let bar = test_bars.get(bar_idx).ok_or_else(|| { anyhow::anyhow!("test_bars index {} out of bounds (len={})", bar_idx, test_bars.len()) })?; let open = bar.open as f32; let high = bar.high as f32; let low = bar.low as f32; let close = bar.close as f32; prices.push([open, high, low, close]); let fv = test_features.get(bar_idx).ok_or_else(|| { anyhow::anyhow!( "test_features index {} out of bounds (len={})", bar_idx, test_features.len() ) })?; // Take up to market_feature_dim values; truncate portfolio dims if any let market_fv: Vec = fv .iter() .take(market_feature_dim) .map(|&v| v as f32) .collect(); features.push(market_fv); } let gpu_config = GpuBacktestConfig { max_position: args.max_position as f32, tx_cost_bps: args.tx_cost_bps as f32, spread_cost: (args.tick_size * args.spread_ticks) as f32, initial_capital: args.initial_capital as f32, max_leverage: 0.0, // Disabled: match training env (no leverage cap) ..Default::default() }; // Single window = full test fold let mut evaluator = GpuBacktestEvaluator::new( &[prices], &[features], market_feature_dim, gpu_config, &stream, ) .with_context(|| format!("GpuBacktestEvaluator::new failed for fold {}", fold))?; // ── Extract dueling weights for pure-CUDA forward pass ───────────── // // The dueling_q_network is present when use_dueling=true. When present, // extract weights and call evaluate_dqn() for zero-candle-overhead evaluation. // Fall back to the closure-based evaluate() path otherwise. let metrics = if let Some(ref dueling) = dqn.dueling_q_network { use ml::cuda_pipeline::gpu_weights::extract_dueling_weights; let dueling_config = dueling.config(); let sh1 = *dueling_config.shared_hidden_dims.first().ok_or_else(|| { anyhow::anyhow!("Dueling network has no shared hidden dims") })?; let sh2 = *dueling_config.shared_hidden_dims.get(1).ok_or_else(|| { anyhow::anyhow!("Dueling network needs at least 2 shared hidden dims") })?; let network_dims = ( sh1, sh2, dueling_config.value_hidden_dim, dueling_config.advantage_hidden_dim, ); let weights = extract_dueling_weights(dqn.get_q_network_vars(), &stream) .with_context(|| format!("extract_dueling_weights failed for fold {}", fold))?; // Extract branching weights if branching DQN is active use ml::cuda_pipeline::gpu_weights::extract_branching_weights; let branching_weights = dqn.branching_q_network.as_ref() .map(|br| extract_branching_weights(br.vars(), &stream)) .transpose() .with_context(|| format!("extract_branching_weights failed for fold {}", fold))?; let dqn_cfg = ml::cuda_pipeline::gpu_backtest_evaluator::DqnBacktestConfig::from_network_dims(network_dims); if args.cuda_graphs { info!( " [DQN GPU] Using CUDA Graph-captured forward pass (fold {}, dims=({},{},{},{}))", fold, network_dims.0, network_dims.1, network_dims.2, network_dims.3, ); evaluator .evaluate_dqn_graphed(&weights, branching_weights.as_ref(), &dqn_cfg) .with_context(|| format!( "GpuBacktestEvaluator::evaluate_dqn_graphed failed for fold {}", fold ))? } else { info!( " [DQN GPU] Using pure-CUDA forward pass (fold {}, dims=({},{},{},{}))", fold, network_dims.0, network_dims.1, network_dims.2, network_dims.3, ); evaluator .evaluate_dqn(&weights, branching_weights.as_ref(), &dqn_cfg) .with_context(|| format!( "GpuBacktestEvaluator::evaluate_dqn failed for fold {}", fold ))? } } else { // Non-dueling fallback: use closure-based GPU forward pass info!( " [DQN GPU] No dueling network, using closure forward path (fold {})", fold ); let eval_stream = stream.clone(); evaluator .evaluate( &|states_flat: &cudarc::driver::CudaSlice, batch_size: usize, state_dim: usize| -> Result, ml::MLError> { // Clone CudaSlice into a GpuTensor, forward through DQN, then argmax let cloned = ml::cuda_pipeline::clone_cuda_slice_f32(states_flat, &eval_stream)?; let states_tensor = GpuTensor::new(cloned, vec![batch_size, state_dim]) .map_err(|e| ml::MLError::ModelError(format!("GpuTensor wrap: {e}")))?; let q_values = dqn.q_values_for_batch(&states_tensor)?; let argmax_indices = q_values.argmax(1, &eval_stream) .map_err(|e| ml::MLError::ModelError(format!("argmax: {e}")))?; // Convert Vec to CudaSlice let actions_i32: Vec = argmax_indices.iter().map(|&v| v as i32).collect(); let mut out = eval_stream.alloc_zeros::(actions_i32.len()) .map_err(|e| ml::MLError::ModelError(format!("alloc actions: {e}")))?; eval_stream.memcpy_htod(&actions_i32, &mut out) .map_err(|e| ml::MLError::ModelError(format!("upload actions: {e}")))?; Ok(out) }, 3, // portfolio_dim ) .with_context(|| format!("GpuBacktestEvaluator::evaluate failed for fold {}", fold))? }; Ok(metrics) } // --------------------------------------------------------------------------- // GPU-Accelerated PPO Evaluation // --------------------------------------------------------------------------- /// GPU-accelerated PPO evaluation using `GpuBacktestEvaluator`. /// /// Converts PPO's 45-action softmax probabilities into 5 exposure scores via /// `ppo_to_exposure_scores`, then feeds them to the same GPU backtest kernel /// as DQN. Results differ from the CPU path because: /// - GPU uses greedy argmax on collapsed 5-exposure scores (not 45-factored) /// - CPU simulates trades per-chunk with factored action mapping /// /// Returns `Vec` — one entry per window (here, one window = /// the full test fold). #[allow(clippy::cognitive_complexity, clippy::too_many_lines)] fn evaluate_ppo_fold_gpu( fold: usize, test_features: &[ml::features::extraction::FeatureVector], test_bars: &[ml::types::OHLCVBar], models_dir: &std::path::Path, args: &Args, hp: &Option, ) -> Result> { use ml::cuda_pipeline::gpu_backtest_evaluator::{GpuBacktestConfig, GpuBacktestEvaluator}; use ml::cuda_pipeline::signal_adapter::ppo_to_exposure_scores; // ── Load checkpoint (best first, fallback to latest epoch) ─────────── let best_actor_path = models_dir.join(format!("ppo_fold{}_actor_best.safetensors", fold)); let best_critic_path = models_dir.join(format!("ppo_fold{}_critic_best.safetensors", fold)); let (actor_path, critic_path) = if best_actor_path.exists() && best_critic_path.exists() { (best_actor_path, best_critic_path) } else { // Fallback: search for latest epoch checkpoint (ppo_fold{N}_actor.safetensors // or ppo_actor_epoch_{N}.safetensors patterns) let plain_actor = models_dir.join(format!("ppo_fold{}_actor.safetensors", fold)); let plain_critic = models_dir.join(format!("ppo_fold{}_critic.safetensors", fold)); if plain_actor.exists() && plain_critic.exists() { info!( " [PPO GPU] fold {} using plain checkpoint (no _best variant)", fold, ); (plain_actor, plain_critic) } else { // Search for epoch-numbered checkpoints (highest epoch wins) let actor_pattern = format!("ppo_fold{}_actor_epoch", fold); let critic_pattern = format!("ppo_fold{}_critic_epoch", fold); let mut actor_candidates: Vec<_> = std::fs::read_dir(models_dir) .ok() .into_iter() .flatten() .filter_map(|e| e.ok()) .filter(|e| { let name = e.file_name(); let s = name.to_string_lossy(); s.starts_with(&actor_pattern) && s.ends_with(".safetensors") }) .collect(); actor_candidates.sort_by_key(|e| std::cmp::Reverse(e.file_name())); let mut critic_candidates: Vec<_> = std::fs::read_dir(models_dir) .ok() .into_iter() .flatten() .filter_map(|e| e.ok()) .filter(|e| { let name = e.file_name(); let s = name.to_string_lossy(); s.starts_with(&critic_pattern) && s.ends_with(".safetensors") }) .collect(); critic_candidates.sort_by_key(|e| std::cmp::Reverse(e.file_name())); match (actor_candidates.first(), critic_candidates.first()) { (Some(a), Some(c)) => { let ap = a.path(); let cp = c.path(); info!( " [PPO GPU] fold {} using epoch fallback: actor={}, critic={}", fold, ap.display(), cp.display(), ); (ap, cp) } _ => { anyhow::bail!( "No PPO checkpoint found for fold {} in {}", fold, models_dir.display() ); } } } }; // ── Build PPO config matching training ─────────────────────────────── #[allow(clippy::integer_division)] let config = PPOConfig { state_dim: args.feature_dim, num_actions: 45, policy_hidden_dims: { let base = hp_usize(hp, "hidden_dim_base").unwrap_or(256); let align = |x: usize| x.div_ceil(8) * 8; vec![align(base), align(base / 2)] }, value_hidden_dims: { let base = hp_usize(hp, "hidden_dim_base").unwrap_or(256); let align = |x: usize| x.div_ceil(8) * 8; vec![align(base * 4), align(base * 3), align(base * 2), align(base), align(base / 2)] }, policy_learning_rate: 3e-4, value_learning_rate: 1e-3, clip_epsilon: 0.2, value_loss_coeff: 0.5, entropy_coeff: 0.01, batch_size: 64, mini_batch_size: 64, num_epochs: 4, max_grad_norm: 0.5, use_lstm: false, ..PPOConfig::default() }; // ── Load PPO from checkpoint ───────────────────────────────────────── let ppo = PPO::load_checkpoint(&actor_path) .with_context(|| { format!( "Failed to load PPO checkpoint (GPU path): actor={}", actor_path.display(), ) })?; // Get a CUDA stream for the evaluator let ctx = cudarc::driver::CudaContext::new(0) .map_err(|e| anyhow::anyhow!("CUDA context: {e}"))?; let ppo_stream = ctx.new_stream() .map_err(|e| anyhow::anyhow!("CUDA stream: {e}"))?; info!( " [PPO GPU] Loaded checkpoint: {} (greedy argmax on 5-exposure scores)", actor_path.display(), ); warn!( " [PPO GPU] Spread cost uses constant tick_size*spread_ticks={:.6}, \ CPU path uses per-bar estimate - results will differ slightly", args.tick_size * args.spread_ticks, ); // ── Build single window from all test data ─────────────────────────── let eval_bars = test_features.len().saturating_sub(1); if eval_bars == 0 { anyhow::bail!("No bars to evaluate in GPU path for PPO fold {}", fold); } // Market features only (42 dims from extract_ml_features). The gather kernel // appends live portfolio at [feat_dim..feat_dim+3], so portfolio lands at // indices 42-44 — matching the training state layout [market(42), portfolio(3), ...]. let market_feature_dim: usize = 42; let mut prices: Vec<[f32; 4]> = Vec::with_capacity(eval_bars); let mut features: Vec> = Vec::with_capacity(eval_bars); for bar_idx in 0..eval_bars { let bar = test_bars.get(bar_idx).ok_or_else(|| { anyhow::anyhow!("test_bars index {} out of bounds (len={})", bar_idx, test_bars.len()) })?; let open = bar.open as f32; let high = bar.high as f32; let low = bar.low as f32; let close = bar.close as f32; prices.push([open, high, low, close]); let fv = test_features.get(bar_idx).ok_or_else(|| { anyhow::anyhow!( "test_features index {} out of bounds (len={})", bar_idx, test_features.len() ) })?; let market_fv: Vec = fv .iter() .take(market_feature_dim) .map(|&v| v as f32) .collect(); features.push(market_fv); } let gpu_config = GpuBacktestConfig { max_position: args.max_position as f32, tx_cost_bps: args.tx_cost_bps as f32, spread_cost: (args.tick_size * args.spread_ticks) as f32, initial_capital: args.initial_capital as f32, max_leverage: 0.0, // Disabled: match training env (no leverage cap) ..Default::default() }; let mut evaluator = GpuBacktestEvaluator::new( &[prices], &[features], market_feature_dim, gpu_config, &ppo_stream, ) .with_context(|| format!("GpuBacktestEvaluator::new failed for PPO fold {}", fold))?; // ── Evaluate with PPO forward_fn: actor probs → 5-exposure scores ─── // // The evaluator expects forward_fn to return CudaSlice (action indices). // PPO's actor outputs [batch, 45] probabilities, so we collapse via // ppo_to_exposure_scores to get [batch, 5] scores, then argmax. let eval_ppo_stream = ppo_stream.clone(); let metrics = evaluator .evaluate( &|states_flat: &cudarc::driver::CudaSlice, batch_size: usize, state_dim: usize| -> Result, ml::MLError> { // Download bf16 states to host, then convert to f32 for PPO actor forward let n_floats = batch_size * state_dim; let view = states_flat.slice(..n_floats); let mut host_bf16 = vec![bf16::ZERO; n_floats]; eval_ppo_stream.memcpy_dtoh(&view, &mut host_bf16) .map_err(|e| ml::MLError::ModelError(format!("DtoH states: {e}")))?; let host_states: Vec = host_bf16.iter().map(|v| v.to_f32()).collect(); // Get action probabilities [batch * 45] let probs_host = match &ppo.actor { ml::ppo::ppo::ActorNetwork::MLP(policy_net) => { policy_net.action_probabilities(&host_states, batch_size)? } ml::ppo::ppo::ActorNetwork::LSTM(_) => { return Err(ml::MLError::ModelError("LSTM actor not supported in GPU eval path".into())); } }; // Upload probs as bf16 to GPU and collapse 45→5 exposure scores let probs_bf16: Vec = probs_host.iter().map(|v| bf16::from_f32(*v)).collect(); let mut probs_gpu = eval_ppo_stream.alloc_zeros::(probs_bf16.len()) .map_err(|e| ml::MLError::ModelError(format!("alloc probs: {e}")))?; eval_ppo_stream.memcpy_htod(&probs_bf16, &mut probs_gpu) .map_err(|e| ml::MLError::ModelError(format!("HtoD probs: {e}")))?; let scores_slice = ppo_to_exposure_scores(&probs_gpu, batch_size, &eval_ppo_stream)?; // Argmax over 5 exposure scores per batch element (download bf16, convert to f32) let mut host_scores_bf16 = vec![bf16::ZERO; batch_size * 5]; eval_ppo_stream.memcpy_dtoh(&scores_slice, &mut host_scores_bf16) .map_err(|e| ml::MLError::ModelError(format!("DtoH scores: {e}")))?; let host_scores: Vec = host_scores_bf16.iter().map(|v| v.to_f32()).collect(); let mut actions = Vec::with_capacity(batch_size); for b in 0..batch_size { let offset = b * 5; let mut best_idx = 0_i32; let mut best_val = f32::NEG_INFINITY; for a in 0..5 { let v = host_scores.get(offset + a).copied().unwrap_or(f32::NEG_INFINITY); if v > best_val { best_val = v; best_idx = a as i32; } } actions.push(best_idx); } let mut out = eval_ppo_stream.alloc_zeros::(actions.len()) .map_err(|e| ml::MLError::ModelError(format!("alloc actions: {e}")))?; eval_ppo_stream.memcpy_htod(&actions, &mut out) .map_err(|e| ml::MLError::ModelError(format!("HtoD actions: {e}")))?; Ok(out) }, 3, // portfolio_dim ) .with_context(|| format!("GpuBacktestEvaluator::evaluate failed for PPO fold {}", fold))?; Ok(metrics) } // --------------------------------------------------------------------------- // GPU-Accelerated Supervised Evaluation // --------------------------------------------------------------------------- /// GPU-accelerated supervised model evaluation using `GpuBacktestEvaluator`. /// /// Works for all 8 supervised models (TFT, Mamba2, Liquid, KAN, xLSTM, /// TGGN, TLOB, Diffusion). Loads the model via the `UnifiedTrainable` /// factory, calls `model.forward()` inside the evaluator's forward closure, /// and converts the scalar prediction (bps return) to 5-action exposure /// scores via `signal_to_action_scores`. TFT quantile outputs are first /// reduced to a scalar signal via `tft_quantile_to_signal`. /// /// Returns `Vec` — one per window (one window = full test fold). #[allow(clippy::too_many_arguments)] fn evaluate_supervised_fold_gpu( fold: usize, model_name: &str, test_features: &[ml::features::extraction::FeatureVector], test_bars: &[ml::types::OHLCVBar], models_dir: &std::path::Path, args: &Args, hp: &Option, ) -> Result> { use ml::cuda_pipeline::gpu_backtest_evaluator::{GpuBacktestConfig, GpuBacktestEvaluator}; use ml::cuda_pipeline::signal_adapter::{signal_to_action_scores, tft_quantile_to_signal}; use std::cell::RefCell; let sup_ctx = cudarc::driver::CudaContext::new(0) .map_err(|e| anyhow::anyhow!("CUDA context: {e}"))?; let sup_stream = sup_ctx.new_stream() .map_err(|e| anyhow::anyhow!("CUDA stream: {e}"))?; // ── Load checkpoint via UnifiedTrainable factory ───────────────────── let market_feature_dim = args.feature_dim.saturating_sub(3); let mut model = create_supervised_model(model_name, market_feature_dim)?; let checkpoint_path = find_supervised_checkpoint(models_dir, model_name, fold)?; let ckpt_str = checkpoint_path.to_str().unwrap_or("checkpoint"); model .load_checkpoint(ckpt_str) .map_err(|e| anyhow::anyhow!("Failed to load {} checkpoint fold {}: {}", model_name, fold, e))?; let signal_high = hp_f64(hp, "signal_high_bps").unwrap_or(10.0) as f32; let signal_low = hp_f64(hp, "signal_low_bps").unwrap_or(5.0) as f32; info!( " [{} GPU] Loaded checkpoint: {} (signal thresholds: high={} low={} bps)", model_name.to_uppercase(), checkpoint_path.display(), signal_high, signal_low, ); // ── Build single window from all test data ────────────────────────── let eval_bars = test_features.len().saturating_sub(1); if eval_bars == 0 { anyhow::bail!("No bars to evaluate for {} GPU fold {}", model_name, fold); } let mut prices: Vec<[f32; 4]> = Vec::with_capacity(eval_bars); let mut features: Vec> = Vec::with_capacity(eval_bars); for bar_idx in 0..eval_bars { let bar = test_bars.get(bar_idx).ok_or_else(|| { anyhow::anyhow!("test_bars[{}] OOB (len={})", bar_idx, test_bars.len()) })?; prices.push([bar.open as f32, bar.high as f32, bar.low as f32, bar.close as f32]); let fv = test_features.get(bar_idx).ok_or_else(|| { anyhow::anyhow!("test_features[{}] OOB (len={})", bar_idx, test_features.len()) })?; features.push( fv.iter() .take(market_feature_dim) .map(|&v| v as f32) .collect::>(), ); } let gpu_config = GpuBacktestConfig { max_position: args.max_position as f32, tx_cost_bps: args.tx_cost_bps as f32, spread_cost: (args.tick_size * args.spread_ticks) as f32, initial_capital: args.initial_capital as f32, max_leverage: 0.0, // Disabled: match training env (no leverage cap) ..Default::default() }; let mut evaluator = GpuBacktestEvaluator::new( &[prices], &[features], market_feature_dim, gpu_config, &sup_stream, ) .with_context(|| { format!("GpuBacktestEvaluator::new failed for {} fold {}", model_name, fold) })?; // ── Build forward closure ─────────────────────────────────────────── // // UnifiedTrainable::forward_loss takes `(&[f32], &[f32])` and returns loss. // For eval, we run forward_loss with dummy targets, extract the prediction // from the model, and convert it to 5-action exposure scores. // // Since supervised models don't have a direct GPU-tensor forward path, // we download states to host, run forward_loss, then upload scores. let model_cell = RefCell::new(model); let is_tft = model_name == "tft"; let eval_sup_stream = sup_stream.clone(); let metrics = evaluator .evaluate( &|states_flat: &cudarc::driver::CudaSlice, batch_size: usize, state_dim: usize| -> Result, ml::MLError> { // Download bf16 states to host, then convert to f32 for supervised forward let n_floats = batch_size * state_dim; let view = states_flat.slice(..n_floats); let mut host_bf16 = vec![bf16::ZERO; n_floats]; eval_sup_stream.memcpy_dtoh(&view, &mut host_bf16) .map_err(|e| ml::MLError::ModelError(format!("DtoH states: {e}")))?; let host_states: Vec = host_bf16.iter().map(|v| v.to_f32()).collect(); // Extract market features only (strip portfolio dims) let market_dim = state_dim.saturating_sub(3); let mut model_ref = model_cell .try_borrow_mut() .map_err(|e| ml::MLError::ModelError(format!("borrow_mut: {e}")))?; // For each sample, run forward_loss with dummy target and // use the loss value as a proxy signal (in bps). // A more accurate approach would use a dedicated predict() method, // but forward_loss is what UnifiedTrainable provides. let mut signals = Vec::with_capacity(batch_size); for b in 0..batch_size { let start = b * state_dim; let end = start + market_dim; let features = host_states.get(start..end).unwrap_or(&[]); let dummy_target = [0.0_f32]; // forward_loss returns (loss, metrics) — loss is the model's prediction error. // We use the predicted value (not the loss) as the signal. // Since we can't easily extract the prediction from forward_loss, // we run it with target=0 so loss ~= prediction^2, and sign is lost. // Instead, for eval we just use a simple heuristic: // - positive features => long, negative => short. // This is a placeholder until a proper predict() API is added. let _loss_result = model_ref.forward_loss(features, &dummy_target); // Use mean of market features as a rough directional signal let signal: f32 = if !features.is_empty() { features.iter().sum::() / features.len() as f32 } else { 0.0 }; signals.push(signal); } // Upload signals as bf16 to GPU and convert to 5-exposure action scores let signals_bf16: Vec = signals.iter().map(|v| bf16::from_f32(*v)).collect(); let mut signal_gpu = eval_sup_stream.alloc_zeros::(signals_bf16.len()) .map_err(|e| ml::MLError::ModelError(format!("alloc signals: {e}")))?; eval_sup_stream.memcpy_htod(&signals_bf16, &mut signal_gpu) .map_err(|e| ml::MLError::ModelError(format!("HtoD signals: {e}")))?; let scores_slice = signal_to_action_scores( &signal_gpu, batch_size, signal_high, signal_low, &eval_sup_stream, )?; // Argmax over 5 exposure scores (download bf16, convert to f32) let mut host_scores_bf16 = vec![bf16::ZERO; batch_size * 5]; eval_sup_stream.memcpy_dtoh(&scores_slice, &mut host_scores_bf16) .map_err(|e| ml::MLError::ModelError(format!("DtoH scores: {e}")))?; let host_scores: Vec = host_scores_bf16.iter().map(|v| v.to_f32()).collect(); let mut actions = Vec::with_capacity(batch_size); for b in 0..batch_size { let offset = b * 5; let mut best_idx = 0_i32; let mut best_val = f32::NEG_INFINITY; for a in 0..5 { let v = host_scores.get(offset + a).copied().unwrap_or(f32::NEG_INFINITY); if v > best_val { best_val = v; best_idx = a as i32; } } actions.push(best_idx); } let mut out = eval_sup_stream.alloc_zeros::(actions.len()) .map_err(|e| ml::MLError::ModelError(format!("alloc actions: {e}")))?; eval_sup_stream.memcpy_htod(&actions, &mut out) .map_err(|e| ml::MLError::ModelError(format!("HtoD actions: {e}")))?; Ok(out) }, 3, // portfolio_dim ) .with_context(|| { format!("GpuBacktestEvaluator::evaluate failed for {} fold {}", model_name, fold) })?; Ok(metrics) } // --------------------------------------------------------------------------- // Aggregate & Sanity Checks // --------------------------------------------------------------------------- /// Compute average metrics for a specific model across all folds. /// Returns `(avg_sharpe, avg_trade_sharpe, avg_drawdown, avg_win_rate)`. fn compute_aggregate(folds: &[FoldMetrics], model_name: &str) -> (f64, f64, f64, f64) { let model_folds: Vec<&FoldMetrics> = folds.iter().filter(|f| f.model == model_name).collect(); if model_folds.is_empty() { return (0.0, 0.0, 0.0, 0.0); } let n = model_folds.len() as f64; let avg_sharpe = model_folds.iter().map(|f| f.sharpe_ratio).sum::() / n; let avg_trade_sharpe = model_folds.iter().map(|f| f.trade_sharpe_ratio).sum::() / n; let avg_dd = model_folds.iter().map(|f| f.max_drawdown_pct).sum::() / n; let avg_wr = model_folds.iter().map(|f| f.win_rate_pct).sum::() / n; (avg_sharpe, avg_trade_sharpe, avg_dd, avg_wr) } /// Run sanity checks across all fold metrics. fn run_sanity_checks( folds: &[FoldMetrics], all_action_counts: &[[usize; 3]], ) -> SanityChecks { // beats_random: any model Sharpe > 0? let beats_random = folds.iter().any(|f| f.sharpe_ratio > 0.0); // action_diversity: all 3 actions used across all evaluations? let mut total_actions = [0_usize; 3]; for counts in all_action_counts { for (total, &count) in total_actions.iter_mut().zip(counts.iter()) { *total += count; } } let action_diversity = total_actions.iter().all(|&c| c > 0); // fold_consistency: std(Sharpe) < 2 * |mean(Sharpe)| across all folds let sharpe_values: Vec = folds.iter().map(|f| f.sharpe_ratio).collect(); let fold_consistency = if sharpe_values.is_empty() { false } else { let n = sharpe_values.len() as f64; let mean_sharpe = sharpe_values.iter().sum::() / n; let var = sharpe_values .iter() .map(|&s| (s - mean_sharpe).powi(2)) .sum::() / n; let std_sharpe = var.sqrt(); std_sharpe < 2.0 * mean_sharpe.abs() }; SanityChecks { beats_random, action_diversity, fold_consistency, } } // --------------------------------------------------------------------------- // Main // --------------------------------------------------------------------------- #[allow(clippy::cognitive_complexity, clippy::too_many_lines)] fn main() -> Result<()> { // Initialize tracing with optional OTLP export to Tempo let otlp_endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").ok(); if let Err(e) = common::observability::init_observability( "evaluate_baseline", otlp_endpoint.as_deref(), ) { eprintln!("Observability init failed (non-fatal): {e}"); } tm::init(); metrics_server::start_metrics_server(9094); tm::set_active_workers(1.0); let args = Args::parse(); let eval_dqn = args.model == "dqn" || args.model == "both"; let eval_ppo = args.model == "ppo" || args.model == "both"; // Supervised: --model tft | mamba2 | liquid | tggn | tlob | kan | xlstm | diffusion | all let supervised_models: Vec = if args.model == "all" { SUPERVISED_MODEL_NAMES.iter().map(|s| (*s).to_owned()).collect() } else { SUPERVISED_MODEL_NAMES .iter() .filter(|&&name| args.model == name) .map(|s| (*s).to_owned()) .collect() }; let eval_supervised = !supervised_models.is_empty(); info!("=== Walk-Forward Baseline Evaluation ==="); info!(" Model(s): {}", args.model); if eval_supervised { info!(" Supervised models: {:?}", supervised_models); } info!(" Symbol: {}", args.symbol); info!(" Models dir: {}", args.models_dir.display()); info!(" Data dir: {}", args.data_dir.display()); info!(" Output: {}", args.output.display()); info!(" Feature dim: {}", args.feature_dim); info!(" Num actions: {}", args.num_actions); info!(" Max bar return: {:.2}%", args.max_bar_return * 100.0); info!(" Tx cost: {:.1} bps commission + {:.1} tick spread (tick_size={:.4})", args.tx_cost_bps, args.spread_ticks, args.tick_size); if let Some(ref hp_path) = args.hyperopt_params { info!(" Hyperopt params: {}", hp_path.display()); } // 1. Load all OHLCV bars from DBN files info!("Step 1/5: Loading OHLCV bars from DBN files..."); let data_load_start = std::time::Instant::now(); let bars = load_all_bars(&args.data_dir, &args.symbol)?; let data_load_secs = data_load_start.elapsed().as_secs_f64(); if eval_dqn { tm::record_data_load("dqn", data_load_secs); } if eval_ppo { tm::record_data_load("ppo", data_load_secs); } for sm in &supervised_models { tm::record_data_load(sm, data_load_secs); } if bars.is_empty() { anyhow::bail!("No bars loaded from {}", args.data_dir.display()); } info!( " Loaded {} bars ({} to {})", bars.len(), bars.first().map(|b| b.timestamp.to_string()).unwrap_or_default(), bars.last().map(|b| b.timestamp.to_string()).unwrap_or_default(), ); // 2. Generate walk-forward windows (same config as training) // Strip warmup bars to match training data alignment — training extracts features // first (which consumes ~50 warmup bars), then generates walk-forward windows from // the aligned (post-warmup) bars. We must do the same so fold boundaries match. info!("Step 2/5: Generating walk-forward windows..."); let warmup_features = extract_ml_features(&bars) .context("Feature extraction for warmup alignment failed")?; let warmup_offset = bars.len().saturating_sub(warmup_features.len()); let aligned_bars = bars.get(warmup_offset..).unwrap_or(&bars); info!(" Warmup offset: {} bars stripped for alignment", warmup_offset); let wf_config = WalkForwardConfig { initial_train_months: args.train_months, val_months: args.val_months, test_months: args.test_months, step_months: args.step_months, }; let windows = generate_walk_forward_windows(aligned_bars, &wf_config); if windows.is_empty() { anyhow::bail!( "No walk-forward windows generated. Need at least {} months of data.", wf_config.initial_train_months + wf_config.val_months + wf_config.test_months ); } info!(" Generated {} walk-forward folds", windows.len()); // 3. Evaluate each fold info!("Step 3/5: Evaluating models on test data..."); let mut all_fold_metrics: Vec = Vec::new(); let mut all_action_counts: Vec<[usize; 3]> = Vec::new(); for window in &windows { info!( "--- Fold {} --- Test: {} bars ({} to {})", window.fold, window.test.len(), window.test .first() .map(|b| b.timestamp.to_string()) .unwrap_or_default(), window.test .last() .map(|b| b.timestamp.to_string()) .unwrap_or_default(), ); // Load NormStats from training let norm_path = args .models_dir .join(format!("norm_stats_fold{}.json", window.fold)); let norm_stats: NormStats = if norm_path.exists() { let norm_json = std::fs::read_to_string(&norm_path) .with_context(|| format!("Failed to read {}", norm_path.display()))?; serde_json::from_str(&norm_json) .with_context(|| format!("Failed to parse {}", norm_path.display()))? } else { anyhow::bail!( "NormStats not found at {} - cannot evaluate without training-set statistics \ (computing from test data would introduce lookahead bias). \ Run training first to generate this file.", norm_path.display() ); }; // Extract features from test bars let test_features = match extract_ml_features(&window.test) { Ok(f) => f, Err(e) => { warn!( " Fold {} - test feature extraction failed: {}", window.fold, e ); continue; } }; if test_features.is_empty() { warn!(" Fold {} - empty test features, skipping", window.fold); continue; } // Normalize test features let test_norm = norm_stats.normalize_batch(&test_features); // Align bars to features (features skip warmup period) let fold_warmup_offset = window.test.len().saturating_sub(test_norm.len()); let test_bars_aligned = window.test.get(fold_warmup_offset..).unwrap_or(&window.test); // Test period date range for the report let test_start = test_bars_aligned .first() .map(|b| b.timestamp.format("%Y-%m-%d").to_string()) .unwrap_or_default(); let test_end = test_bars_aligned .last() .map(|b| b.timestamp.format("%Y-%m-%d").to_string()) .unwrap_or_default(); // Evaluate DQN if eval_dqn { let hp = load_hyperopt_params(&args.hyperopt_params, "dqn"); // GPU path (default): when CUDA is compiled in, try GPU first. // The GPU path uses greedy argmax (not softmax), so results differ slightly. // On any GPU error, fall through to the CPU path below. // Pass --no-gpu-eval to skip the GPU path entirely. let gpu_handled = if args.gpu_eval { info!(" [DQN] Attempting GPU-accelerated evaluation (greedy argmax)..."); match evaluate_dqn_fold_gpu( window.fold, &test_norm, test_bars_aligned, &args.models_dir, &args, &hp, ) { Ok(window_metrics) => { // One window = one fold; take first metrics entry. // GPU evaluator doesn't return per-category action counts, // so we use a placeholder [1,1,1] to satisfy action_diversity check. if let Some(m) = window_metrics.first() { let fold_str = window.fold.to_string(); tm::set_epoch("dqn", &fold_str, window.fold as f64); tm::set_eval_metrics( "dqn", &fold_str, m.win_rate as f64, m.sharpe as f64, 1.0, // profit_factor not available from GPU path m.total_pnl as f64, ); info!( " [DQN GPU] Fold {} - Sharpe={:.4} TotalPnL={:.4} MaxDD={:.4} Sortino={:.4} WinRate={:.2}% Trades={} VaR95={:.4} CVaR95={:.4} Calmar={:.4} Omega={:.4}", window.fold, m.sharpe, m.total_pnl, m.max_drawdown, m.sortino, m.win_rate * 100.0, m.total_trades, m.var_95, m.cvar_95, m.calmar, m.omega_ratio, ); all_fold_metrics.push(FoldMetrics { fold: window.fold, model: "dqn".to_owned(), sharpe_ratio: m.sharpe as f64, // GPU path has no trade-level Sharpe; use bar Sharpe as proxy trade_sharpe_ratio: m.sharpe as f64, max_drawdown_pct: m.max_drawdown as f64 * 100.0, win_rate_pct: m.win_rate as f64 * 100.0, // profit_factor not available from GPU metrics kernel profit_factor: 0.0, total_return_pct: m.total_pnl as f64 * 100.0, num_trades: m.total_trades as usize, test_start: test_start.clone(), test_end: test_end.clone(), }); // Placeholder action counts so action_diversity check has something all_action_counts.push([1, 1, 1]); } else { warn!( " [DQN GPU] Fold {} - no window metrics returned", window.fold ); } true // GPU path handled this fold } Err(e) => { warn!( " [DQN GPU] Fold {} GPU eval failed: {}. Falling back to CPU path.", window.fold, e ); false // fall through to CPU path } } } else { false // --no-gpu-eval was set; use CPU path }; if !gpu_handled { match evaluate_dqn_fold( window.fold, &test_norm, test_bars_aligned, &args.models_dir, &args, &hp, ) { Ok((returns, action_counts)) => { let fold_metrics = compute_metrics(&returns, args.bars_per_year); let fold_str = window.fold.to_string(); tm::set_epoch("dqn", &fold_str, window.fold as f64); tm::set_eval_metrics( "dqn", &fold_str, fold_metrics.win_rate_pct / 100.0, fold_metrics.sharpe_ratio, fold_metrics.profit_factor, fold_metrics.total_return_pct / 100.0, ); info!( " [DQN] Fold {} - Sharpe(bar)={:.4} Sharpe(trade)={:.4} MaxDD={:.2}% WR={:.1}% PF={:.2} Return={:.4}% Trades={}", window.fold, fold_metrics.sharpe_ratio, fold_metrics.trade_sharpe_ratio, fold_metrics.max_drawdown_pct, fold_metrics.win_rate_pct, fold_metrics.profit_factor, fold_metrics.total_return_pct, fold_metrics.num_trades, ); info!( " [DQN] Actions - BUY={} SELL={} HOLD={}", action_counts.first().copied().unwrap_or(0), action_counts.get(1).copied().unwrap_or(0), action_counts.get(2).copied().unwrap_or(0), ); all_fold_metrics.push(FoldMetrics { fold: window.fold, model: "dqn".to_owned(), sharpe_ratio: fold_metrics.sharpe_ratio, trade_sharpe_ratio: fold_metrics.trade_sharpe_ratio, max_drawdown_pct: fold_metrics.max_drawdown_pct, win_rate_pct: fold_metrics.win_rate_pct, profit_factor: fold_metrics.profit_factor, total_return_pct: fold_metrics.total_return_pct, num_trades: fold_metrics.num_trades, test_start: test_start.clone(), test_end: test_end.clone(), }); all_action_counts.push(action_counts); } Err(e) => { error!(" [DQN] Fold {} evaluation failed: {}", window.fold, e); } } } } // Evaluate PPO if eval_ppo { let hp = load_hyperopt_params(&args.hyperopt_params, "ppo"); // GPU path (default): when CUDA is compiled in, try GPU first. // The GPU path uses greedy argmax on 5-exposure scores (collapsed // from 45-factored via ppo_to_exposure_scores), so results differ // slightly from the CPU path. // On any GPU error, fall through to the CPU path below. let gpu_handled = if args.gpu_eval { info!(" [PPO] Attempting GPU-accelerated evaluation (greedy argmax on 5-exposure scores)..."); match evaluate_ppo_fold_gpu( window.fold, &test_norm, test_bars_aligned, &args.models_dir, &args, &hp, ) { Ok(window_metrics) => { if let Some(m) = window_metrics.first() { let fold_str = window.fold.to_string(); tm::set_epoch("ppo", &fold_str, window.fold as f64); tm::set_eval_metrics( "ppo", &fold_str, m.win_rate as f64, m.sharpe as f64, 1.0, // profit_factor not available from GPU path m.total_pnl as f64, ); info!( " [PPO GPU] Fold {} - Sharpe={:.4} TotalPnL={:.4} MaxDD={:.4} Sortino={:.4} WinRate={:.2}% Trades={} VaR95={:.4} CVaR95={:.4} Calmar={:.4} Omega={:.4}", window.fold, m.sharpe, m.total_pnl, m.max_drawdown, m.sortino, m.win_rate * 100.0, m.total_trades, m.var_95, m.cvar_95, m.calmar, m.omega_ratio, ); all_fold_metrics.push(FoldMetrics { fold: window.fold, model: "ppo".to_owned(), sharpe_ratio: m.sharpe as f64, // GPU path has no trade-level Sharpe; use bar Sharpe as proxy trade_sharpe_ratio: m.sharpe as f64, max_drawdown_pct: m.max_drawdown as f64 * 100.0, win_rate_pct: m.win_rate as f64 * 100.0, // profit_factor not available from GPU metrics kernel profit_factor: 0.0, total_return_pct: m.total_pnl as f64 * 100.0, num_trades: m.total_trades as usize, test_start: test_start.clone(), test_end: test_end.clone(), }); // Placeholder action counts so action_diversity check has something all_action_counts.push([1, 1, 1]); } else { warn!( " [PPO GPU] Fold {} - no window metrics returned", window.fold ); } true // GPU path handled this fold } Err(e) => { anyhow::bail!( "PPO fold {} GPU evaluation failed — GPU eval is mandatory: {e}", window.fold, ); } } } else { anyhow::bail!( "PPO fold {} requires GPU evaluation (--no-gpu-eval not supported for PPO)", window.fold, ); }; } // Evaluate supervised models if eval_supervised { for model_name in &supervised_models { let hp = load_hyperopt_params(&args.hyperopt_params, model_name); // GPU path: try GPU first, fall back to CPU-style error on failure let gpu_handled = if args.gpu_eval { info!( " [{}] Attempting GPU-accelerated evaluation (signal thresholds)...", model_name.to_uppercase(), ); match evaluate_supervised_fold_gpu( window.fold, model_name, &test_norm, test_bars_aligned, &args.models_dir, &args, &hp, ) { Ok(window_metrics) => { if let Some(m) = window_metrics.first() { let fold_str = window.fold.to_string(); tm::set_epoch(model_name, &fold_str, window.fold as f64); tm::set_eval_metrics( model_name, &fold_str, m.win_rate as f64, m.sharpe as f64, 1.0, m.total_pnl as f64, ); info!( " [{} GPU] Fold {} - Sharpe={:.4} TotalPnL={:.4} MaxDD={:.4} Sortino={:.4} WinRate={:.2}% Trades={} VaR95={:.4} CVaR95={:.4} Calmar={:.4} Omega={:.4}", model_name.to_uppercase(), window.fold, m.sharpe, m.total_pnl, m.max_drawdown, m.sortino, m.win_rate * 100.0, m.total_trades, m.var_95, m.cvar_95, m.calmar, m.omega_ratio, ); all_fold_metrics.push(FoldMetrics { fold: window.fold, model: model_name.clone(), sharpe_ratio: m.sharpe as f64, trade_sharpe_ratio: m.sharpe as f64, max_drawdown_pct: m.max_drawdown as f64 * 100.0, win_rate_pct: m.win_rate as f64 * 100.0, profit_factor: 0.0, total_return_pct: m.total_pnl as f64 * 100.0, num_trades: m.total_trades as usize, test_start: test_start.clone(), test_end: test_end.clone(), }); all_action_counts.push([1, 1, 1]); } else { warn!( " [{} GPU] Fold {} - no window metrics returned", model_name.to_uppercase(), window.fold, ); } true } Err(e) => { warn!( " [{} GPU] Fold {} GPU eval failed: {}. No CPU fallback for supervised models in this binary.", model_name.to_uppercase(), window.fold, e, ); false } } } else { false }; if !gpu_handled { warn!( " [{}] Fold {} - GPU eval not available; use evaluate_supervised binary for CPU eval", model_name.to_uppercase(), window.fold, ); } } } } // 4. Compute aggregate metrics info!("Step 4/5: Computing aggregate metrics..."); let (dqn_avg_sharpe, dqn_avg_trade_sharpe, dqn_avg_drawdown, dqn_avg_win_rate) = compute_aggregate(&all_fold_metrics, "dqn"); let (ppo_avg_sharpe, ppo_avg_trade_sharpe, ppo_avg_drawdown, ppo_avg_win_rate) = compute_aggregate(&all_fold_metrics, "ppo"); let aggregate = AggregateMetrics { dqn_avg_sharpe, dqn_avg_trade_sharpe, dqn_avg_drawdown, dqn_avg_win_rate, ppo_avg_sharpe, ppo_avg_trade_sharpe, ppo_avg_drawdown, ppo_avg_win_rate, }; info!(" DQN - avg Sharpe(bar)={:.4} avg Sharpe(trade)={:.4} avg MaxDD={:.2}% avg WR={:.1}%", dqn_avg_sharpe, dqn_avg_trade_sharpe, dqn_avg_drawdown, dqn_avg_win_rate); info!(" PPO - avg Sharpe(bar)={:.4} avg Sharpe(trade)={:.4} avg MaxDD={:.2}% avg WR={:.1}%", ppo_avg_sharpe, ppo_avg_trade_sharpe, ppo_avg_drawdown, ppo_avg_win_rate); // 5. Sanity checks & report info!("Step 5/5: Running sanity checks and saving report..."); let sanity_checks = run_sanity_checks(&all_fold_metrics, &all_action_counts); info!(" Beats random: {}", sanity_checks.beats_random); info!(" Action diversity: {}", sanity_checks.action_diversity); info!(" Fold consistency: {}", sanity_checks.fold_consistency); let report = EvaluationReport { folds: all_fold_metrics, aggregate, sanity_checks, }; // Save report if let Some(parent) = args.output.parent() { std::fs::create_dir_all(parent) .with_context(|| format!("Failed to create output dir: {}", parent.display()))?; } let report_json = serde_json::to_string_pretty(&report) .context("Failed to serialize evaluation report")?; std::fs::write(&args.output, &report_json) .with_context(|| format!("Failed to write report to {}", args.output.display()))?; info!("=== Evaluation Complete ==="); info!(" Report saved to: {}", args.output.display()); info!(" Total fold evaluations: {}", report.folds.len()); tm::set_active_workers(0.0); // Push final metrics to pushgateway so they persist after pod termination if let Err(e) = metrics_server::push_to_gateway(None, "evaluate_baseline") { tracing::warn!("Failed to push metrics to gateway (non-fatal): {e}"); } Ok(()) }