diff --git a/crates/ml/examples/train_baseline_rl.rs b/crates/ml/examples/train_baseline_rl.rs index f220f9dff..c6d7fed94 100644 --- a/crates/ml/examples/train_baseline_rl.rs +++ b/crates/ml/examples/train_baseline_rl.rs @@ -134,7 +134,7 @@ struct Args { epochs: usize, /// Training batch size - #[arg(long, default_value_t = 128)] + #[arg(long, default_value_t = 0)] batch_size: usize, /// Path to directory containing .dbn.zst files (env: FOXHUNT_DATA_DIR) @@ -519,7 +519,7 @@ fn train_dqn_fold( epsilon_end: hp_f64(hp, "epsilon_end").unwrap_or(0.01), epsilon_decay: hp_f64(hp, "epsilon_decay").unwrap_or(0.995), buffer_size: hp_usize(hp, "buffer_size").unwrap_or(gpu_profile.training.buffer_size), - min_replay_size: hp_usize(hp, "batch_size").unwrap_or(args.batch_size), + min_replay_size: hp_usize(hp, "min_replay_size").unwrap_or(1000), epochs: args.epochs, checkpoint_frequency: 10, hidden_dim_base: dqn_hidden_base, @@ -575,7 +575,12 @@ fn train_dqn_fold( profile.apply_to(&mut hyperparams); // CLI args override profile: re-apply any arg that the user can set explicitly. hyperparams.epochs = args.epochs; - hyperparams.batch_size = hp_usize(hp, "batch_size").unwrap_or(args.batch_size); + // Only override batch_size from CLI/hyperopt if explicitly non-zero. + // batch_size=0 is the auto-compute sentinel — let the constructor handle it. + let hp_batch = hp_usize(hp, "batch_size").unwrap_or(args.batch_size); + if hp_batch > 0 { + hyperparams.batch_size = hp_batch; + } hyperparams.learning_rate = hp_f64(hp, "learning_rate").unwrap_or(args.learning_rate); hyperparams.max_training_steps_per_epoch = args.max_steps_per_epoch; hyperparams.initial_capital = args.initial_capital as f32; diff --git a/crates/ml/src/benchmark/dqn_benchmark.rs b/crates/ml/src/benchmark/dqn_benchmark.rs index 3b4cad763..98b04f2cd 100644 --- a/crates/ml/src/benchmark/dqn_benchmark.rs +++ b/crates/ml/src/benchmark/dqn_benchmark.rs @@ -449,8 +449,8 @@ impl DqnBenchmarkRunner { noisy_sigma_init: 0.5, // BUG #37 FIX: Q-value clipping (prevents step-level explosions) - q_value_clip_min: -500.0, - q_value_clip_max: 500.0, + q_value_clip_min: -50.0, + q_value_clip_max: 50.0, // WAVE 23 P0 Fix #1: Adaptive gradient collapse threshold gradient_collapse_multiplier: 100.0, diff --git a/crates/ml/src/trainers/dqn/financials.rs b/crates/ml/src/trainers/dqn/financials.rs index b6149cbec..355e1b29b 100644 --- a/crates/ml/src/trainers/dqn/financials.rs +++ b/crates/ml/src/trainers/dqn/financials.rs @@ -74,31 +74,44 @@ pub(crate) fn compute_epoch_financials( 0.0 }; - // Sharpe ratio from step_returns (per-bar, annualized) - let annualization = bars_per_year.sqrt(); + // Sharpe ratio from PER-TRADE returns (not per-bar). + // Per-bar step_returns collapse variance when most bars are flat, + // producing bogus Sharpe (e.g., 19.75 with PF=0.03). Per-trade + // returns use the GPU-aggregated sum_returns / sum_sq_returns. + // + // Annualization: sqrt(trades_per_year) where trades_per_year is + // estimated from the epoch's trade frequency. + let n_trades = total_trades as f64; + let n_bars = trade_stats.step_returns.len() as f64; + let trades_per_year = if n_bars > 0.0 { + (n_trades / n_bars) * bars_per_year + } else { + 0.0 + }; + let trade_annualization = trades_per_year.max(1.0).sqrt(); - let returns = &trade_stats.step_returns; - let n = returns.len(); - let (sharpe, sortino) = if n > 1 { - let mean: f64 = returns.iter().sum::() / n as f64; - let variance: f64 = returns.iter().map(|r| (r - mean).powi(2)).sum::() / n as f64; - let std = variance.sqrt(); + let (sharpe, sortino) = if total_trades > 1 { + // Per-trade mean and variance from GPU-aggregated sums + let mean = trade_stats.sum_returns / n_trades; + let var = (trade_stats.sum_sq_returns / n_trades) - mean * mean; + let std = var.max(0.0).sqrt(); let s = if std > 1e-10 { - (mean / std) * annualization + (mean / std) * trade_annualization } else { 0.0 }; - // Sortino from negative step_returns only (denominator uses total n, not - // count of negative returns — matches Sortino & Price 1994 and the CUDA kernel) - let downside: Vec = returns.iter().filter(|&&r| r < 0.0).copied().collect(); - let sort = if downside.len() > 1 { - let down_var: f64 = downside.iter().map(|r| r.powi(2)).sum::() - / n as f64; - let down_std = down_var.sqrt(); + // Sortino: downside deviation from losing trades only. + // sum_losses is sum of |returns| for losers. We need sum of squared + // returns for losers — approximate from loss_count * (avg_loss)^2. + // This is conservative (underestimates variance of losing trades). + let sort = if trade_stats.losing_trades > 0 { + let avg_loss = trade_stats.sum_losses / trade_stats.losing_trades as f64; + let down_var = avg_loss * avg_loss; // E[loss^2] ≈ E[|loss|]^2 (lower bound) + let down_std = (down_var * (trade_stats.losing_trades as f64 / n_trades)).sqrt(); if down_std > 1e-10 { - (mean / down_std) * annualization + (mean / down_std) * trade_annualization } else { 0.0 } @@ -118,6 +131,7 @@ pub(crate) fn compute_epoch_financials( // Without this, the equity curve compounds across episode resets (capital floor // circuit breaker), producing fake 92%+ MaxDD from cross-episode contamination. // The backtest evaluator already isolates per-window; this makes training match. + let returns = &trade_stats.step_returns; const MAX_DD_WINDOW: usize = 10_000; let dd_start = returns.len().saturating_sub(MAX_DD_WINDOW); let mut equity = initial_capital; @@ -199,10 +213,10 @@ mod tests { total_trades: 5, winning_trades: 5, losing_trades: 0, - sum_wins: 0.05, + sum_wins: 0.10, sum_losses: 0.0, - sum_returns: 0.05, - sum_sq_returns: 0.0005, + sum_returns: 0.10, + sum_sq_returns: 0.00225, step_returns: vec![0.01, 0.02, 0.03, 0.015, 0.025], ..Default::default() }; @@ -222,8 +236,8 @@ mod tests { losing_trades: 2, sum_wins: 0.03, sum_losses: 0.01, - sum_returns: 0.02, - sum_sq_returns: 0.001, + sum_returns: 0.015, + sum_sq_returns: 0.000223, step_returns: vec![0.01, -0.005, 0.008, -0.003, 0.005], ..Default::default() }; diff --git a/crates/ml/src/trainers/dqn/trainer/constructor.rs b/crates/ml/src/trainers/dqn/trainer/constructor.rs index 17e39341b..4a4da4a61 100644 --- a/crates/ml/src/trainers/dqn/trainer/constructor.rs +++ b/crates/ml/src/trainers/dqn/trainer/constructor.rs @@ -93,12 +93,24 @@ impl DQNTrainer { } }; - // batch_size == 0 → auto-compute from VRAM (no artificial cap) + // batch_size == 0 → auto-compute from VRAM (capped at 8192 for RL) if hyperparams.batch_size == 0 { - hyperparams.batch_size = max_safe_batch; + hyperparams.batch_size = max_safe_batch.min(8192); info!("AutoBatchSizer: batch_size auto-computed to {}", hyperparams.batch_size); } + + // Floor: prevent hyperopt from starving the GPU with tiny batches. + // On H100 (ceiling ~2M), floor = 2M/4096 ≈ 512 — reasonable for DQN. + let vram_batch_floor = (max_safe_batch / 4096).clamp(256, 8192); + if hyperparams.batch_size > 0 && hyperparams.batch_size < vram_batch_floor { + info!( + "AutoBatchSizer: batch_size scaled UP {} → {} (VRAM floor)", + hyperparams.batch_size, vram_batch_floor + ); + hyperparams.batch_size = vram_batch_floor; + } + // Cap to VRAM ceiling (never exceed what the GPU can handle) if hyperparams.batch_size > max_safe_batch { info!( @@ -181,7 +193,7 @@ impl DQNTrainer { } else { 0.70 // default fraction }; - (hw.free_memory_mb * frac * 1024.0 * 1024.0) as usize + (hw.free_memory_mb * frac / 3.0 * 1024.0 * 1024.0) as usize } Err(e) => { return Err(anyhow::anyhow!( @@ -196,15 +208,18 @@ impl DQNTrainer { // buffer_size >= 100K (sizer's own minimum — smaller values are intentional). if hyperparams.replay_buffer_vram_fraction > 0.0 && matches!(device, MlDevice::Cuda { .. }) - && hyperparams.buffer_size >= AUTO_REPLAY_MIN_THRESHOLD + && (hyperparams.buffer_size == 0 || hyperparams.buffer_size >= AUTO_REPLAY_MIN_THRESHOLD) { use ml_core::memory_optimization::detect_gpu_hardware; if let Ok(hw) = detect_gpu_hardware() { let raw_sd = if !hyperparams.mbp10_data_dir.is_empty() { 74 } else { 66 }; let aligned_sd = (raw_sd + 7) & !7; + // Divide VRAM budget by 3 regime heads (each gets its own PER buffer) + let num_regime_heads = 3; + let per_head_fraction = hyperparams.replay_buffer_vram_fraction / num_regime_heads as f64; let replay_cfg = hw.optimal_replay_config( aligned_sd, - hyperparams.replay_buffer_vram_fraction, + per_head_fraction, ); per_max_memory_bytes = replay_cfg.per_max_buffer_bytes; if replay_cfg.capacity != hyperparams.buffer_size { @@ -297,8 +312,8 @@ impl DQNTrainer { noisy_sigma_init: hyperparams.noisy_sigma_init, // Rainbow DQN standard: 0.5 // BUG #37 FIX: Q-value clipping (prevents step-level explosions) - q_value_clip_min: -500.0, - q_value_clip_max: 500.0, + q_value_clip_min: hyperparams.q_clip_min as f32, + q_value_clip_max: hyperparams.q_clip_max as f32, // WAVE 23 P0 Fix #1: Adaptive gradient collapse threshold (from hyperparams) gradient_collapse_multiplier: hyperparams.gradient_collapse_multiplier, @@ -498,7 +513,7 @@ impl DQNTrainer { }; let config = HindsightReplayConfig { base_config: PrioritizedReplayConfig { - capacity: hyperparams.buffer_size, + capacity: original_buffer_size, ..Default::default() }, her_ratio: hyperparams.her_ratio, diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index cd8506933..61719a6cb 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -2135,7 +2135,10 @@ impl DQNTrainer { // When model was doing well, INCREASE LR to kick out of overfit minima. // When struggling, decrease to stabilize. Opposite of standard practice. // #24 Anti-intuitive LR: always active (one production path) - if !self.sharpe_history.is_empty() { + // Guard: skip anti-LR for first 5 epochs — early Sharpe is unreliable + // (random policy, tiny replay buffer, Sharpe from few trades). + let anti_lr_warmup = 5; + if epoch >= anti_lr_warmup && !self.sharpe_history.is_empty() { let prev_sharpe = *self.sharpe_history.last().unwrap(); let thresh = self.hyperparams.anti_lr_sharpe_threshold; let base_lr = self.lr_scheduler.get_initial_lr();