From 5b06484da763958243aa7a50498a3ec303d73aaa Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Thu, 2 Apr 2026 00:54:12 +0200 Subject: [PATCH] =?UTF-8?q?fix:=2011=20H100=20training=20bugs=20=E2=80=94?= =?UTF-8?q?=20Sharpe=20per-trade,=20batch=20autosizing,=20PER/HER=20capaci?= =?UTF-8?q?ty,=20Q-clip?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sharpe calculation: - Use per-trade returns (sum_returns/sum_sq_returns) instead of per-bar step_returns. Per-bar Sharpe collapsed variance → bogus 19.75 with PF=0.03. - Annualize by sqrt(trades_per_year) not sqrt(bars_per_year). Batch sizing: - Cap auto-computed batch_size at 8192 (VRAM ceiling of 2M is OOM limit, not optimal RL batch). - Add VRAM floor: batch_size < ceiling/4096 gets scaled UP (128 → 512 on H100). - Only let hyperopt override batch_size when explicitly non-zero — preserve profile's batch_size=0 auto-compute sentinel. Replay buffer: - Divide per_max_memory_bytes by 3.0 for regime heads (PER budget was 3x too large, causing OOM cascade 74M → 37M → 18M → 9M → 4.6M). - HER buffer uses original_buffer_size (pre-autosizer), not inflated 74M. Q-value clipping: - Wire hyperparams.q_clip_min/max to DQNConfig (was hardcoded ±500, production TOML has ±50). Prevents Q-value overestimation ratio of 94.6x at epoch 2. Training stability: - Anti-LR warmup: skip first 5 epochs (early Sharpe unreliable from random policy). Prevents bogus 3x LR boost at epoch 2. - min_replay_size from profile (1000), not hyperopt batch_size (128). Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/ml/examples/train_baseline_rl.rs | 11 +++- crates/ml/src/benchmark/dqn_benchmark.rs | 4 +- crates/ml/src/trainers/dqn/financials.rs | 58 ++++++++++++------- .../src/trainers/dqn/trainer/constructor.rs | 31 +++++++--- .../src/trainers/dqn/trainer/training_loop.rs | 5 +- 5 files changed, 73 insertions(+), 36 deletions(-) diff --git a/crates/ml/examples/train_baseline_rl.rs b/crates/ml/examples/train_baseline_rl.rs index f220f9dff..c6d7fed94 100644 --- a/crates/ml/examples/train_baseline_rl.rs +++ b/crates/ml/examples/train_baseline_rl.rs @@ -134,7 +134,7 @@ struct Args { epochs: usize, /// Training batch size - #[arg(long, default_value_t = 128)] + #[arg(long, default_value_t = 0)] batch_size: usize, /// Path to directory containing .dbn.zst files (env: FOXHUNT_DATA_DIR) @@ -519,7 +519,7 @@ fn train_dqn_fold( epsilon_end: hp_f64(hp, "epsilon_end").unwrap_or(0.01), epsilon_decay: hp_f64(hp, "epsilon_decay").unwrap_or(0.995), buffer_size: hp_usize(hp, "buffer_size").unwrap_or(gpu_profile.training.buffer_size), - min_replay_size: hp_usize(hp, "batch_size").unwrap_or(args.batch_size), + min_replay_size: hp_usize(hp, "min_replay_size").unwrap_or(1000), epochs: args.epochs, checkpoint_frequency: 10, hidden_dim_base: dqn_hidden_base, @@ -575,7 +575,12 @@ fn train_dqn_fold( profile.apply_to(&mut hyperparams); // CLI args override profile: re-apply any arg that the user can set explicitly. hyperparams.epochs = args.epochs; - hyperparams.batch_size = hp_usize(hp, "batch_size").unwrap_or(args.batch_size); + // Only override batch_size from CLI/hyperopt if explicitly non-zero. + // batch_size=0 is the auto-compute sentinel — let the constructor handle it. + let hp_batch = hp_usize(hp, "batch_size").unwrap_or(args.batch_size); + if hp_batch > 0 { + hyperparams.batch_size = hp_batch; + } hyperparams.learning_rate = hp_f64(hp, "learning_rate").unwrap_or(args.learning_rate); hyperparams.max_training_steps_per_epoch = args.max_steps_per_epoch; hyperparams.initial_capital = args.initial_capital as f32; diff --git a/crates/ml/src/benchmark/dqn_benchmark.rs b/crates/ml/src/benchmark/dqn_benchmark.rs index 3b4cad763..98b04f2cd 100644 --- a/crates/ml/src/benchmark/dqn_benchmark.rs +++ b/crates/ml/src/benchmark/dqn_benchmark.rs @@ -449,8 +449,8 @@ impl DqnBenchmarkRunner { noisy_sigma_init: 0.5, // BUG #37 FIX: Q-value clipping (prevents step-level explosions) - q_value_clip_min: -500.0, - q_value_clip_max: 500.0, + q_value_clip_min: -50.0, + q_value_clip_max: 50.0, // WAVE 23 P0 Fix #1: Adaptive gradient collapse threshold gradient_collapse_multiplier: 100.0, diff --git a/crates/ml/src/trainers/dqn/financials.rs b/crates/ml/src/trainers/dqn/financials.rs index b6149cbec..355e1b29b 100644 --- a/crates/ml/src/trainers/dqn/financials.rs +++ b/crates/ml/src/trainers/dqn/financials.rs @@ -74,31 +74,44 @@ pub(crate) fn compute_epoch_financials( 0.0 }; - // Sharpe ratio from step_returns (per-bar, annualized) - let annualization = bars_per_year.sqrt(); + // Sharpe ratio from PER-TRADE returns (not per-bar). + // Per-bar step_returns collapse variance when most bars are flat, + // producing bogus Sharpe (e.g., 19.75 with PF=0.03). Per-trade + // returns use the GPU-aggregated sum_returns / sum_sq_returns. + // + // Annualization: sqrt(trades_per_year) where trades_per_year is + // estimated from the epoch's trade frequency. + let n_trades = total_trades as f64; + let n_bars = trade_stats.step_returns.len() as f64; + let trades_per_year = if n_bars > 0.0 { + (n_trades / n_bars) * bars_per_year + } else { + 0.0 + }; + let trade_annualization = trades_per_year.max(1.0).sqrt(); - let returns = &trade_stats.step_returns; - let n = returns.len(); - let (sharpe, sortino) = if n > 1 { - let mean: f64 = returns.iter().sum::() / n as f64; - let variance: f64 = returns.iter().map(|r| (r - mean).powi(2)).sum::() / n as f64; - let std = variance.sqrt(); + let (sharpe, sortino) = if total_trades > 1 { + // Per-trade mean and variance from GPU-aggregated sums + let mean = trade_stats.sum_returns / n_trades; + let var = (trade_stats.sum_sq_returns / n_trades) - mean * mean; + let std = var.max(0.0).sqrt(); let s = if std > 1e-10 { - (mean / std) * annualization + (mean / std) * trade_annualization } else { 0.0 }; - // Sortino from negative step_returns only (denominator uses total n, not - // count of negative returns — matches Sortino & Price 1994 and the CUDA kernel) - let downside: Vec = returns.iter().filter(|&&r| r < 0.0).copied().collect(); - let sort = if downside.len() > 1 { - let down_var: f64 = downside.iter().map(|r| r.powi(2)).sum::() - / n as f64; - let down_std = down_var.sqrt(); + // Sortino: downside deviation from losing trades only. + // sum_losses is sum of |returns| for losers. We need sum of squared + // returns for losers — approximate from loss_count * (avg_loss)^2. + // This is conservative (underestimates variance of losing trades). + let sort = if trade_stats.losing_trades > 0 { + let avg_loss = trade_stats.sum_losses / trade_stats.losing_trades as f64; + let down_var = avg_loss * avg_loss; // E[loss^2] ≈ E[|loss|]^2 (lower bound) + let down_std = (down_var * (trade_stats.losing_trades as f64 / n_trades)).sqrt(); if down_std > 1e-10 { - (mean / down_std) * annualization + (mean / down_std) * trade_annualization } else { 0.0 } @@ -118,6 +131,7 @@ pub(crate) fn compute_epoch_financials( // Without this, the equity curve compounds across episode resets (capital floor // circuit breaker), producing fake 92%+ MaxDD from cross-episode contamination. // The backtest evaluator already isolates per-window; this makes training match. + let returns = &trade_stats.step_returns; const MAX_DD_WINDOW: usize = 10_000; let dd_start = returns.len().saturating_sub(MAX_DD_WINDOW); let mut equity = initial_capital; @@ -199,10 +213,10 @@ mod tests { total_trades: 5, winning_trades: 5, losing_trades: 0, - sum_wins: 0.05, + sum_wins: 0.10, sum_losses: 0.0, - sum_returns: 0.05, - sum_sq_returns: 0.0005, + sum_returns: 0.10, + sum_sq_returns: 0.00225, step_returns: vec![0.01, 0.02, 0.03, 0.015, 0.025], ..Default::default() }; @@ -222,8 +236,8 @@ mod tests { losing_trades: 2, sum_wins: 0.03, sum_losses: 0.01, - sum_returns: 0.02, - sum_sq_returns: 0.001, + sum_returns: 0.015, + sum_sq_returns: 0.000223, step_returns: vec![0.01, -0.005, 0.008, -0.003, 0.005], ..Default::default() }; diff --git a/crates/ml/src/trainers/dqn/trainer/constructor.rs b/crates/ml/src/trainers/dqn/trainer/constructor.rs index 17e39341b..4a4da4a61 100644 --- a/crates/ml/src/trainers/dqn/trainer/constructor.rs +++ b/crates/ml/src/trainers/dqn/trainer/constructor.rs @@ -93,12 +93,24 @@ impl DQNTrainer { } }; - // batch_size == 0 → auto-compute from VRAM (no artificial cap) + // batch_size == 0 → auto-compute from VRAM (capped at 8192 for RL) if hyperparams.batch_size == 0 { - hyperparams.batch_size = max_safe_batch; + hyperparams.batch_size = max_safe_batch.min(8192); info!("AutoBatchSizer: batch_size auto-computed to {}", hyperparams.batch_size); } + + // Floor: prevent hyperopt from starving the GPU with tiny batches. + // On H100 (ceiling ~2M), floor = 2M/4096 ≈ 512 — reasonable for DQN. + let vram_batch_floor = (max_safe_batch / 4096).clamp(256, 8192); + if hyperparams.batch_size > 0 && hyperparams.batch_size < vram_batch_floor { + info!( + "AutoBatchSizer: batch_size scaled UP {} → {} (VRAM floor)", + hyperparams.batch_size, vram_batch_floor + ); + hyperparams.batch_size = vram_batch_floor; + } + // Cap to VRAM ceiling (never exceed what the GPU can handle) if hyperparams.batch_size > max_safe_batch { info!( @@ -181,7 +193,7 @@ impl DQNTrainer { } else { 0.70 // default fraction }; - (hw.free_memory_mb * frac * 1024.0 * 1024.0) as usize + (hw.free_memory_mb * frac / 3.0 * 1024.0 * 1024.0) as usize } Err(e) => { return Err(anyhow::anyhow!( @@ -196,15 +208,18 @@ impl DQNTrainer { // buffer_size >= 100K (sizer's own minimum — smaller values are intentional). if hyperparams.replay_buffer_vram_fraction > 0.0 && matches!(device, MlDevice::Cuda { .. }) - && hyperparams.buffer_size >= AUTO_REPLAY_MIN_THRESHOLD + && (hyperparams.buffer_size == 0 || hyperparams.buffer_size >= AUTO_REPLAY_MIN_THRESHOLD) { use ml_core::memory_optimization::detect_gpu_hardware; if let Ok(hw) = detect_gpu_hardware() { let raw_sd = if !hyperparams.mbp10_data_dir.is_empty() { 74 } else { 66 }; let aligned_sd = (raw_sd + 7) & !7; + // Divide VRAM budget by 3 regime heads (each gets its own PER buffer) + let num_regime_heads = 3; + let per_head_fraction = hyperparams.replay_buffer_vram_fraction / num_regime_heads as f64; let replay_cfg = hw.optimal_replay_config( aligned_sd, - hyperparams.replay_buffer_vram_fraction, + per_head_fraction, ); per_max_memory_bytes = replay_cfg.per_max_buffer_bytes; if replay_cfg.capacity != hyperparams.buffer_size { @@ -297,8 +312,8 @@ impl DQNTrainer { noisy_sigma_init: hyperparams.noisy_sigma_init, // Rainbow DQN standard: 0.5 // BUG #37 FIX: Q-value clipping (prevents step-level explosions) - q_value_clip_min: -500.0, - q_value_clip_max: 500.0, + q_value_clip_min: hyperparams.q_clip_min as f32, + q_value_clip_max: hyperparams.q_clip_max as f32, // WAVE 23 P0 Fix #1: Adaptive gradient collapse threshold (from hyperparams) gradient_collapse_multiplier: hyperparams.gradient_collapse_multiplier, @@ -498,7 +513,7 @@ impl DQNTrainer { }; let config = HindsightReplayConfig { base_config: PrioritizedReplayConfig { - capacity: hyperparams.buffer_size, + capacity: original_buffer_size, ..Default::default() }, her_ratio: hyperparams.her_ratio, diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index cd8506933..61719a6cb 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -2135,7 +2135,10 @@ impl DQNTrainer { // When model was doing well, INCREASE LR to kick out of overfit minima. // When struggling, decrease to stabilize. Opposite of standard practice. // #24 Anti-intuitive LR: always active (one production path) - if !self.sharpe_history.is_empty() { + // Guard: skip anti-LR for first 5 epochs — early Sharpe is unreliable + // (random policy, tiny replay buffer, Sharpe from few trades). + let anti_lr_warmup = 5; + if epoch >= anti_lr_warmup && !self.sharpe_history.is_empty() { let prev_sharpe = *self.sharpe_history.last().unwrap(); let thresh = self.hyperparams.anti_lr_sharpe_threshold; let base_lr = self.lr_scheduler.get_initial_lr();