gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: 11 H100 training bugs — Sharpe per-trade, batch autosizing, PER/HER capacity, Q-clip

Browse Source 
Sharpe calculation:
- Use per-trade returns (sum_returns/sum_sq_returns) instead of per-bar
  step_returns. Per-bar Sharpe collapsed variance → bogus 19.75 with PF=0.03.
- Annualize by sqrt(trades_per_year) not sqrt(bars_per_year).

Batch sizing:
- Cap auto-computed batch_size at 8192 (VRAM ceiling of 2M is OOM limit, not
  optimal RL batch).
- Add VRAM floor: batch_size < ceiling/4096 gets scaled UP (128 → 512 on H100).
- Only let hyperopt override batch_size when explicitly non-zero — preserve
  profile's batch_size=0 auto-compute sentinel.

Replay buffer:
- Divide per_max_memory_bytes by 3.0 for regime heads (PER budget was 3x too
  large, causing OOM cascade 74M → 37M → 18M → 9M → 4.6M).
- HER buffer uses original_buffer_size (pre-autosizer), not inflated 74M.

Q-value clipping:
- Wire hyperparams.q_clip_min/max to DQNConfig (was hardcoded ±500, production
  TOML has ±50). Prevents Q-value overestimation ratio of 94.6x at epoch 2.

Training stability:
- Anti-LR warmup: skip first 5 epochs (early Sharpe unreliable from random
  policy). Prevents bogus 3x LR boost at epoch 2.
- min_replay_size from profile (1000), not hyperopt batch_size (128).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-04-02 00:54:12 +02:00
parent 3dd7449c6a
commit 5b06484da7
5 changed files with 73 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
crates/ml/examples/train_baseline_rl.rs
View File

@@ -134,7 +134,7 @@ struct Args {
epochs: usize,
/// Training batch size
#[arg(long, default_value_t = 128)]
#[arg(long, default_value_t = 0)]
batch_size: usize,
/// Path to directory containing .dbn.zst files (env: FOXHUNT_DATA_DIR)
@@ -519,7 +519,7 @@ fn train_dqn_fold(
epsilon_end: hp_f64(hp, "epsilon_end").unwrap_or(0.01),
epsilon_decay: hp_f64(hp, "epsilon_decay").unwrap_or(0.995),
buffer_size: hp_usize(hp, "buffer_size").unwrap_or(gpu_profile.training.buffer_size),
min_replay_size: hp_usize(hp, "batch_size").unwrap_or(args.batch_size),
min_replay_size: hp_usize(hp, "min_replay_size").unwrap_or(1000),
epochs: args.epochs,
checkpoint_frequency: 10,
hidden_dim_base: dqn_hidden_base,
@@ -575,7 +575,12 @@ fn train_dqn_fold(
profile.apply_to(&mut hyperparams);
// CLI args override profile: re-apply any arg that the user can set explicitly.
hyperparams.epochs = args.epochs;
hyperparams.batch_size = hp_usize(hp, "batch_size").unwrap_or(args.batch_size);
// Only override batch_size from CLI/hyperopt if explicitly non-zero.
// batch_size=0 is the auto-compute sentinel — let the constructor handle it.
let hp_batch = hp_usize(hp, "batch_size").unwrap_or(args.batch_size);
if hp_batch > 0 {
hyperparams.batch_size = hp_batch;
}
hyperparams.learning_rate = hp_f64(hp, "learning_rate").unwrap_or(args.learning_rate);
hyperparams.max_training_steps_per_epoch = args.max_steps_per_epoch;
hyperparams.initial_capital = args.initial_capital as f32;

4
crates/ml/src/benchmark/dqn_benchmark.rs
View File

@@ -449,8 +449,8 @@ impl DqnBenchmarkRunner {
noisy_sigma_init: 0.5,
// BUG #37 FIX: Q-value clipping (prevents step-level explosions)
q_value_clip_min: -500.0,
q_value_clip_max: 500.0,
q_value_clip_min: -50.0,
q_value_clip_max: 50.0,
// WAVE 23 P0 Fix #1: Adaptive gradient collapse threshold
gradient_collapse_multiplier: 100.0,

58
crates/ml/src/trainers/dqn/financials.rs
View File

@@ -74,31 +74,44 @@ pub(crate) fn compute_epoch_financials(
0.0
};
// Sharpe ratio from step_returns (per-bar, annualized)
let annualization = bars_per_year.sqrt();
// Sharpe ratio from PER-TRADE returns (not per-bar).
// Per-bar step_returns collapse variance when most bars are flat,
// producing bogus Sharpe (e.g., 19.75 with PF=0.03). Per-trade
// returns use the GPU-aggregated sum_returns / sum_sq_returns.
//
// Annualization: sqrt(trades_per_year) where trades_per_year is
// estimated from the epoch's trade frequency.
let n_trades = total_trades as f64;
let n_bars = trade_stats.step_returns.len() as f64;
let trades_per_year = if n_bars > 0.0 {
(n_trades / n_bars) * bars_per_year
} else {
0.0
};
let trade_annualization = trades_per_year.max(1.0).sqrt();
let returns = &trade_stats.step_returns;
let n = returns.len();
let (sharpe, sortino) = if n > 1 {
let mean: f64 = returns.iter().sum::<f64>() / n as f64;
let variance: f64 = returns.iter().map(|r| (r - mean).powi(2)).sum::<f64>() / n as f64;
let std = variance.sqrt();
let (sharpe, sortino) = if total_trades > 1 {
// Per-trade mean and variance from GPU-aggregated sums
let mean = trade_stats.sum_returns / n_trades;
let var = (trade_stats.sum_sq_returns / n_trades) - mean * mean;
let std = var.max(0.0).sqrt();
let s = if std > 1e-10 {
(mean / std) * annualization
(mean / std) * trade_annualization
} else {
0.0
};
// Sortino from negative step_returns only (denominator uses total n, not
// count of negative returns — matches Sortino & Price 1994 and the CUDA kernel)
let downside: Vec<f64> = returns.iter().filter(|&&r| r < 0.0).copied().collect();
let sort = if downside.len() > 1 {
let down_var: f64 = downside.iter().map(|r| r.powi(2)).sum::<f64>()
/ n as f64;
let down_std = down_var.sqrt();
// Sortino: downside deviation from losing trades only.
// sum_losses is sum of |returns| for losers. We need sum of squared
// returns for losers — approximate from loss_count * (avg_loss)^2.
// This is conservative (underestimates variance of losing trades).
let sort = if trade_stats.losing_trades > 0 {
let avg_loss = trade_stats.sum_losses / trade_stats.losing_trades as f64;
let down_var = avg_loss * avg_loss; // E[loss^2] ≈ E[|loss|]^2 (lower bound)
let down_std = (down_var * (trade_stats.losing_trades as f64 / n_trades)).sqrt();
if down_std > 1e-10 {
(mean / down_std) * annualization
(mean / down_std) * trade_annualization
} else {
0.0
}
@@ -118,6 +131,7 @@ pub(crate) fn compute_epoch_financials(
// Without this, the equity curve compounds across episode resets (capital floor
// circuit breaker), producing fake 92%+ MaxDD from cross-episode contamination.
// The backtest evaluator already isolates per-window; this makes training match.
let returns = &trade_stats.step_returns;
const MAX_DD_WINDOW: usize = 10_000;
let dd_start = returns.len().saturating_sub(MAX_DD_WINDOW);
let mut equity = initial_capital;
@@ -199,10 +213,10 @@ mod tests {
total_trades: 5,
winning_trades: 5,
losing_trades: 0,
sum_wins: 0.05,
sum_wins: 0.10,
sum_losses: 0.0,
sum_returns: 0.05,
sum_sq_returns: 0.0005,
sum_returns: 0.10,
sum_sq_returns: 0.00225,
step_returns: vec![0.01, 0.02, 0.03, 0.015, 0.025],
..Default::default()
};
@@ -222,8 +236,8 @@ mod tests {
losing_trades: 2,
sum_wins: 0.03,
sum_losses: 0.01,
sum_returns: 0.02,
sum_sq_returns: 0.001,
sum_returns: 0.015,
sum_sq_returns: 0.000223,
step_returns: vec![0.01, -0.005, 0.008, -0.003, 0.005],
..Default::default()
};

31
crates/ml/src/trainers/dqn/trainer/constructor.rs
View File

@@ -93,12 +93,24 @@ impl DQNTrainer {
}
};
// batch_size == 0 → auto-compute from VRAM (no artificial cap)
// batch_size == 0 → auto-compute from VRAM (capped at 8192 for RL)
if hyperparams.batch_size == 0 {
hyperparams.batch_size = max_safe_batch;
hyperparams.batch_size = max_safe_batch.min(8192);
info!("AutoBatchSizer: batch_size auto-computed to {}", hyperparams.batch_size);
}
// Floor: prevent hyperopt from starving the GPU with tiny batches.
// On H100 (ceiling ~2M), floor = 2M/4096 ≈ 512 — reasonable for DQN.
let vram_batch_floor = (max_safe_batch / 4096).clamp(256, 8192);
if hyperparams.batch_size > 0 && hyperparams.batch_size < vram_batch_floor {
info!(
"AutoBatchSizer: batch_size scaled UP {} → {} (VRAM floor)",
hyperparams.batch_size, vram_batch_floor
);
hyperparams.batch_size = vram_batch_floor;
}
// Cap to VRAM ceiling (never exceed what the GPU can handle)
if hyperparams.batch_size > max_safe_batch {
info!(
@@ -181,7 +193,7 @@ impl DQNTrainer {
} else {
0.70 // default fraction
};
(hw.free_memory_mb * frac * 1024.0 * 1024.0) as usize
(hw.free_memory_mb * frac / 3.0 * 1024.0 * 1024.0) as usize
}
Err(e) => {
return Err(anyhow::anyhow!(
@@ -196,15 +208,18 @@ impl DQNTrainer {
// buffer_size >= 100K (sizer's own minimum — smaller values are intentional).
if hyperparams.replay_buffer_vram_fraction > 0.0
&& matches!(device, MlDevice::Cuda { .. })
&& hyperparams.buffer_size >= AUTO_REPLAY_MIN_THRESHOLD
&& (hyperparams.buffer_size == 0 || hyperparams.buffer_size >= AUTO_REPLAY_MIN_THRESHOLD)
{
use ml_core::memory_optimization::detect_gpu_hardware;
if let Ok(hw) = detect_gpu_hardware() {
let raw_sd = if !hyperparams.mbp10_data_dir.is_empty() { 74 } else { 66 };
let aligned_sd = (raw_sd + 7) & !7;
// Divide VRAM budget by 3 regime heads (each gets its own PER buffer)
let num_regime_heads = 3;
let per_head_fraction = hyperparams.replay_buffer_vram_fraction / num_regime_heads as f64;
let replay_cfg = hw.optimal_replay_config(
aligned_sd,
hyperparams.replay_buffer_vram_fraction,
per_head_fraction,
);
per_max_memory_bytes = replay_cfg.per_max_buffer_bytes;
if replay_cfg.capacity != hyperparams.buffer_size {
@@ -297,8 +312,8 @@ impl DQNTrainer {
noisy_sigma_init: hyperparams.noisy_sigma_init, // Rainbow DQN standard: 0.5
// BUG #37 FIX: Q-value clipping (prevents step-level explosions)
q_value_clip_min: -500.0,
q_value_clip_max: 500.0,
q_value_clip_min: hyperparams.q_clip_min as f32,
q_value_clip_max: hyperparams.q_clip_max as f32,
// WAVE 23 P0 Fix #1: Adaptive gradient collapse threshold (from hyperparams)
gradient_collapse_multiplier: hyperparams.gradient_collapse_multiplier,
@@ -498,7 +513,7 @@ impl DQNTrainer {
};
let config = HindsightReplayConfig {
base_config: PrioritizedReplayConfig {
capacity: hyperparams.buffer_size,
capacity: original_buffer_size,
..Default::default()
},
her_ratio: hyperparams.her_ratio,

5
crates/ml/src/trainers/dqn/trainer/training_loop.rs
View File

@@ -2135,7 +2135,10 @@ impl DQNTrainer {
// When model was doing well, INCREASE LR to kick out of overfit minima.
// When struggling, decrease to stabilize. Opposite of standard practice.
// #24 Anti-intuitive LR: always active (one production path)
if !self.sharpe_history.is_empty() {
// Guard: skip anti-LR for first 5 epochs — early Sharpe is unreliable
// (random policy, tiny replay buffer, Sharpe from few trades).
let anti_lr_warmup = 5;
if epoch >= anti_lr_warmup && !self.sharpe_history.is_empty() {
let prev_sharpe = *self.sharpe_history.last().unwrap();
let thresh = self.hyperparams.anti_lr_sharpe_threshold;
let base_lr = self.lr_scheduler.get_initial_lr();