gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(ml): remove warmup_steps bug that prevented DQN training

Browse Source 
The batch training path (select_actions_batch → forward()) does NOT
increment DQN::total_steps. Only select_action() does. When
warmup_steps > 0, train_step() checks total_steps < warmup_steps
and returns (0.0, 0.0) — zero loss, zero gradients. The model
never trained; "results" were random initialization Q-values.

Fix: force warmup_steps=0 in hyperopt adapter and remove
warmup_ratio from the 31D→30D search space (saves a dimension
for TPE/PSO effectiveness).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-06 00:29:42 +01:00
parent 351bdaf8f1
commit b6f6d9c7c9
1 changed files with 71 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
crates/ml/src/hyperopt/adapters/dqn.rs
View File

@@ -310,10 +310,9 @@ pub struct DQNParams {
pub beta_entropy: f64,
// Note: variance_cap is NOT in search space, it's fixed in DQNHyperparameters
// WAVE 26 P1.5: Learning rate warmup ratio (0.0-0.2 = 0-20% of training)
/// Learning rate warmup ratio (0.0-0.2)
/// Fraction of training steps to use for linear LR warmup from 0 to target LR
/// 0.0 = no warmup, 0.1 = 10% warmup, 0.2 = 20% warmup
// NOTE: warmup_ratio removed from search space — warmup_steps forced to 0 because
// the batch training path (select_actions_batch) never increments DQN::total_steps.
// Kept in struct for serde backward compat with saved JSON results.
pub warmup_ratio: f64,
// WAVE 26 P1.8: Curiosity-Driven Exploration
@@ -474,11 +473,11 @@ impl Default for DQNParams {
impl ParameterSpace for DQNParams {
fn continuous_bounds() -> Vec<(f64, f64)> {
// 31D search space (expanded from 27D)
// 30D search space (reduced from 31D: warmup_ratio removed — no effect in batch training)
// 16 exotic parameters fixed to validated defaults in from_continuous()
// This makes PSO (20 particles) and TPE dramatically more effective
//
// Kept: the 31 params that genuinely affect trading performance
// Kept: the 30 params that genuinely affect trading performance
// Fixed: ensemble/architecture params that rarely deviate from defaults
vec![
// Base parameters (11D)
@@ -529,35 +528,32 @@ impl ParameterSpace for DQNParams {
// (raised from 0.1: temps below 0.3 produce degenerate trials with 1-3/45 actions)
(0.5_f64.ln(), 2.0_f64.ln()), // 26: eval_softmax_temp (log scale)
// === NEW: Training dynamics parameters (4D) ===
// These were hardcoded but critically affect convergence and OOS generalization.
// LR warmup ratio (fraction of training spent ramping LR from 0 to target)
// Prevents early Q-value collapse when starting from random weights
(0.0, 0.15), // 27: warmup_ratio (linear, 0.0=no warmup, 0.15=15% warmup)
// === Training dynamics parameters (3D, indices 27-29) ===
// warmup_ratio REMOVED: batch training path never increments total_steps,
// so warmup_steps > 0 causes train_step() to return (0.0, 0.0) forever.
// Learning rate schedule (0=constant, 1=linear decay, 2=cosine annealing)
// Cosine annealing prevents Q-value inflation in late epochs
(0.0, 2.0), // 28: lr_decay_type (discrete: round to nearest int)
(0.0, 2.0), // 27: lr_decay_type (discrete: round to nearest int)
// Minimum epochs before early stopping fires (prevents premature termination)
// For 8-epoch hyperopt trials, range [2,6] lets optimizer find the sweet spot
(2.0, 6.0), // 29: min_epochs_before_stopping (discrete: round to int)
(2.0, 6.0), // 28: min_epochs_before_stopping (discrete: round to int)
// Minimum profit factor for trade execution (BUG #7: margin above breakeven)
// 1.1 = 10% above costs, 2.0 = 100% above costs. Filters marginal trades.
(1.1, 2.0), // 30: minimum_profit_factor (linear)
(1.1, 2.0), // 29: minimum_profit_factor (linear)
]
}
fn from_continuous(x: &[f64]) -> Result<Self, MLError> {
if x.len() != 31 {
if x.len() != 30 {
return Err(MLError::ConfigError {
reason: format!("Expected 31 continuous parameters, got {}", x.len()),
reason: format!("Expected 30 continuous parameters, got {}", x.len()),
});
}
// === 31 TUNED parameters (from search space) ===
// === 30 TUNED parameters (from search space) ===
let learning_rate = x[0].exp();
let mut batch_size = x[1].round().max(64.0) as usize; // Only enforce floor. PSO + HardwareBudget control the upper bound.
let buffer_size = x[3].exp().round().max(50_000.0) as usize;
@@ -605,11 +601,11 @@ impl ParameterSpace for DQNParams {
// Eval softmax temperature (floor 0.5 — below this, action diversity collapses)
let eval_softmax_temp = x[26].exp().clamp(0.5, 2.0);
// === NEW: Training dynamics parameters (4D, indices 27-30) ===
let warmup_ratio = x[27].clamp(0.0, 0.15);
let lr_decay_type = x[28].round().clamp(0.0, 2.0); // 0=constant, 1=linear, 2=cosine
let min_epochs_before_stopping = x[29].round().clamp(2.0, 6.0) as usize;
let minimum_profit_factor = x[30].clamp(1.1, 2.0);
// === Training dynamics parameters (3D, indices 27-29) ===
// warmup_ratio removed: batch training never increments total_steps → always stuck in warmup
let lr_decay_type = x[27].round().clamp(0.0, 2.0); // 0=constant, 1=linear, 2=cosine
let min_epochs_before_stopping = x[28].round().clamp(2.0, 6.0) as usize;
let minimum_profit_factor = x[29].clamp(1.1, 2.0);
// === 16 FIXED parameters (validated defaults, removed from search) ===
let kelly_min_trades: usize = 20;
@@ -683,7 +679,7 @@ impl ParameterSpace for DQNParams {
beta_variance,
beta_disagreement,
beta_entropy,
warmup_ratio,
warmup_ratio: 0.0, // Fixed: batch training never increments total_steps
curiosity_weight,
tau,
td_error_clamp_max,
@@ -713,8 +709,9 @@ impl ParameterSpace for DQNParams {
}
fn to_continuous(&self) -> Vec<f64> {
// 31D search space — only the tuned parameters
// 30D search space — only the tuned parameters
// Fixed parameters are NOT emitted (they get their defaults in from_continuous)
// warmup_ratio excluded: forced to 0.0 (batch training never increments total_steps)
vec![
self.learning_rate.ln(), // 0
self.batch_size as f64, // 1
@@ -743,16 +740,16 @@ impl ParameterSpace for DQNParams {
self.noisy_epsilon_floor, // 24
self.cql_alpha, // 25
self.eval_softmax_temp.ln(), // 26
// Training dynamics (4D)
self.warmup_ratio, // 27
self.lr_decay_type, // 28
self.min_epochs_before_stopping as f64, // 29
self.minimum_profit_factor, // 30
// Training dynamics (3D)
self.lr_decay_type, // 27
self.min_epochs_before_stopping as f64, // 28
self.minimum_profit_factor, // 29
]
}
fn param_names() -> Vec<&'static str> {
// 31 tuned parameters (matches continuous_bounds / from_continuous / to_continuous)
// 30 tuned parameters (matches continuous_bounds / from_continuous / to_continuous)
// warmup_ratio removed: forced to 0.0 (batch training never increments total_steps)
vec![
"learning_rate", // 0
"batch_size", // 1
@@ -781,10 +778,9 @@ impl ParameterSpace for DQNParams {
"noisy_epsilon_floor", // 24
"cql_alpha", // 25
"eval_softmax_temp", // 26
"warmup_ratio", // 27
"lr_decay_type", // 28
"min_epochs_before_stopping", // 29
"minimum_profit_factor", // 30
"lr_decay_type", // 27
"min_epochs_before_stopping", // 28
"minimum_profit_factor", // 29
]
}
@@ -796,7 +792,7 @@ impl ParameterSpace for DQNParams {
batch_bound.1 = max_batch;
}
}
// Cap hidden_dim_base by VRAM (index 23 in 31D space)
// Cap hidden_dim_base by VRAM (index 23 in 30D space)
let max_base = budget.max_hidden_dim_base(4, 256, 54, 45);
if let Some(dim_bound) = bounds.get_mut(23) {
dim_bound.1 = max_base as f64;
@@ -2270,7 +2266,6 @@ impl HyperparameterOptimizable for DQNTrainer {
info!(" N-steps: {}", params.n_steps);
info!(" Num atoms: {}", params.num_atoms);
info!(" Eval softmax temp: {:.3}", params.eval_softmax_temp);
info!(" Warmup ratio: {:.3}", params.warmup_ratio);
info!(" LR decay type: {} (0=const, 1=linear, 2=cosine)", params.lr_decay_type.round() as i32);
info!(" Min epochs before stopping: {}", params.min_epochs_before_stopping);
info!(" Minimum profit factor: {:.2}", params.minimum_profit_factor);
@@ -2406,9 +2401,11 @@ impl HyperparameterOptimizable for DQNTrainer {
// Note: tau is set later at line 2086 (WAVE 26 P1.12)
target_update_mode: self.target_update_mode.clone(),
target_update_frequency: self.target_update_frequency,
// Warmup steps computed from warmup_ratio × epochs × ~2000 steps/epoch
// For 8-epoch hyperopt trials: ratio 0.1 → ~1600 steps warmup
warmup_steps: (params.warmup_ratio * (self.epochs as f64) * 2000.0) as usize,
// CRITICAL FIX: warmup_steps MUST be 0 for hyperopt.
// The batch training path (select_actions_batch → forward()) does NOT
// increment DQN::total_steps. Only select_action() does. So if warmup_steps > 0,
// train_step() returns (0.0, 0.0) forever — zero loss, zero gradients, no learning.
warmup_steps: 0,
// P2-A Enhancement
#[allow(clippy::cast_possible_truncation)]
@@ -3432,8 +3429,8 @@ mod tests {
#[test]
fn test_dqn_params_roundtrip() {
// Roundtrip test for the 31D search space
// Only the 31 tuned parameters roundtrip; the 16 fixed params get defaults from from_continuous
// Roundtrip test for the 30D search space
// Only the 30 tuned parameters roundtrip; the 17 fixed params get defaults from from_continuous
let params = DQNParams {
learning_rate: 3.37e-05,
batch_size: 92,
@@ -3459,8 +3456,8 @@ mod tests {
noisy_sigma_init: 0.5,
// Now tuned (moved from fixed):
minimum_profit_factor: 1.5,
warmup_ratio: 0.05, // 5% warmup
lr_decay_type: 2.0, // Cosine annealing
warmup_ratio: 0.0, // Fixed to 0.0 (not in search space)
lr_decay_type: 2.0, // Cosine annealing
min_epochs_before_stopping: 4,
// Still fixed — these will be overwritten by from_continuous defaults
weight_decay: 1e-4,
@@ -3494,10 +3491,10 @@ mod tests {
};
let continuous = params.to_continuous();
assert_eq!(continuous.len(), 31, "to_continuous must return 31D vector");
assert_eq!(continuous.len(), 30, "to_continuous must return 30D vector");
let recovered = DQNParams::from_continuous(&continuous).unwrap();
// Tuned parameters must roundtrip exactly (31D)
// Tuned parameters must roundtrip exactly (30D)
assert!((recovered.learning_rate - params.learning_rate).abs() < 1e-6);
assert_eq!(recovered.batch_size, params.batch_size);
assert!((recovered.gamma - params.gamma).abs() < 1e-6);
@@ -3516,13 +3513,13 @@ mod tests {
assert!((recovered.noisy_epsilon_floor - params.noisy_epsilon_floor).abs() < 1e-6);
assert!((recovered.cql_alpha - params.cql_alpha).abs() < 1e-6);
assert!((recovered.eval_softmax_temp - params.eval_softmax_temp).abs() < 1e-3);
// New tuned params:
assert!((recovered.warmup_ratio - params.warmup_ratio).abs() < 1e-6);
// warmup_ratio is fixed to 0.0, not in search space
assert!((recovered.warmup_ratio - 0.0).abs() < 1e-6);
assert!((recovered.lr_decay_type - params.lr_decay_type).abs() < 1e-6);
assert_eq!(recovered.min_epochs_before_stopping, params.min_epochs_before_stopping);
assert!((recovered.minimum_profit_factor - params.minimum_profit_factor).abs() < 1e-6);
// Fixed parameters must get their validated defaults (16 remaining)
// Fixed parameters must get their validated defaults (17 remaining, including warmup_ratio)
assert_eq!(recovered.kelly_min_trades, 20);
assert!((recovered.ensemble_size - 5.0).abs() < 1e-6);
assert!((recovered.td_error_clamp_max - 10.0).abs() < 1e-6);
@@ -3537,7 +3534,7 @@ mod tests {
#[test]
fn test_dqn_params_bounds() {
let bounds = DQNParams::continuous_bounds();
assert_eq!(bounds.len(), 31); // 31D search space
assert_eq!(bounds.len(), 30); // 30D search space
// Check log-scale bounds are reasonable
assert!(bounds[0].0 < bounds[0].1); // learning_rate
@@ -3586,7 +3583,7 @@ mod tests {
#[test]
fn test_param_names() {
let names = DQNParams::param_names();
assert_eq!(names.len(), 31); // 31D search space
assert_eq!(names.len(), 30); // 30D search space
assert_eq!(names[0], "learning_rate");
assert_eq!(names[1], "batch_size");
assert_eq!(names[2], "gamma");
@@ -3613,12 +3610,16 @@ mod tests {
assert_eq!(names[23], "hidden_dim_base");
assert_eq!(names[24], "noisy_epsilon_floor");
assert_eq!(names[25], "cql_alpha");
assert_eq!(names[26], "eval_softmax_temp");
assert_eq!(names[27], "lr_decay_type");
assert_eq!(names[28], "min_epochs_before_stopping");
assert_eq!(names[29], "minimum_profit_factor");
}
#[test]
fn test_per_params_always_enabled() {
// Test that PER is always enabled with tunable alpha/beta parameters
// 31D search space
// 30D search space (warmup_ratio removed)
let continuous = vec![
3e-5_f64.ln(), 92.0, 0.9588, 97_273_f64.ln(), 1.404, 4.0, 24.77_f64.ln(), 0.05, 1.0,
0.6, 0.4, // 9-10: per_alpha, per_beta_start
@@ -3633,10 +3634,9 @@ mod tests {
0.05, // 24: noisy_epsilon_floor
0.1, // 25: cql_alpha
0.8_f64.ln(), // 26: eval_softmax_temp (>= 0.5 floor)
0.05, // 27: warmup_ratio
2.0, // 28: lr_decay_type (cosine)
4.0, // 29: min_epochs_before_stopping
1.5, // 30: minimum_profit_factor
2.0, // 27: lr_decay_type (cosine)
4.0, // 28: min_epochs_before_stopping
1.5, // 29: minimum_profit_factor
];
let params = DQNParams::from_continuous(&continuous).unwrap();
@@ -3646,8 +3646,9 @@ mod tests {
assert!(params.use_dueling);
assert!(params.use_distributional);
assert!(params.use_noisy_nets);
assert!((params.warmup_ratio - 0.0).abs() < 1e-6); // Always fixed to 0.0
// Test PER parameter bounds (min values) — 31D
// Test PER parameter bounds (min values) — 30D
let continuous_min = vec![
2e-5_f64.ln(), 64.0, 0.95, 50_000_f64.ln(), 1.0, 1.0, 10.0_f64.ln(), 0.01, 0.5,
0.4, 0.2, // per_alpha min, per_beta_start min
@@ -3662,10 +3663,9 @@ mod tests {
0.02, // noisy_epsilon_floor min
0.0, // cql_alpha min
0.5_f64.ln(), // eval_softmax_temp min
0.0, // 27: warmup_ratio min
0.0, // 28: lr_decay_type (constant)
2.0, // 29: min_epochs_before_stopping min
1.1, // 30: minimum_profit_factor min
0.0, // 27: lr_decay_type (constant)
2.0, // 28: min_epochs_before_stopping min
1.1, // 29: minimum_profit_factor min
];
let params_min = DQNParams::from_continuous(&continuous_min).unwrap();
assert!((params_min.per_alpha - 0.4).abs() < 1e-6);
@@ -3674,7 +3674,7 @@ mod tests {
assert!(params_min.use_distributional);
assert!(params_min.use_noisy_nets);
// Test PER parameter bounds (max values) — 31D
// Test PER parameter bounds (max values) — 30D
let continuous_max = vec![
8e-5_f64.ln(), 4096.0, 0.99, 100_000_f64.ln(), 2.0, 4.0, 40.0_f64.ln(), 0.2, 2.0,
0.8, 0.6, // per_alpha max, per_beta_start max
@@ -3689,10 +3689,9 @@ mod tests {
0.10, // noisy_epsilon_floor max
0.5, // cql_alpha max
2.0_f64.ln(), // eval_softmax_temp max
0.15, // 27: warmup_ratio max
2.0, // 28: lr_decay_type (cosine)
6.0, // 29: min_epochs_before_stopping max
2.0, // 30: minimum_profit_factor max
2.0, // 27: lr_decay_type (cosine)
6.0, // 28: min_epochs_before_stopping max
2.0, // 29: minimum_profit_factor max
];
let params_max = DQNParams::from_continuous(&continuous_max).unwrap();
assert!((params_max.per_alpha - 0.8).abs() < 1e-6);
@@ -3992,7 +3991,7 @@ mod tests {
fn test_qr_dqn_roundtrip_continuous() {
let params = DQNParams::default();
let continuous = params.to_continuous();
assert_eq!(continuous.len(), 31, "Should have 31 continuous dimensions");
assert_eq!(continuous.len(), 30, "Should have 30 continuous dimensions");
let roundtrip = DQNParams::from_continuous(&continuous).unwrap();
// num_quantiles and qr_kappa are now fixed defaults (not in search space)
assert_eq!(roundtrip.num_quantiles, 64); // Fixed default
@@ -4017,7 +4016,7 @@ mod tests {
fn test_batch_size_respects_wide_bounds() {
// Simulate PSO choosing batch_size=2048 (within VRAM-aware bounds)
let bounds = DQNParams::continuous_bounds();
let mut params = vec![0.0_f64; 31];
let mut params = vec![0.0_f64; 30];
params[1] = 2048.0; // batch_size (index 1)
// Fill other required params with valid defaults
params[0] = (1e-4_f64).ln(); // learning_rate
@@ -4030,8 +4029,8 @@ mod tests {
params[8] = 0.5; // transaction_cost_multiplier
params[9] = 0.6; // per_alpha
params[10] = 0.4; // per_beta_start
// Remaining params (indices 11-30) -- use midpoint of bounds
for i in 11..31 {
// Remaining params (indices 11-29) -- use midpoint of bounds
for i in 11..30 {
params[i] = (bounds[i].0 + bounds[i].1) / 2.0;
}
let result = DQNParams::from_continuous(&params).unwrap();
@@ -4155,9 +4154,9 @@ mod tests {
);
// from_continuous should clamp to 0.5 even if value is below
let mut low_temp_vec = vec![0.0_f64; 31];
let mut low_temp_vec = vec![0.0_f64; 30];
// Set all to midpoint of bounds
for i in 0..31 {
for i in 0..30 {
low_temp_vec[i] = (bounds[i].0 + bounds[i].1) / 2.0;
}
low_temp_vec[26] = 0.01_f64.ln(); // Way below floor