gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: unify Default impls — all config structs match conservative() values

Browse Source 
DQNConfig::default(): learning_rate 1e-4→3e-5, gamma 0.99→0.95,
batch_size 64→1024, v_min/v_max -25/25→-50/50, q_clip -100/100→-200/200.

DQNConfig::conservative(): learning_rate 1e-4→3e-5, gamma 0.99→0.95,
batch_size 32→1024, v_min/v_max -25/25→-50/50, q_clip -500/500→-200/200.

DQNConfig::emergency_safe_defaults(): v_min/v_max -25/25→-50/50,
q_clip -500/500→-200/200.

GpuDqnTrainConfig::default(): state_dim 72→48, v_min/v_max -2/2→-50/50,
lr 3e-4→3e-5, weight_decay 1e-5→1e-4, batch_size 256→64.

ExperienceCollectorConfig::default(): use_noisy_nets false→true,
use_distributional false→true, num_atoms 1→51,
v_min/v_max -2/2→-50/50, q_clip -500/500→-200/200.

DqnBacktestConfig::from_network_dims(): v_min/v_max -2/2→-50/50.
GpuExperienceCollector constructor: v_min/v_max -2/2→-50/50.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-25 09:57:04 +01:00
parent 04d8802c94
commit 1a51e67ba0
4 changed files with 44 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
crates/ml-dqn/src/dqn.rs
View File

@@ -244,14 +244,14 @@ impl Default for DQNConfig {
state_dim: 48, // 40 market + 3 portfolio = 43, padded to 48 for tensor core alignment
num_actions: 9, // 9 exposure levels (25% steps)
hidden_dims: vec![256, 256],
learning_rate: 1e-4,
gamma: 0.99,
learning_rate: 3e-5, // Conservative default for stable training (matches conservative())
gamma: 0.95,
epsilon_start: 1.0,
epsilon_end: 0.01,
epsilon_decay: 0.995,
replay_buffer_capacity: 100_000,
collapse_warmup_capacity: 100_000,
batch_size: 64,
batch_size: 1024, // H100 target (AutoBatchSizer caps for smaller GPUs)
min_replay_size: 1000,
target_update_freq: 1000,
huber_delta: 1.0,
@@ -272,12 +272,12 @@ impl Default for DQNConfig {
per_max_memory_bytes: 4 * 1024 * 1024 * 1024,
dueling_hidden_dim: 128,
num_atoms: 51,
v_min: -25.0, // DSR Q-values: rewards ±2 with gamma=0.92 → Q ≈ ±25
v_max: 25.0, // DSR Q-values: rewards ±2 with gamma=0.92 → Q ≈ ±25
v_min: -50.0, // Reward v6 scale: mark-to-market returns with gamma=0.95
v_max: 50.0, // Reward v6 scale: mark-to-market returns with gamma=0.95
noisy_sigma_init: 0.5,
enable_q_value_clipping: true,
q_value_clip_min: -100.0,
q_value_clip_max: 100.0,
q_value_clip_min: -200.0, // Reward v6: tighter than old -500 but covers v_range + safety margin
q_value_clip_max: 200.0,
gradient_collapse_multiplier: 2.0,
gradient_collapse_patience: 100,
@@ -617,13 +617,13 @@ impl DQNConfig {
state_dim: 32,
num_actions: 3,
hidden_dims: vec![256, 256, 128], // P1 FIX: Increased capacity for 45-action space (2x output params/action)
learning_rate: 1e-4,
gamma: 0.99,
learning_rate: 3e-5, // Conservative default for stable training
gamma: 0.95,
epsilon_start: 0.3,
epsilon_end: 0.05,
epsilon_decay: 0.995,
replay_buffer_capacity: 10000,
batch_size: 32,
batch_size: 1024, // H100 target (AutoBatchSizer caps for smaller GPUs)
min_replay_size: 1000,
target_update_freq: 1000,
huber_delta: 10.0, // Conservative default (hyperopt can scale to 15-40)
@@ -644,14 +644,14 @@ impl DQNConfig {
per_max_memory_bytes: 4 * 1024 * 1024 * 1024,
dueling_hidden_dim: 64,
num_atoms: 51,
v_min: -25.0, // DSR Q-values: rewards ±2 with gamma=0.92 → Q ≈ ±25
v_max: 25.0, // DSR Q-values: rewards ±2 with gamma=0.92 → Q ≈ ±25
v_min: -50.0, // Reward v6 scale: mark-to-market returns with gamma=0.95
v_max: 50.0, // Reward v6 scale: mark-to-market returns with gamma=0.95
noisy_sigma_init: 0.5,
// BUG #37 FIX: Q-value clipping (prevents step-level explosions)
enable_q_value_clipping: true,
q_value_clip_min: -500.0, // Conservative bounds (±375 baseline × 1.33 safety margin)
q_value_clip_max: 500.0,
q_value_clip_min: -200.0, // Reward v6: tighter than old -500 but covers v_range + safety margin
q_value_clip_max: 200.0,
// WAVE 23 P0 Fix #1: Adaptive gradient collapse threshold
gradient_collapse_multiplier: 100.0,
@@ -718,14 +718,14 @@ impl DQNConfig {
per_max_memory_bytes: 4 * 1024 * 1024 * 1024,
dueling_hidden_dim: 64,
num_atoms: 51,
v_min: -25.0, // DSR Q-values: rewards ±2 with gamma=0.92 → Q ≈ ±25
v_max: 25.0, // DSR Q-values: rewards ±2 with gamma=0.92 → Q ≈ ±25
v_min: -50.0, // Reward v6 scale: mark-to-market returns with gamma=0.95
v_max: 50.0, // Reward v6 scale: mark-to-market returns with gamma=0.95
noisy_sigma_init: 0.5,
// BUG #37 FIX: Q-value clipping (prevents step-level explosions)
enable_q_value_clipping: true,
q_value_clip_min: -500.0, // Conservative bounds for emergency mode
q_value_clip_max: 500.0,
q_value_clip_min: -200.0, // Reward v6: tighter than old -500 but covers v_range + safety margin
q_value_clip_max: 200.0,
// WAVE 23 P0 Fix #1: Adaptive gradient collapse threshold
gradient_collapse_multiplier: 100.0,

10
crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs
View File

@@ -175,7 +175,7 @@ pub struct DqnBacktestConfig {
impl DqnBacktestConfig {
/// Create from the legacy `network_dims` tuple with standard C51 defaults.
///
/// Uses branch sizes [9, 3, 3], num_atoms=51, v_min=-2.0, v_max=2.0.
/// Uses branch sizes [9, 3, 3], num_atoms=51, v_min=-50.0, v_max=50.0.
pub fn from_network_dims(network_dims: (usize, usize, usize, usize)) -> Self {
Self {
shared_h1: network_dims.0,
@@ -186,8 +186,8 @@ impl DqnBacktestConfig {
branch_0_size: 9,
branch_1_size: 3,
branch_2_size: 3,
v_min: -2.0, // Reward v4: must match training v_range
v_max: 2.0,
v_min: -50.0, // Reward v6 scale: mark-to-market returns with gamma=0.95
v_max: 50.0,
}
}
}
@@ -2105,8 +2105,8 @@ mod tests {
assert_eq!(cfg.branch_0_size, 9);
assert_eq!(cfg.branch_1_size, 3);
assert_eq!(cfg.branch_2_size, 3);
assert!((cfg.v_min - (-2.0)).abs() < f32::EPSILON);
assert!((cfg.v_max - 2.0).abs() < f32::EPSILON);
assert!((cfg.v_min - (-50.0)).abs() < f32::EPSILON);
assert!((cfg.v_max - 50.0).abs() < f32::EPSILON);
}
#[test]

16
crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs
View File

@@ -114,9 +114,9 @@ pub struct GpuDqnTrainConfig {
pub adv_h: usize,
/// Number of C51 distributional atoms (default: 51).
pub num_atoms: usize,
/// C51 minimum support value (default: -25.0).
/// C51 minimum support value (default: -50.0).
pub v_min: f32,
/// C51 maximum support value (default: 25.0).
/// C51 maximum support value (default: 50.0).
pub v_max: f32,
/// Branch 0 (exposure) action count (default: 5).
pub branch_0_size: usize,
@@ -169,25 +169,25 @@ pub struct GpuDqnTrainConfig {
impl Default for GpuDqnTrainConfig {
fn default() -> Self {
Self {
state_dim: 72,
state_dim: 48, // Actual aligned feature dim (40 market + 3 portfolio = 43, padded to 48)
shared_h1: 256,
shared_h2: 256,
value_h: 128,
adv_h: 128,
num_atoms: 51,
v_min: -2.0, // Reward v4: tight C51 atoms for per-bar percentage returns
v_max: 2.0,
v_min: -50.0, // Reward v6 scale: mark-to-market returns with gamma=0.95
v_max: 50.0,
branch_0_size: 9,
branch_1_size: 3,
branch_2_size: 3,
batch_size: 256,
batch_size: 64, // Smoke test default (production overrides via from_hyperparams)
gamma: 0.99,
n_steps: 1,
lr: 3e-4,
lr: 3e-5, // Conservative default for stable training
beta1: 0.9,
beta2: 0.999,
epsilon: 1e-8,
weight_decay: 1e-5,
weight_decay: 1e-4, // Standard L2 regularization strength
max_grad_norm: 10.0,
spectral_norm_sigma_max: 3.0,
iqn_lambda: 0.25,

26
crates/ml/src/cuda_pipeline/gpu_experience_collector.rs
View File

@@ -187,9 +187,9 @@ pub struct ExperienceCollectorConfig {
pub tx_cost_multiplier: f32,
/// UCB count-bonus coefficient for GPU action selection (0.0 = disabled)
pub count_bonus_coefficient: f32,
/// Minimum Q-value clamp (e.g. -500.0)
/// Minimum Q-value clamp (e.g. -200.0)
pub q_clip_min: f32,
/// Maximum Q-value clamp (e.g. 500.0)
/// Maximum Q-value clamp (e.g. 200.0)
pub q_clip_max: f32,
/// Huber loss kappa for TD error (robust priority). 0.0 = disabled (raw L1).
pub huber_kappa: f32,
@@ -201,9 +201,9 @@ pub struct ExperienceCollectorConfig {
pub use_distributional: bool,
/// D6: Number of atoms in C51 distribution (default: 51)
pub num_atoms: i32,
/// D6: Minimum value support for C51 (default: -25.0)
/// D6: Minimum value support for C51 (default: -50.0)
pub v_min: f32,
/// D6: Maximum value support for C51 (default: 25.0)
/// D6: Maximum value support for C51 (default: 50.0)
pub v_max: f32,
/// EMA decay rate for per-thread reward normalization in the GPU kernel.
pub reward_norm_alpha: f32,
@@ -259,15 +259,15 @@ impl Default for ExperienceCollectorConfig {
loss_aversion: 1.5,
tx_cost_multiplier: 1.0,
count_bonus_coefficient: 0.0,
q_clip_min: -500.0,
q_clip_max: 500.0,
q_clip_min: -200.0, // Reward v6: tighter than old -500 but covers v_range + safety margin
q_clip_max: 200.0,
huber_kappa: 0.0,
use_noisy_nets: false,
use_noisy_nets: true, // Rainbow DQN: NoisyNet exploration always enabled
noisy_sigma_init: 0.5,
use_distributional: false,
num_atoms: 1,
v_min: -2.0, // Reward v4: tight C51 atoms for per-bar percentage returns
v_max: 2.0,
use_distributional: true, // Rainbow DQN: C51 distributional RL always enabled
num_atoms: 51, // C51 standard atom count
v_min: -50.0, // Reward v6 scale: mark-to-market returns with gamma=0.95
v_max: 50.0,
reward_norm_alpha: 0.0, // Disabled for reward v4: per-bar returns are already naturally scaled
fill_median_spread: 0.0,
fill_median_vol: 0.0,
@@ -826,8 +826,8 @@ impl GpuExperienceCollector {
market_dim,
network_dims,
num_atoms,
v_min: -2.0, // Reward v4: tight C51 atoms, hyperopt overrides
v_max: 2.0,
v_min: -50.0, // Reward v6 scale: mark-to-market returns, hyperopt overrides
v_max: 50.0,
branch_sizes,
alloc_episodes,
alloc_timesteps,