diff --git a/crates/ml-dqn/src/dqn.rs b/crates/ml-dqn/src/dqn.rs index a72357a88..c04111e33 100644 --- a/crates/ml-dqn/src/dqn.rs +++ b/crates/ml-dqn/src/dqn.rs @@ -244,14 +244,14 @@ impl Default for DQNConfig { state_dim: 48, // 40 market + 3 portfolio = 43, padded to 48 for tensor core alignment num_actions: 9, // 9 exposure levels (25% steps) hidden_dims: vec![256, 256], - learning_rate: 1e-4, - gamma: 0.99, + learning_rate: 3e-5, // Conservative default for stable training (matches conservative()) + gamma: 0.95, epsilon_start: 1.0, epsilon_end: 0.01, epsilon_decay: 0.995, replay_buffer_capacity: 100_000, collapse_warmup_capacity: 100_000, - batch_size: 64, + batch_size: 1024, // H100 target (AutoBatchSizer caps for smaller GPUs) min_replay_size: 1000, target_update_freq: 1000, huber_delta: 1.0, @@ -272,12 +272,12 @@ impl Default for DQNConfig { per_max_memory_bytes: 4 * 1024 * 1024 * 1024, dueling_hidden_dim: 128, num_atoms: 51, - v_min: -25.0, // DSR Q-values: rewards ±2 with gamma=0.92 → Q ≈ ±25 - v_max: 25.0, // DSR Q-values: rewards ±2 with gamma=0.92 → Q ≈ ±25 + v_min: -50.0, // Reward v6 scale: mark-to-market returns with gamma=0.95 + v_max: 50.0, // Reward v6 scale: mark-to-market returns with gamma=0.95 noisy_sigma_init: 0.5, enable_q_value_clipping: true, - q_value_clip_min: -100.0, - q_value_clip_max: 100.0, + q_value_clip_min: -200.0, // Reward v6: tighter than old -500 but covers v_range + safety margin + q_value_clip_max: 200.0, gradient_collapse_multiplier: 2.0, gradient_collapse_patience: 100, @@ -617,13 +617,13 @@ impl DQNConfig { state_dim: 32, num_actions: 3, hidden_dims: vec![256, 256, 128], // P1 FIX: Increased capacity for 45-action space (2x output params/action) - learning_rate: 1e-4, - gamma: 0.99, + learning_rate: 3e-5, // Conservative default for stable training + gamma: 0.95, epsilon_start: 0.3, epsilon_end: 0.05, epsilon_decay: 0.995, replay_buffer_capacity: 10000, - batch_size: 32, + batch_size: 1024, // H100 target (AutoBatchSizer caps for smaller GPUs) min_replay_size: 1000, target_update_freq: 1000, huber_delta: 10.0, // Conservative default (hyperopt can scale to 15-40) @@ -644,14 +644,14 @@ impl DQNConfig { per_max_memory_bytes: 4 * 1024 * 1024 * 1024, dueling_hidden_dim: 64, num_atoms: 51, - v_min: -25.0, // DSR Q-values: rewards ±2 with gamma=0.92 → Q ≈ ±25 - v_max: 25.0, // DSR Q-values: rewards ±2 with gamma=0.92 → Q ≈ ±25 + v_min: -50.0, // Reward v6 scale: mark-to-market returns with gamma=0.95 + v_max: 50.0, // Reward v6 scale: mark-to-market returns with gamma=0.95 noisy_sigma_init: 0.5, // BUG #37 FIX: Q-value clipping (prevents step-level explosions) enable_q_value_clipping: true, - q_value_clip_min: -500.0, // Conservative bounds (±375 baseline × 1.33 safety margin) - q_value_clip_max: 500.0, + q_value_clip_min: -200.0, // Reward v6: tighter than old -500 but covers v_range + safety margin + q_value_clip_max: 200.0, // WAVE 23 P0 Fix #1: Adaptive gradient collapse threshold gradient_collapse_multiplier: 100.0, @@ -718,14 +718,14 @@ impl DQNConfig { per_max_memory_bytes: 4 * 1024 * 1024 * 1024, dueling_hidden_dim: 64, num_atoms: 51, - v_min: -25.0, // DSR Q-values: rewards ±2 with gamma=0.92 → Q ≈ ±25 - v_max: 25.0, // DSR Q-values: rewards ±2 with gamma=0.92 → Q ≈ ±25 + v_min: -50.0, // Reward v6 scale: mark-to-market returns with gamma=0.95 + v_max: 50.0, // Reward v6 scale: mark-to-market returns with gamma=0.95 noisy_sigma_init: 0.5, // BUG #37 FIX: Q-value clipping (prevents step-level explosions) enable_q_value_clipping: true, - q_value_clip_min: -500.0, // Conservative bounds for emergency mode - q_value_clip_max: 500.0, + q_value_clip_min: -200.0, // Reward v6: tighter than old -500 but covers v_range + safety margin + q_value_clip_max: 200.0, // WAVE 23 P0 Fix #1: Adaptive gradient collapse threshold gradient_collapse_multiplier: 100.0, diff --git a/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs b/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs index 8bc891109..684d1c842 100644 --- a/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs +++ b/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs @@ -175,7 +175,7 @@ pub struct DqnBacktestConfig { impl DqnBacktestConfig { /// Create from the legacy `network_dims` tuple with standard C51 defaults. /// - /// Uses branch sizes [9, 3, 3], num_atoms=51, v_min=-2.0, v_max=2.0. + /// Uses branch sizes [9, 3, 3], num_atoms=51, v_min=-50.0, v_max=50.0. pub fn from_network_dims(network_dims: (usize, usize, usize, usize)) -> Self { Self { shared_h1: network_dims.0, @@ -186,8 +186,8 @@ impl DqnBacktestConfig { branch_0_size: 9, branch_1_size: 3, branch_2_size: 3, - v_min: -2.0, // Reward v4: must match training v_range - v_max: 2.0, + v_min: -50.0, // Reward v6 scale: mark-to-market returns with gamma=0.95 + v_max: 50.0, } } } @@ -2105,8 +2105,8 @@ mod tests { assert_eq!(cfg.branch_0_size, 9); assert_eq!(cfg.branch_1_size, 3); assert_eq!(cfg.branch_2_size, 3); - assert!((cfg.v_min - (-2.0)).abs() < f32::EPSILON); - assert!((cfg.v_max - 2.0).abs() < f32::EPSILON); + assert!((cfg.v_min - (-50.0)).abs() < f32::EPSILON); + assert!((cfg.v_max - 50.0).abs() < f32::EPSILON); } #[test] diff --git a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs index 147f673c2..80064e006 100644 --- a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs +++ b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs @@ -114,9 +114,9 @@ pub struct GpuDqnTrainConfig { pub adv_h: usize, /// Number of C51 distributional atoms (default: 51). pub num_atoms: usize, - /// C51 minimum support value (default: -25.0). + /// C51 minimum support value (default: -50.0). pub v_min: f32, - /// C51 maximum support value (default: 25.0). + /// C51 maximum support value (default: 50.0). pub v_max: f32, /// Branch 0 (exposure) action count (default: 5). pub branch_0_size: usize, @@ -169,25 +169,25 @@ pub struct GpuDqnTrainConfig { impl Default for GpuDqnTrainConfig { fn default() -> Self { Self { - state_dim: 72, + state_dim: 48, // Actual aligned feature dim (40 market + 3 portfolio = 43, padded to 48) shared_h1: 256, shared_h2: 256, value_h: 128, adv_h: 128, num_atoms: 51, - v_min: -2.0, // Reward v4: tight C51 atoms for per-bar percentage returns - v_max: 2.0, + v_min: -50.0, // Reward v6 scale: mark-to-market returns with gamma=0.95 + v_max: 50.0, branch_0_size: 9, branch_1_size: 3, branch_2_size: 3, - batch_size: 256, + batch_size: 64, // Smoke test default (production overrides via from_hyperparams) gamma: 0.99, n_steps: 1, - lr: 3e-4, + lr: 3e-5, // Conservative default for stable training beta1: 0.9, beta2: 0.999, epsilon: 1e-8, - weight_decay: 1e-5, + weight_decay: 1e-4, // Standard L2 regularization strength max_grad_norm: 10.0, spectral_norm_sigma_max: 3.0, iqn_lambda: 0.25, diff --git a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs index e6f9e1e14..d2769e512 100644 --- a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs +++ b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs @@ -187,9 +187,9 @@ pub struct ExperienceCollectorConfig { pub tx_cost_multiplier: f32, /// UCB count-bonus coefficient for GPU action selection (0.0 = disabled) pub count_bonus_coefficient: f32, - /// Minimum Q-value clamp (e.g. -500.0) + /// Minimum Q-value clamp (e.g. -200.0) pub q_clip_min: f32, - /// Maximum Q-value clamp (e.g. 500.0) + /// Maximum Q-value clamp (e.g. 200.0) pub q_clip_max: f32, /// Huber loss kappa for TD error (robust priority). 0.0 = disabled (raw L1). pub huber_kappa: f32, @@ -201,9 +201,9 @@ pub struct ExperienceCollectorConfig { pub use_distributional: bool, /// D6: Number of atoms in C51 distribution (default: 51) pub num_atoms: i32, - /// D6: Minimum value support for C51 (default: -25.0) + /// D6: Minimum value support for C51 (default: -50.0) pub v_min: f32, - /// D6: Maximum value support for C51 (default: 25.0) + /// D6: Maximum value support for C51 (default: 50.0) pub v_max: f32, /// EMA decay rate for per-thread reward normalization in the GPU kernel. pub reward_norm_alpha: f32, @@ -259,15 +259,15 @@ impl Default for ExperienceCollectorConfig { loss_aversion: 1.5, tx_cost_multiplier: 1.0, count_bonus_coefficient: 0.0, - q_clip_min: -500.0, - q_clip_max: 500.0, + q_clip_min: -200.0, // Reward v6: tighter than old -500 but covers v_range + safety margin + q_clip_max: 200.0, huber_kappa: 0.0, - use_noisy_nets: false, + use_noisy_nets: true, // Rainbow DQN: NoisyNet exploration always enabled noisy_sigma_init: 0.5, - use_distributional: false, - num_atoms: 1, - v_min: -2.0, // Reward v4: tight C51 atoms for per-bar percentage returns - v_max: 2.0, + use_distributional: true, // Rainbow DQN: C51 distributional RL always enabled + num_atoms: 51, // C51 standard atom count + v_min: -50.0, // Reward v6 scale: mark-to-market returns with gamma=0.95 + v_max: 50.0, reward_norm_alpha: 0.0, // Disabled for reward v4: per-bar returns are already naturally scaled fill_median_spread: 0.0, fill_median_vol: 0.0, @@ -826,8 +826,8 @@ impl GpuExperienceCollector { market_dim, network_dims, num_atoms, - v_min: -2.0, // Reward v4: tight C51 atoms, hyperopt overrides - v_max: 2.0, + v_min: -50.0, // Reward v6 scale: mark-to-market returns, hyperopt overrides + v_max: 50.0, branch_sizes, alloc_episodes, alloc_timesteps,