diff --git a/crates/ml/src/trainers/dqn/trainer/constructor.rs b/crates/ml/src/trainers/dqn/trainer/constructor.rs index 63e4439a5..5a25be4bc 100644 --- a/crates/ml/src/trainers/dqn/trainer/constructor.rs +++ b/crates/ml/src/trainers/dqn/trainer/constructor.rs @@ -616,6 +616,8 @@ impl DQNTrainer { trade_history: VecDeque::with_capacity(500), volatility_returns: VecDeque::with_capacity(20), // Use default instead of moved hyperparams trade_stats_history: Vec::new(), + training_sharpe_ema: 0.0, + training_sharpe_ema_initialized: false, // Wave 16 Portfolio Features enable_action_masking, diff --git a/crates/ml/src/trainers/dqn/trainer/mod.rs b/crates/ml/src/trainers/dqn/trainer/mod.rs index 1476e216d..62f246dc0 100644 --- a/crates/ml/src/trainers/dqn/trainer/mod.rs +++ b/crates/ml/src/trainers/dqn/trainer/mod.rs @@ -222,6 +222,12 @@ pub struct DQNTrainer { /// Replaces pnl_history — real trade metrics instead of averaged rewards. pub(crate) trade_stats_history: Vec, + /// Training Sharpe EMA for adaptive DSR reward shaping. + /// Updated at epoch end from GPU experience collection Sharpe. + pub(crate) training_sharpe_ema: f32, + /// Whether training_sharpe_ema has been initialized. + pub(crate) training_sharpe_ema_initialized: bool, + // Wave 16 Portfolio Features /// Enable action masking (filters invalid actions before Q-value computation) pub enable_action_masking: bool, diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 0de3ab865..e82ff6e6f 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -1143,7 +1143,11 @@ impl DQNTrainer { feature_noise_scale: self.hyperparams.feature_noise_scale as f32, vol_normalizer: self.epoch_vol_normalizer, feature_mask: self.epoch_feature_mask.clone(), - w_dsr: self.hyperparams.w_dsr as f32, + w_dsr: { + let w_dsr_base = self.hyperparams.w_dsr as f32; + let adaptive_mult = (1.0_f32 - self.training_sharpe_ema).clamp(0.5, 3.0); + w_dsr_base * adaptive_mult + }, micro_reward_scale: self.hyperparams.micro_reward_scale as f32, current_epoch: self.current_epoch as i32, total_epochs: self.hyperparams.epochs as i32, @@ -2187,6 +2191,20 @@ impl DQNTrainer { self.val_loss_history.push(val_loss); self.sharpe_history.push(epoch_sharpe); + // ── Adaptive DSR: w_dsr = w_dsr_base * clamp(1.0 - sharpe_ema, 0.5, 3.0) ── + { + let sharpe_f32 = epoch_sharpe as f32; + if !self.training_sharpe_ema_initialized { + self.training_sharpe_ema = sharpe_f32; + self.training_sharpe_ema_initialized = true; + } else { + // Adaptive EMA alpha: faster when signal changes magnitude + let err = (sharpe_f32 - self.training_sharpe_ema).abs(); + let alpha = (0.05_f32 + 0.3 * err).min(0.5); + self.training_sharpe_ema = (1.0 - alpha) * self.training_sharpe_ema + alpha * sharpe_f32; + } + } + // Limit history vectors to prevent unbounded growth const MAX_HISTORY_LEN: usize = 100; if self.loss_history.len() > MAX_HISTORY_LEN {