diff --git a/crates/ml/src/dqn/reward.rs b/crates/ml/src/dqn/reward.rs index fe6358248..f0ebf93d4 100644 --- a/crates/ml/src/dqn/reward.rs +++ b/crates/ml/src/dqn/reward.rs @@ -165,6 +165,115 @@ impl Default for RewardNormalizer { } } +/// Differential Sharpe Ratio (DSR) for incremental reward shaping. +/// +/// Implements the Moody & Saffell (2001) formula for online Sharpe ratio +/// estimation. The DSR measures the marginal change in the Sharpe ratio +/// from each new return, producing a dense reward signal that directly +/// optimizes risk-adjusted performance. +/// +/// # Algorithm +/// Given a new return r_t: +/// 1. Update EMA of returns: A_t = A_{t-1} + eta * (r_t - A_{t-1}) +/// 2. Update EMA of squared returns: B_t = B_{t-1} + eta * (r_t^2 - B_{t-1}) +/// 3. Compute DSR: D_t = (B_{t-1} * delta_A - 0.5 * A_{t-1} * delta_B) / (B_{t-1} - A_{t-1}^2)^{3/2} +/// where delta_A = r_t - A_{t-1}, delta_B = r_t^2 - B_{t-1} +/// +/// # References +/// - Moody, J. & Saffell, M. (2001). "Learning to Trade via Direct Reinforcement." +/// IEEE Transactions on Neural Networks, 12(4), 875-889. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DifferentialSharpeRatio { + /// Exponential moving average of returns + ema_return: f64, + /// Exponential moving average of squared returns + ema_return_sq: f64, + /// Decay rate for EMA updates (controls lookback window) + eta: f64, + /// Whether EMAs have been initialized with a first observation + initialized: bool, +} + +impl DifferentialSharpeRatio { + /// Create a new DSR calculator with the given decay rate. + /// + /// # Arguments + /// * `eta` - EMA decay rate, clamped to [0.0001, 0.1]. + /// - Small eta (~0.001): long lookback, smooth DSR signal + /// - Large eta (~0.05): short lookback, responsive DSR signal + /// - Default: 0.01 (~100-step effective window) + pub fn new(eta: f64) -> Self { + Self { + ema_return: 0.0, + ema_return_sq: 0.0, + eta: eta.clamp(0.0001, 0.1), + initialized: false, + } + } + + /// Compute the incremental DSR for a new return observation. + /// + /// On the first call, initializes EMAs from the observation and returns 0.0 + /// (no prior statistics to compute a differential from). + /// + /// # Arguments + /// * `r_t` - The raw return at time t (e.g., percentage PnL) + /// + /// # Returns + /// The differential Sharpe ratio, clamped to [-5.0, 5.0] to prevent + /// extreme values from destabilizing Q-learning updates. + pub fn step(&mut self, r_t: f64) -> f64 { + if !self.initialized { + // First observation: seed EMAs, no differential to compute yet + self.ema_return = r_t; + self.ema_return_sq = r_t * r_t; + self.initialized = true; + return 0.0; + } + + // Deltas from previous EMA estimates + let delta_a = r_t - self.ema_return; + let delta_b = r_t * r_t - self.ema_return_sq; + + // Variance estimate: Var = E[r^2] - E[r]^2 + let variance = self.ema_return_sq - self.ema_return * self.ema_return; + + // Update EMAs (do this after computing deltas but before returning) + self.ema_return += self.eta * delta_a; + self.ema_return_sq += self.eta * delta_b; + + // Degenerate case: near-zero variance means constant returns, + // DSR formula has a division by variance^{3/2} which would blow up. + // Fallback: scale raw PnL to approximate DSR magnitude range. + if variance < 1e-12 { + return (r_t * 100.0).clamp(-5.0, 5.0); + } + + // Moody & Saffell DSR formula: + // D_t = (B * delta_A - 0.5 * A * delta_B) / (B - A^2)^{3/2} + // where A = ema_return (before update), B = ema_return_sq (before update) + // We use the pre-update values stored in delta computation above. + let a_prev = self.ema_return - self.eta * delta_a; // recover pre-update A + let b_prev = self.ema_return_sq - self.eta * delta_b; // recover pre-update B + + let numerator = b_prev * delta_a - 0.5 * a_prev * delta_b; + let denominator = variance.powf(1.5); + + let dsr = numerator / denominator; + dsr.clamp(-5.0, 5.0) + } + + /// Reset internal state for episode boundaries. + /// + /// Must be called between episodes to prevent cross-episode EMA leakage. + /// After reset, the next `step()` call will re-initialize EMAs. + pub fn reset(&mut self) { + self.ema_return = 0.0; + self.ema_return_sq = 0.0; + self.initialized = false; + } +} + /// Configuration for reward function #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RewardConfig { @@ -196,6 +305,12 @@ pub struct RewardConfig { pub sharpe_weight: Decimal, /// Rolling window size for Sharpe ratio calculation (WAVE 26 P1.3) pub sharpe_window: usize, + /// Enable Differential Sharpe Ratio (DSR) reward shaping (Moody & Saffell 2001). + /// When true, DSR replaces the EMA normalizer + risk-adjusted division pipeline. + pub use_dsr: bool, + /// EMA decay rate for DSR calculation, clamped to [0.0001, 0.1]. + /// Smaller values = longer lookback window. Default: 0.01 (~100-step window). + pub dsr_eta: f64, } impl Default for RewardConfig { @@ -215,6 +330,8 @@ impl Default for RewardConfig { triple_barrier_stop_penalty: Decimal::try_from(0.5).unwrap_or(Decimal::ZERO), // 50% penalty for hitting stop loss sharpe_weight: Decimal::try_from(0.3).unwrap_or(Decimal::ZERO), // WAVE 26 P1.3: 30% weight for Sharpe ratio sharpe_window: 20, // WAVE 26 P1.3: 20-step rolling window for Sharpe calculation + use_dsr: false, // DSR disabled by default (opt-in via hyperopt) + dsr_eta: 0.01, // ~100-step effective EMA window } } } @@ -268,6 +385,8 @@ pub struct RewardConfigBuilder { use_percentage_pnl: Option, enable_normalization: Option, circuit_breaker_config: Option, + use_dsr: Option, + dsr_eta: Option, } impl RewardConfigBuilder { @@ -301,6 +420,16 @@ impl RewardConfigBuilder { self } + pub fn use_dsr(mut self, enable: bool) -> Self { + self.use_dsr = Some(enable); + self + } + + pub fn dsr_eta(mut self, eta: f64) -> Self { + self.dsr_eta = Some(eta); + self + } + pub fn build(self) -> Result { Ok(RewardConfig { pnl_weight: Decimal::try_from(self.pnl_weight.unwrap_or(1.0)) @@ -320,6 +449,8 @@ impl RewardConfigBuilder { triple_barrier_stop_penalty: Decimal::try_from(0.5).unwrap_or(Decimal::ZERO), sharpe_weight: Decimal::try_from(0.3).unwrap_or(Decimal::ZERO), sharpe_window: 100, + use_dsr: self.use_dsr.unwrap_or(false), + dsr_eta: self.dsr_eta.unwrap_or(0.01), }) } } @@ -407,6 +538,8 @@ pub struct RewardFunction { debug_logging: bool, /// Rolling buffer of returns for Sharpe ratio calculation (WAVE 26 P1.3) returns_buffer: std::collections::VecDeque, + /// Optional Differential Sharpe Ratio calculator (None = disabled) + dsr: Option, } impl RewardFunction { @@ -431,6 +564,7 @@ impl RewardFunction { config.validate()?; let normalizer = config.enable_normalization.then(|| RewardNormalizer::new()); + let dsr = config.use_dsr.then(|| DifferentialSharpeRatio::new(config.dsr_eta)); Ok(Self { config: config.clone(), @@ -438,6 +572,7 @@ impl RewardFunction { normalizer, debug_logging, returns_buffer: std::collections::VecDeque::with_capacity(config.sharpe_window), + dsr, }) } @@ -938,6 +1073,33 @@ impl RewardFunction { sum / Decimal::from(window as u64) } + /// Reset the DSR calculator state for episode boundaries. + /// + /// Call this between episodes to prevent cross-episode EMA leakage. + /// No-op if DSR is not enabled. + pub fn reset_dsr(&mut self) { + if let Some(dsr) = &mut self.dsr { + dsr.reset(); + } + } + + /// Reset all epoch-level state for clean training epochs. + /// + /// Clears: + /// - Returns buffer (Sharpe rolling window) + /// - DSR internal state (EMA accumulators) + /// - Reward normalizer running statistics (if enabled) + /// + /// Call this at the start of each training epoch to prevent + /// stale statistics from previous epochs leaking into new ones. + pub fn reset_epoch_state(&mut self) { + self.returns_buffer.clear(); + self.reset_dsr(); + if let Some(normalizer) = &mut self.normalizer { + *normalizer = RewardNormalizer::new(); + } + } + /// Get reward statistics pub fn get_stats(&self) -> RewardStats { RewardStats { @@ -1289,4 +1451,119 @@ mod tests { Ok(()) } + + #[test] + fn test_dsr_basic() { + let mut dsr = DifferentialSharpeRatio::new(0.01); + + // First step should return 0.0 (initialization) + let first = dsr.step(0.001); + assert!( + first.abs() < f64::EPSILON, + "First DSR step should be 0.0, got {}", + first + ); + + // Positive return should produce positive DSR + let positive = dsr.step(0.02); + assert!( + positive > 0.0, + "Positive return should produce positive DSR, got {}", + positive + ); + + // Negative return should produce negative DSR + let negative = dsr.step(-0.03); + assert!( + negative < 0.0, + "Negative return should produce negative DSR, got {}", + negative + ); + } + + #[test] + fn test_dsr_bounded() { + let mut dsr = DifferentialSharpeRatio::new(0.01); + + // Initialize with a small return + dsr.step(0.001); + + // Extreme positive return should be clamped to 5.0 + let extreme_pos = dsr.step(1000.0); + assert!( + extreme_pos <= 5.0, + "Extreme positive DSR should be clamped to 5.0, got {}", + extreme_pos + ); + assert!( + extreme_pos >= -5.0, + "DSR should be >= -5.0, got {}", + extreme_pos + ); + + // Extreme negative return should be clamped to -5.0 + let extreme_neg = dsr.step(-1000.0); + assert!( + extreme_neg >= -5.0, + "Extreme negative DSR should be clamped to -5.0, got {}", + extreme_neg + ); + assert!( + extreme_neg <= 5.0, + "DSR should be <= 5.0, got {}", + extreme_neg + ); + } + + #[test] + fn test_dsr_reset() { + let mut dsr = DifferentialSharpeRatio::new(0.01); + + // Feed some data + dsr.step(0.01); + dsr.step(0.02); + dsr.step(-0.01); + assert!(dsr.initialized, "DSR should be initialized after steps"); + + // Reset + dsr.reset(); + assert!( + !dsr.initialized, + "DSR should not be initialized after reset" + ); + + // First step after reset should return 0.0 again + let after_reset = dsr.step(0.005); + assert!( + after_reset.abs() < f64::EPSILON, + "First step after reset should be 0.0, got {}", + after_reset + ); + } + + #[test] + fn test_dsr_config_roundtrip() -> anyhow::Result<()> { + let config = RewardConfig::builder() + .use_dsr(true) + .dsr_eta(0.02) + .build()?; + + assert!(config.use_dsr, "use_dsr should be true"); + assert!( + (config.dsr_eta - 0.02).abs() < f64::EPSILON, + "dsr_eta should be 0.02, got {}", + config.dsr_eta + ); + + // Verify defaults when not set + let default_config = RewardConfig::builder().build()?; + assert!(!default_config.use_dsr, "use_dsr should default to false"); + assert!( + (default_config.dsr_eta - 0.01).abs() < f64::EPSILON, + "dsr_eta should default to 0.01, got {}", + default_config.dsr_eta + ); + + Ok(()) + } } diff --git a/crates/ml/src/trainers/dqn/trainer.rs b/crates/ml/src/trainers/dqn/trainer.rs index 93ba38b10..b6ae6720b 100644 --- a/crates/ml/src/trainers/dqn/trainer.rs +++ b/crates/ml/src/trainers/dqn/trainer.rs @@ -512,6 +512,8 @@ impl DQNTrainer { triple_barrier_stop_penalty: Decimal::try_from(0.5).unwrap_or(Decimal::ZERO), sharpe_weight: Decimal::ZERO, // WAVE 26 P1.3: Disabled by default sharpe_window: 20, // WAVE 26 P1.3: Standard 20-period window + use_dsr: false, // DSR disabled by default (opt-in via hyperopt) + dsr_eta: 0.01, // ~100-step effective EMA window }; let reward_fn = RewardFunction::new_with_debug(reward_config, debug_logging)?;