gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(ml): add Differential Sharpe Ratio (DSR) struct and config fields

Browse Source 
Implement the Moody & Saffell (2001) DSR for incremental reward shaping
that directly optimizes risk-adjusted returns. This replaces the broken
double-normalization pipeline (EMA normalizer + risk-adjusted division)
that was producing random noise and preventing DQN learning.

- DifferentialSharpeRatio struct: step(), reset(), eta clamping, +/-5 bounds
- RewardConfig: use_dsr (default false), dsr_eta (default 0.01)
- RewardConfigBuilder: use_dsr() and dsr_eta() builder methods
- RewardFunction: dsr field, reset_dsr(), reset_epoch_state()
- 4 new unit tests (basic, bounded, reset, config roundtrip)
- All 40 reward tests pass

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-07 10:42:03 +01:00
parent c63611fb03
commit 1b40fecccd
2 changed files with 279 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

277
crates/ml/src/dqn/reward.rs
View File

@@ -165,6 +165,115 @@ impl Default for RewardNormalizer {
}
}
/// Differential Sharpe Ratio (DSR) for incremental reward shaping.
///
/// Implements the Moody & Saffell (2001) formula for online Sharpe ratio
/// estimation. The DSR measures the marginal change in the Sharpe ratio
/// from each new return, producing a dense reward signal that directly
/// optimizes risk-adjusted performance.
///
/// # Algorithm
/// Given a new return r_t:
/// 1. Update EMA of returns: A_t = A_{t-1} + eta * (r_t - A_{t-1})
/// 2. Update EMA of squared returns: B_t = B_{t-1} + eta * (r_t^2 - B_{t-1})
/// 3. Compute DSR: D_t = (B_{t-1} * delta_A - 0.5 * A_{t-1} * delta_B) / (B_{t-1} - A_{t-1}^2)^{3/2}
/// where delta_A = r_t - A_{t-1}, delta_B = r_t^2 - B_{t-1}
///
/// # References
/// - Moody, J. & Saffell, M. (2001). "Learning to Trade via Direct Reinforcement."
/// IEEE Transactions on Neural Networks, 12(4), 875-889.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DifferentialSharpeRatio {
/// Exponential moving average of returns
ema_return: f64,
/// Exponential moving average of squared returns
ema_return_sq: f64,
/// Decay rate for EMA updates (controls lookback window)
eta: f64,
/// Whether EMAs have been initialized with a first observation
initialized: bool,
}
impl DifferentialSharpeRatio {
/// Create a new DSR calculator with the given decay rate.
///
/// # Arguments
/// * `eta` - EMA decay rate, clamped to [0.0001, 0.1].
/// - Small eta (~0.001): long lookback, smooth DSR signal
/// - Large eta (~0.05): short lookback, responsive DSR signal
/// - Default: 0.01 (~100-step effective window)
pub fn new(eta: f64) -> Self {
Self {
ema_return: 0.0,
ema_return_sq: 0.0,
eta: eta.clamp(0.0001, 0.1),
initialized: false,
}
}
/// Compute the incremental DSR for a new return observation.
///
/// On the first call, initializes EMAs from the observation and returns 0.0
/// (no prior statistics to compute a differential from).
///
/// # Arguments
/// * `r_t` - The raw return at time t (e.g., percentage PnL)
///
/// # Returns
/// The differential Sharpe ratio, clamped to [-5.0, 5.0] to prevent
/// extreme values from destabilizing Q-learning updates.
pub fn step(&mut self, r_t: f64) -> f64 {
if !self.initialized {
// First observation: seed EMAs, no differential to compute yet
self.ema_return = r_t;
self.ema_return_sq = r_t * r_t;
self.initialized = true;
return 0.0;
}
// Deltas from previous EMA estimates
let delta_a = r_t - self.ema_return;
let delta_b = r_t * r_t - self.ema_return_sq;
// Variance estimate: Var = E[r^2] - E[r]^2
let variance = self.ema_return_sq - self.ema_return * self.ema_return;
// Update EMAs (do this after computing deltas but before returning)
self.ema_return += self.eta * delta_a;
self.ema_return_sq += self.eta * delta_b;
// Degenerate case: near-zero variance means constant returns,
// DSR formula has a division by variance^{3/2} which would blow up.
// Fallback: scale raw PnL to approximate DSR magnitude range.
if variance < 1e-12 {
return (r_t * 100.0).clamp(-5.0, 5.0);
}
// Moody & Saffell DSR formula:
// D_t = (B * delta_A - 0.5 * A * delta_B) / (B - A^2)^{3/2}
// where A = ema_return (before update), B = ema_return_sq (before update)
// We use the pre-update values stored in delta computation above.
let a_prev = self.ema_return - self.eta * delta_a; // recover pre-update A
let b_prev = self.ema_return_sq - self.eta * delta_b; // recover pre-update B
let numerator = b_prev * delta_a - 0.5 * a_prev * delta_b;
let denominator = variance.powf(1.5);
let dsr = numerator / denominator;
dsr.clamp(-5.0, 5.0)
}
/// Reset internal state for episode boundaries.
///
/// Must be called between episodes to prevent cross-episode EMA leakage.
/// After reset, the next `step()` call will re-initialize EMAs.
pub fn reset(&mut self) {
self.ema_return = 0.0;
self.ema_return_sq = 0.0;
self.initialized = false;
}
}
/// Configuration for reward function
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RewardConfig {
@@ -196,6 +305,12 @@ pub struct RewardConfig {
pub sharpe_weight: Decimal,
/// Rolling window size for Sharpe ratio calculation (WAVE 26 P1.3)
pub sharpe_window: usize,
/// Enable Differential Sharpe Ratio (DSR) reward shaping (Moody & Saffell 2001).
/// When true, DSR replaces the EMA normalizer + risk-adjusted division pipeline.
pub use_dsr: bool,
/// EMA decay rate for DSR calculation, clamped to [0.0001, 0.1].
/// Smaller values = longer lookback window. Default: 0.01 (~100-step window).
pub dsr_eta: f64,
}
impl Default for RewardConfig {
@@ -215,6 +330,8 @@ impl Default for RewardConfig {
triple_barrier_stop_penalty: Decimal::try_from(0.5).unwrap_or(Decimal::ZERO), // 50% penalty for hitting stop loss
sharpe_weight: Decimal::try_from(0.3).unwrap_or(Decimal::ZERO), // WAVE 26 P1.3: 30% weight for Sharpe ratio
sharpe_window: 20, // WAVE 26 P1.3: 20-step rolling window for Sharpe calculation
use_dsr: false, // DSR disabled by default (opt-in via hyperopt)
dsr_eta: 0.01, // ~100-step effective EMA window
}
}
}
@@ -268,6 +385,8 @@ pub struct RewardConfigBuilder {
use_percentage_pnl: Option<bool>,
enable_normalization: Option<bool>,
circuit_breaker_config: Option<CircuitBreakerConfig>,
use_dsr: Option<bool>,
dsr_eta: Option<f64>,
}
impl RewardConfigBuilder {
@@ -301,6 +420,16 @@ impl RewardConfigBuilder {
self
}
pub fn use_dsr(mut self, enable: bool) -> Self {
self.use_dsr = Some(enable);
self
}
pub fn dsr_eta(mut self, eta: f64) -> Self {
self.dsr_eta = Some(eta);
self
}
pub fn build(self) -> Result<RewardConfig, MLError> {
Ok(RewardConfig {
pnl_weight: Decimal::try_from(self.pnl_weight.unwrap_or(1.0))
@@ -320,6 +449,8 @@ impl RewardConfigBuilder {
triple_barrier_stop_penalty: Decimal::try_from(0.5).unwrap_or(Decimal::ZERO),
sharpe_weight: Decimal::try_from(0.3).unwrap_or(Decimal::ZERO),
sharpe_window: 100,
use_dsr: self.use_dsr.unwrap_or(false),
dsr_eta: self.dsr_eta.unwrap_or(0.01),
})
}
}
@@ -407,6 +538,8 @@ pub struct RewardFunction {
debug_logging: bool,
/// Rolling buffer of returns for Sharpe ratio calculation (WAVE 26 P1.3)
returns_buffer: std::collections::VecDeque<f64>,
/// Optional Differential Sharpe Ratio calculator (None = disabled)
dsr: Option<DifferentialSharpeRatio>,
}
impl RewardFunction {
@@ -431,6 +564,7 @@ impl RewardFunction {
config.validate()?;
let normalizer = config.enable_normalization.then(|| RewardNormalizer::new());
let dsr = config.use_dsr.then(|| DifferentialSharpeRatio::new(config.dsr_eta));
Ok(Self {
config: config.clone(),
@@ -438,6 +572,7 @@ impl RewardFunction {
normalizer,
debug_logging,
returns_buffer: std::collections::VecDeque::with_capacity(config.sharpe_window),
dsr,
})
}
@@ -938,6 +1073,33 @@ impl RewardFunction {
sum / Decimal::from(window as u64)
}
/// Reset the DSR calculator state for episode boundaries.
///
/// Call this between episodes to prevent cross-episode EMA leakage.
/// No-op if DSR is not enabled.
pub fn reset_dsr(&mut self) {
if let Some(dsr) = &mut self.dsr {
dsr.reset();
}
}
/// Reset all epoch-level state for clean training epochs.
///
/// Clears:
/// - Returns buffer (Sharpe rolling window)
/// - DSR internal state (EMA accumulators)
/// - Reward normalizer running statistics (if enabled)
///
/// Call this at the start of each training epoch to prevent
/// stale statistics from previous epochs leaking into new ones.
pub fn reset_epoch_state(&mut self) {
self.returns_buffer.clear();
self.reset_dsr();
if let Some(normalizer) = &mut self.normalizer {
*normalizer = RewardNormalizer::new();
}
}
/// Get reward statistics
pub fn get_stats(&self) -> RewardStats {
RewardStats {
@@ -1289,4 +1451,119 @@ mod tests {
Ok(())
}
#[test]
fn test_dsr_basic() {
let mut dsr = DifferentialSharpeRatio::new(0.01);
// First step should return 0.0 (initialization)
let first = dsr.step(0.001);
assert!(
first.abs() < f64::EPSILON,
"First DSR step should be 0.0, got {}",
first
);
// Positive return should produce positive DSR
let positive = dsr.step(0.02);
assert!(
positive > 0.0,
"Positive return should produce positive DSR, got {}",
positive
);
// Negative return should produce negative DSR
let negative = dsr.step(-0.03);
assert!(
negative < 0.0,
"Negative return should produce negative DSR, got {}",
negative
);
}
#[test]
fn test_dsr_bounded() {
let mut dsr = DifferentialSharpeRatio::new(0.01);
// Initialize with a small return
dsr.step(0.001);
// Extreme positive return should be clamped to 5.0
let extreme_pos = dsr.step(1000.0);
assert!(
extreme_pos <= 5.0,
"Extreme positive DSR should be clamped to 5.0, got {}",
extreme_pos
);
assert!(
extreme_pos >= -5.0,
"DSR should be >= -5.0, got {}",
extreme_pos
);
// Extreme negative return should be clamped to -5.0
let extreme_neg = dsr.step(-1000.0);
assert!(
extreme_neg >= -5.0,
"Extreme negative DSR should be clamped to -5.0, got {}",
extreme_neg
);
assert!(
extreme_neg <= 5.0,
"DSR should be <= 5.0, got {}",
extreme_neg
);
}
#[test]
fn test_dsr_reset() {
let mut dsr = DifferentialSharpeRatio::new(0.01);
// Feed some data
dsr.step(0.01);
dsr.step(0.02);
dsr.step(-0.01);
assert!(dsr.initialized, "DSR should be initialized after steps");
// Reset
dsr.reset();
assert!(
!dsr.initialized,
"DSR should not be initialized after reset"
);
// First step after reset should return 0.0 again
let after_reset = dsr.step(0.005);
assert!(
after_reset.abs() < f64::EPSILON,
"First step after reset should be 0.0, got {}",
after_reset
);
}
#[test]
fn test_dsr_config_roundtrip() -> anyhow::Result<()> {
let config = RewardConfig::builder()
.use_dsr(true)
.dsr_eta(0.02)
.build()?;
assert!(config.use_dsr, "use_dsr should be true");
assert!(
(config.dsr_eta - 0.02).abs() < f64::EPSILON,
"dsr_eta should be 0.02, got {}",
config.dsr_eta
);
// Verify defaults when not set
let default_config = RewardConfig::builder().build()?;
assert!(!default_config.use_dsr, "use_dsr should default to false");
assert!(
(default_config.dsr_eta - 0.01).abs() < f64::EPSILON,
"dsr_eta should default to 0.01, got {}",
default_config.dsr_eta
);
Ok(())
}
}

2
crates/ml/src/trainers/dqn/trainer.rs
View File

@@ -512,6 +512,8 @@ impl DQNTrainer {
triple_barrier_stop_penalty: Decimal::try_from(0.5).unwrap_or(Decimal::ZERO),
sharpe_weight: Decimal::ZERO, // WAVE 26 P1.3: Disabled by default
sharpe_window: 20, // WAVE 26 P1.3: Standard 20-period window
use_dsr: false, // DSR disabled by default (opt-in via hyperopt)
dsr_eta: 0.01, // ~100-step effective EMA window
};
let reward_fn = RewardFunction::new_with_debug(reward_config, debug_logging)?;