From e2427ea1efb07e85ba53305403c6958787b3f77e Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Wed, 15 Apr 2026 21:11:03 +0200 Subject: [PATCH] =?UTF-8?q?feat(G13):=20Sharpe-aware=20reward=20shaping=20?= =?UTF-8?q?=E2=80=94=20normalize=20by=20rolling=20volatility?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reward rank normalization now operates on Sharpe contributions (return - mean) / std instead of raw returns. Aligns reward signal with Sharpe ratio goal. High-return trades during volatile periods get lower rank weight than same return during calm periods. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../cuda_pipeline/gpu_experience_collector.rs | 7 ++++-- .../cuda_pipeline/reward_shaping_kernel.cu | 25 +++++++++++++------ .../src/trainers/dqn/trainer/constructor.rs | 1 + crates/ml/src/trainers/dqn/trainer/mod.rs | 2 ++ .../src/trainers/dqn/trainer/training_loop.rs | 6 +++-- 5 files changed, 30 insertions(+), 11 deletions(-) diff --git a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs index 5a4c2764a..3a92f700d 100644 --- a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs +++ b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs @@ -2428,11 +2428,13 @@ impl GpuExperienceCollector { &self.rewards_out } - /// Rank-normalize rewards: r_shaped = sign(r) * rank(|r|) * std_ema. + /// G13: Sharpe-aware rank-normalize rewards. + /// Computes sharpe[i] = (r[i] - mean) / std, then ranks by |sharpe|. + /// r_shaped[i] = sign(sharpe[i]) * rank(|sharpe[i]|). /// Uses rewards_out as read-only source, writes shaped values to `rewards_dst`. /// Two-buffer design avoids cross-block race condition (in-place would race /// because __syncthreads is block-local but ranking reads ALL rewards). - pub fn shape_rewards(&self, rewards_dst: &mut CudaSlice, n: usize, reward_std: f32) -> Result<(), MLError> { + pub fn shape_rewards(&self, rewards_dst: &mut CudaSlice, n: usize, reward_std: f32, reward_mean: f32) -> Result<(), MLError> { if n == 0 { return Ok(()); } let n_i32 = n as i32; let blocks = ((n + 255) / 256) as u32; @@ -2443,6 +2445,7 @@ impl GpuExperienceCollector { .arg(rewards_dst) // output shaped rewards (separate buffer) .arg(&n_i32) .arg(&reward_std) + .arg(&reward_mean) // G13: running mean for Sharpe contribution .launch(LaunchConfig { grid_dim: (blocks, 1, 1), block_dim: (256, 1, 1), diff --git a/crates/ml/src/cuda_pipeline/reward_shaping_kernel.cu b/crates/ml/src/cuda_pipeline/reward_shaping_kernel.cu index 0ce5a7dbb..02e2c6d96 100644 --- a/crates/ml/src/cuda_pipeline/reward_shaping_kernel.cu +++ b/crates/ml/src/cuda_pipeline/reward_shaping_kernel.cu @@ -1,7 +1,8 @@ /** * Rank-preserving signed reward standardization — tiled shared-memory. * - * r_shaped[i] = sign(r[i]) * rank(|r[i]|) * std_ema + * r_shaped[i] = sign(sharpe[i]) * rank(|sharpe[i]|) + * where sharpe[i] = (r[i] - mean_ema) / std_ema (G13 Sharpe contribution) * * rank(|r[i]|) = count(|r[j]| <= |r[i]| for all j in batch) / N * @@ -21,12 +22,18 @@ extern "C" __global__ void reward_rank_normalize( const float* __restrict__ rewards_in, /* [N] read-only raw rewards */ float* __restrict__ rewards_out, /* [N] output shaped rewards */ int N, - float std_ema /* running std of raw rewards */ + float std_ema, /* running std of raw rewards */ + float return_mean_ema /* running mean of raw rewards */ ) { int i = blockIdx.x * blockDim.x + threadIdx.x; - /* Each thread loads its own |reward| once from global memory */ - float r_i = (i < N) ? rewards_in[i] : 0.0f; + /* Each thread loads its own reward from global memory */ + float raw_r = (i < N) ? rewards_in[i] : 0.0f; + /* G13: Sharpe contribution = (return - mean) / std. + * Rank normalization operates on risk-adjusted contributions, + * so high-return trades during volatile periods get lower + * rank weight than the same return during calm periods. */ + float r_i = (std_ema > 1e-6f) ? (raw_r - return_mean_ema) / std_ema : raw_r; float abs_i = fabsf(r_i); __shared__ float tile[TILE_SIZE]; @@ -37,7 +44,10 @@ extern "C" __global__ void reward_rank_normalize( for (int t = 0; t < num_tiles; t++) { /* Cooperative tile load: each thread loads one element */ int tile_idx = t * TILE_SIZE + threadIdx.x; - tile[threadIdx.x] = (tile_idx < N) ? fabsf(rewards_in[tile_idx]) : 1e30f; + /* G13: tile stores |sharpe| not |raw| — must match abs_i computation */ + float tile_raw = (tile_idx < N) ? rewards_in[tile_idx] : 0.0f; + float tile_sharpe = (std_ema > 1e-6f) ? (tile_raw - return_mean_ema) / std_ema : tile_raw; + tile[threadIdx.x] = (tile_idx < N) ? fabsf(tile_sharpe) : 1e30f; __syncthreads(); /* Count within tile — shared memory scan, no global reads */ @@ -52,8 +62,9 @@ extern "C" __global__ void reward_rank_normalize( if (i >= N) return; - /* Rank in [0, 1], sign preserved, scaled by std_ema */ + /* Rank in [0, 1], sign of Sharpe contribution preserved */ float rank = (float)count / (float)N; float sign = (r_i > 0.0f) ? 1.0f : (r_i < 0.0f) ? -1.0f : 0.0f; - rewards_out[i] = sign * rank * fmaxf(std_ema, 1e-6f); + /* G13: input is already in Sharpe units — no rescale by std_ema needed */ + rewards_out[i] = sign * rank; } diff --git a/crates/ml/src/trainers/dqn/trainer/constructor.rs b/crates/ml/src/trainers/dqn/trainer/constructor.rs index 5a25be4bc..fe761dd16 100644 --- a/crates/ml/src/trainers/dqn/trainer/constructor.rs +++ b/crates/ml/src/trainers/dqn/trainer/constructor.rs @@ -718,6 +718,7 @@ impl DQNTrainer { epoch_atom_entropy: 0.0, epoch_atom_utilization: 0.0, observed_reward_std: 0.0, + observed_reward_mean: 0.0, // GPU pipeline: pre-uploaded training data (initialized lazily at first epoch) gpu_data: None, diff --git a/crates/ml/src/trainers/dqn/trainer/mod.rs b/crates/ml/src/trainers/dqn/trainer/mod.rs index 62f246dc0..e63415f41 100644 --- a/crates/ml/src/trainers/dqn/trainer/mod.rs +++ b/crates/ml/src/trainers/dqn/trainer/mod.rs @@ -349,6 +349,8 @@ pub struct DQNTrainer { pub(crate) epoch_atom_utilization: f32, /// Observed reward std from experience collector (for adaptive v_range floor). pub(crate) observed_reward_std: f32, + /// G13: Observed reward mean from experience collector (for Sharpe-aware shaping). + pub(crate) observed_reward_mean: f32, /// Running ratio of low-drawdown epochs (EMA of binary signal) pub(crate) low_dd_ratio: f64, /// Whether adversarial regime is active this epoch diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 6ab9a291d..fee05c260 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -1179,12 +1179,13 @@ impl DQNTrainer { let count = gpu_batch.n_episodes * gpu_batch.timesteps * 2; // counterfactual doubles experiences - // Rank-normalize rewards: r_shaped = sign(r) * rank(|r|) * std_ema + // G13: Sharpe-aware rank-normalize rewards. + // Computes sharpe[i] = (r - mean) / std, then ranks by |sharpe|. // Amplifies reward SNR from 0.001 to ~0.5 — counting-sort rank is outlier-resistant. // Skip epoch 0: observed_reward_std is 0.0 until GPU monitoring computes it at epoch end. // Raw rewards flow through epoch 0; rank normalization kicks in from epoch 1+. if self.observed_reward_std > 1e-8 { - collector.shape_rewards(&mut gpu_batch.rewards, count, self.observed_reward_std) + collector.shape_rewards(&mut gpu_batch.rewards, count, self.observed_reward_std, self.observed_reward_mean) .map_err(|e| anyhow::anyhow!("reward rank normalize: {e}"))?; } @@ -1658,6 +1659,7 @@ impl DQNTrainer { ); monitor.track_reward(summary.mean_reward); self.observed_reward_std = summary.reward_std; + self.observed_reward_mean = summary.mean_reward; // Feed all 3 branch distributions from GPU into the monitor. // These are the actual model-selected actions, not // deterministic OrderRouter re-derivations.