feat(G13): Sharpe-aware reward shaping — normalize by rolling volatility

Reward rank normalization now operates on Sharpe contributions
(return - mean) / std instead of raw returns. Aligns reward signal
with Sharpe ratio goal. High-return trades during volatile periods
get lower rank weight than same return during calm periods.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

crates/ml/src/cuda_pipeline/gpu_experience_collector.rs

@@ -2428,11 +2428,13 @@ impl GpuExperienceCollector {
         &self.rewards_out
     }
 
-    /// Rank-normalize rewards: r_shaped = sign(r) * rank(|r|) * std_ema.
+    /// G13: Sharpe-aware rank-normalize rewards.
+    /// Computes sharpe[i] = (r[i] - mean) / std, then ranks by |sharpe|.
+    /// r_shaped[i] = sign(sharpe[i]) * rank(|sharpe[i]|).
     /// Uses rewards_out as read-only source, writes shaped values to `rewards_dst`.
     /// Two-buffer design avoids cross-block race condition (in-place would race
     /// because __syncthreads is block-local but ranking reads ALL rewards).
-    pub fn shape_rewards(&self, rewards_dst: &mut CudaSlice<f32>, n: usize, reward_std: f32) -> Result<(), MLError> {
+    pub fn shape_rewards(&self, rewards_dst: &mut CudaSlice<f32>, n: usize, reward_std: f32, reward_mean: f32) -> Result<(), MLError> {
         if n == 0 { return Ok(()); }
         let n_i32 = n as i32;
         let blocks = ((n + 255) / 256) as u32;
@@ -2443,6 +2445,7 @@ impl GpuExperienceCollector {
                 .arg(rewards_dst)         // output shaped rewards (separate buffer)
                 .arg(&n_i32)
                 .arg(&reward_std)
+                .arg(&reward_mean)        // G13: running mean for Sharpe contribution
                 .launch(LaunchConfig {
                     grid_dim: (blocks, 1, 1),
                     block_dim: (256, 1, 1),

crates/ml/src/cuda_pipeline/reward_shaping_kernel.cu

@@ -1,7 +1,8 @@
 /**
  * Rank-preserving signed reward standardization — tiled shared-memory.
  *
- * r_shaped[i] = sign(r[i]) * rank(|r[i]|) * std_ema
+ * r_shaped[i] = sign(sharpe[i]) * rank(|sharpe[i]|)
+ * where sharpe[i] = (r[i] - mean_ema) / std_ema   (G13 Sharpe contribution)
  *
  * rank(|r[i]|) = count(|r[j]| <= |r[i]| for all j in batch) / N
  *
@@ -21,12 +22,18 @@ extern "C" __global__ void reward_rank_normalize(
     const float* __restrict__ rewards_in,  /* [N] read-only raw rewards */
     float* __restrict__ rewards_out,       /* [N] output shaped rewards */
     int N,
-    float std_ema                          /* running std of raw rewards */
+    float std_ema,                         /* running std of raw rewards */
+    float return_mean_ema                  /* running mean of raw rewards */
 ) {
     int i = blockIdx.x * blockDim.x + threadIdx.x;
 
-    /* Each thread loads its own |reward| once from global memory */
-    float r_i = (i < N) ? rewards_in[i] : 0.0f;
+    /* Each thread loads its own reward from global memory */
+    float raw_r = (i < N) ? rewards_in[i] : 0.0f;
+    /* G13: Sharpe contribution = (return - mean) / std.
+     * Rank normalization operates on risk-adjusted contributions,
+     * so high-return trades during volatile periods get lower
+     * rank weight than the same return during calm periods. */
+    float r_i = (std_ema > 1e-6f) ? (raw_r - return_mean_ema) / std_ema : raw_r;
     float abs_i = fabsf(r_i);
 
     __shared__ float tile[TILE_SIZE];
@@ -37,7 +44,10 @@ extern "C" __global__ void reward_rank_normalize(
     for (int t = 0; t < num_tiles; t++) {
         /* Cooperative tile load: each thread loads one element */
         int tile_idx = t * TILE_SIZE + threadIdx.x;
-        tile[threadIdx.x] = (tile_idx < N) ? fabsf(rewards_in[tile_idx]) : 1e30f;
+        /* G13: tile stores |sharpe| not |raw| — must match abs_i computation */
+        float tile_raw = (tile_idx < N) ? rewards_in[tile_idx] : 0.0f;
+        float tile_sharpe = (std_ema > 1e-6f) ? (tile_raw - return_mean_ema) / std_ema : tile_raw;
+        tile[threadIdx.x] = (tile_idx < N) ? fabsf(tile_sharpe) : 1e30f;
         __syncthreads();
 
         /* Count within tile — shared memory scan, no global reads */
@@ -52,8 +62,9 @@ extern "C" __global__ void reward_rank_normalize(
 
     if (i >= N) return;
 
-    /* Rank in [0, 1], sign preserved, scaled by std_ema */
+    /* Rank in [0, 1], sign of Sharpe contribution preserved */
     float rank = (float)count / (float)N;
     float sign = (r_i > 0.0f) ? 1.0f : (r_i < 0.0f) ? -1.0f : 0.0f;
-    rewards_out[i] = sign * rank * fmaxf(std_ema, 1e-6f);
+    /* G13: input is already in Sharpe units — no rescale by std_ema needed */
+    rewards_out[i] = sign * rank;
 }

crates/ml/src/trainers/dqn/trainer/constructor.rs

@@ -718,6 +718,7 @@ impl DQNTrainer {
             epoch_atom_entropy: 0.0,
             epoch_atom_utilization: 0.0,
             observed_reward_std: 0.0,
+            observed_reward_mean: 0.0,
 
             // GPU pipeline: pre-uploaded training data (initialized lazily at first epoch)
             gpu_data: None,

crates/ml/src/trainers/dqn/trainer/mod.rs

@@ -349,6 +349,8 @@ pub struct DQNTrainer {
     pub(crate) epoch_atom_utilization: f32,
     /// Observed reward std from experience collector (for adaptive v_range floor).
     pub(crate) observed_reward_std: f32,
+    /// G13: Observed reward mean from experience collector (for Sharpe-aware shaping).
+    pub(crate) observed_reward_mean: f32,
     /// Running ratio of low-drawdown epochs (EMA of binary signal)
     pub(crate) low_dd_ratio: f64,
     /// Whether adversarial regime is active this epoch

crates/ml/src/trainers/dqn/trainer/training_loop.rs

@@ -1179,12 +1179,13 @@ impl DQNTrainer {
 
         let count = gpu_batch.n_episodes * gpu_batch.timesteps * 2; // counterfactual doubles experiences
 
-        // Rank-normalize rewards: r_shaped = sign(r) * rank(|r|) * std_ema
+        // G13: Sharpe-aware rank-normalize rewards.
+        // Computes sharpe[i] = (r - mean) / std, then ranks by |sharpe|.
         // Amplifies reward SNR from 0.001 to ~0.5 — counting-sort rank is outlier-resistant.
         // Skip epoch 0: observed_reward_std is 0.0 until GPU monitoring computes it at epoch end.
         // Raw rewards flow through epoch 0; rank normalization kicks in from epoch 1+.
         if self.observed_reward_std > 1e-8 {
-            collector.shape_rewards(&mut gpu_batch.rewards, count, self.observed_reward_std)
+            collector.shape_rewards(&mut gpu_batch.rewards, count, self.observed_reward_std, self.observed_reward_mean)
                 .map_err(|e| anyhow::anyhow!("reward rank normalize: {e}"))?;
         }
 
@@ -1658,6 +1659,7 @@ impl DQNTrainer {
                     );
                     monitor.track_reward(summary.mean_reward);
                     self.observed_reward_std = summary.reward_std;
+                    self.observed_reward_mean = summary.mean_reward;
                     // Feed all 3 branch distributions from GPU into the monitor.
                     // These are the actual model-selected actions, not
                     // deterministic OrderRouter re-derivations.