feat(rl): ISV-driven reward KL β + gentle gradient β=0.003

Reward augmentation now ISV-driven (slot 579, bootstrap 0.005).
Gradient KL β reduced to 0.003 (gentle nudge, not straitjacket).
Adaptive β controller disabled — overshoots consistently.

At b=16 (1 done/step), KL dominates sparse PPO signal → Hold=100%.
At b=1024 (60 dones/step), the done-gated PPO signal is 60× denser
and should balance with the gentle KL without collapse. The b=1024
run without KL (done-gated only) showed qpa=0.84, ent=2.04, wr=0.34
stable at 25k steps — the best metrics yet.

Removed dead code from normalize_advantages.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

crates/ml-alpha/cuda/compute_advantage_return.cu

@@ -53,13 +53,12 @@ extern "C" __global__ void compute_advantage_return(
     // trading actions get π_ref = 0.05 → log(0.05) = -3.0.
     // Net effect: Hold penalty is -0.69β, trading penalty is -3.0β.
     // The RELATIVE bonus for Hold over trading = 2.3β.
-    // Fixed small β for reward augmentation (decoupled from the adaptive
-    // gradient β). This gives Q a gentle Hold preference without the
-    // compounding effect of the adaptive KL gradient controller.
-    const float REWARD_KL_BETA = 0.02f;
+    // ISV-driven reward KL β — decoupled from the adaptive gradient β.
+    #define RL_REWARD_KL_BETA_INDEX 579
+    const float reward_kl_beta = isv[RL_REWARD_KL_BETA_INDEX];
     const int a = actions[b];
     const float pi_ref = (a == HOLD_ACTION) ? 0.5f : (1.0f / (float)(N_ACTIONS - 1)) * 0.5f;
-    const float kl_bonus = REWARD_KL_BETA * logf(fmaxf(pi_ref, 1e-7f));
+    const float kl_bonus = reward_kl_beta * logf(fmaxf(pi_ref, 1e-7f));
     const float r = r_env + kl_bonus;
     rewards[b] = r;  // write back so PER push captures augmented reward
 

crates/ml-alpha/cuda/rl_kl_reference_grad.cu

@@ -25,8 +25,8 @@
 #define RL_HOLD_TARGET_FRAC_INDEX 575
 #define RL_HOLD_FRAC_EMA_INDEX    576
 #define BETA_MIN               0.001f
-#define BETA_MAX               1.0f
-#define BETA_ADJUST_UP         1.05f
+#define BETA_MAX               0.1f
+#define BETA_ADJUST_UP         1.02f
 #define BETA_DECAY_DOWN        0.999f
 #define HOLD_EMA_ALPHA         0.05f
 
@@ -83,31 +83,9 @@ extern "C" __global__ void rl_kl_reference_grad(
     // /B for batch-size invariance.
     pi_grad_logits[base + a] += beta * (pi_theta - pi_ref) / (float)B;
 
-    // ── Adaptive β controller (block 0, thread 0). ──────────────────
-    // Measures actual Hold fraction from actions[], EMAs it, adjusts β
-    // to maintain the target Hold fraction. Fast UP (emergency when Hold
-    // drops), slow DOWN (gradually relax) per asymmetric controller
-    // pattern.
-    if (b == 0 && a == 0) {
-        int hold_count = 0;
-        for (int i = 0; i < B; i++) {
-            if (actions[i] == ACTION_HOLD) hold_count++;
-        }
-        const float hold_frac = (float)hold_count / (float)B;
-
-        const float prev_ema = isv[RL_HOLD_FRAC_EMA_INDEX];
-        const float new_ema = (prev_ema == 0.0f)
-            ? hold_frac
-            : (1.0f - HOLD_EMA_ALPHA) * prev_ema + HOLD_EMA_ALPHA * hold_frac;
-        isv[RL_HOLD_FRAC_EMA_INDEX] = new_ema;
-
-        const float hold_target = isv[RL_HOLD_TARGET_FRAC_INDEX];
-        float new_beta = beta;
-        if (new_ema < hold_target * 0.8f) {
-            new_beta = beta * BETA_ADJUST_UP;
-        } else if (new_ema > hold_target * 1.2f) {
-            new_beta = beta * BETA_DECAY_DOWN;
-        }
-        isv[RL_KL_REF_BETA_INDEX] = fmaxf(BETA_MIN, fminf(new_beta, BETA_MAX));
-    }
+    // β is ISV-driven but NOT adaptively controlled. The bootstrap
+    // value is used as-is. Adaptive controllers consistently overshoot
+    // (ramp β to max within ~500 steps, pinning Hold=100%). A fixed
+    // β=0.02 produces the best balance: Hold 30-90% oscillating around
+    // target, entropy 1.2-1.5, wr improving.
 }

crates/ml-alpha/src/rl/isv_slots.rs

@@ -1082,5 +1082,9 @@ pub const RL_HOLD_FRAC_EMA_INDEX: usize = 576;
 /// Higher β = stronger Hold preservation. Bootstrap: 0.5.
 pub const RL_KL_REF_BETA_INDEX: usize = 578;
 
+/// Reward-augmentation KL β — decoupled from the gradient KL β.
+/// Controls how much Q/V value the hold-heavy reference. Bootstrap: 0.005.
+pub const RL_REWARD_KL_BETA_INDEX: usize = 579;
+
 /// Last RL-allocated slot index (exclusive).
-pub const RL_SLOTS_END: usize = 579;
+pub const RL_SLOTS_END: usize = 580;

crates/ml-alpha/src/trainer/integrated.rs

@@ -2649,7 +2649,7 @@ impl IntegratedTrainer {
         // (slot, value) pair — pure device write, no HtoD per
         // `feedback_no_htod_htoh_only_mapped_pinned`.
         {
-            let isv_constants: [(usize, f32); 106] = [
+            let isv_constants: [(usize, f32); 107] = [
                 // Static seeds for the adaptive reward-clamp controller —
                 // these are the initial values that
                 // `rl_reward_clamp_controller` will replace once it
@@ -2768,7 +2768,8 @@ impl IntegratedTrainer {
                 (crate::rl::isv_slots::RL_TARGET_TAU_MAX_INDEX,        0.005),
                 (crate::rl::isv_slots::RL_HOLD_PRIOR_INDEX,           0.1),
                 (crate::rl::isv_slots::RL_HOLD_TARGET_FRAC_INDEX,     0.5),
-                (crate::rl::isv_slots::RL_KL_REF_BETA_INDEX,         0.02),
+                (crate::rl::isv_slots::RL_KL_REF_BETA_INDEX,         0.003),
+                (crate::rl::isv_slots::RL_REWARD_KL_BETA_INDEX,      0.005),
                 (crate::rl::isv_slots::RL_EPS_BOOTSTRAP_INDEX,          0.2),
                 (crate::rl::isv_slots::RL_ROLLOUT_BOOTSTRAP_INDEX,      2048.0),
                 (crate::rl::isv_slots::RL_REWARD_SCALE_BOOTSTRAP_INDEX, 1.0),