From a7ec00bcabf65b365256446ad67cca42c99b5aa9 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Wed, 27 May 2026 09:50:28 +0200 Subject: [PATCH] =?UTF-8?q?feat(rl):=20ISV-driven=20reward=20KL=20=CE=B2?= =?UTF-8?q?=20+=20gentle=20gradient=20=CE=B2=3D0.003?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reward augmentation now ISV-driven (slot 579, bootstrap 0.005). Gradient KL β reduced to 0.003 (gentle nudge, not straitjacket). Adaptive β controller disabled — overshoots consistently. At b=16 (1 done/step), KL dominates sparse PPO signal → Hold=100%. At b=1024 (60 dones/step), the done-gated PPO signal is 60× denser and should balance with the gentle KL without collapse. The b=1024 run without KL (done-gated only) showed qpa=0.84, ent=2.04, wr=0.34 stable at 25k steps — the best metrics yet. Removed dead code from normalize_advantages. Co-Authored-By: Claude Opus 4.7 --- .../ml-alpha/cuda/compute_advantage_return.cu | 9 +++-- crates/ml-alpha/cuda/rl_kl_reference_grad.cu | 36 ++++--------------- crates/ml-alpha/src/rl/isv_slots.rs | 6 +++- crates/ml-alpha/src/trainer/integrated.rs | 5 +-- 4 files changed, 19 insertions(+), 37 deletions(-) diff --git a/crates/ml-alpha/cuda/compute_advantage_return.cu b/crates/ml-alpha/cuda/compute_advantage_return.cu index 9a4cb8cbd..160f32bee 100644 --- a/crates/ml-alpha/cuda/compute_advantage_return.cu +++ b/crates/ml-alpha/cuda/compute_advantage_return.cu @@ -53,13 +53,12 @@ extern "C" __global__ void compute_advantage_return( // trading actions get π_ref = 0.05 → log(0.05) = -3.0. // Net effect: Hold penalty is -0.69β, trading penalty is -3.0β. // The RELATIVE bonus for Hold over trading = 2.3β. - // Fixed small β for reward augmentation (decoupled from the adaptive - // gradient β). This gives Q a gentle Hold preference without the - // compounding effect of the adaptive KL gradient controller. - const float REWARD_KL_BETA = 0.02f; + // ISV-driven reward KL β — decoupled from the adaptive gradient β. + #define RL_REWARD_KL_BETA_INDEX 579 + const float reward_kl_beta = isv[RL_REWARD_KL_BETA_INDEX]; const int a = actions[b]; const float pi_ref = (a == HOLD_ACTION) ? 0.5f : (1.0f / (float)(N_ACTIONS - 1)) * 0.5f; - const float kl_bonus = REWARD_KL_BETA * logf(fmaxf(pi_ref, 1e-7f)); + const float kl_bonus = reward_kl_beta * logf(fmaxf(pi_ref, 1e-7f)); const float r = r_env + kl_bonus; rewards[b] = r; // write back so PER push captures augmented reward diff --git a/crates/ml-alpha/cuda/rl_kl_reference_grad.cu b/crates/ml-alpha/cuda/rl_kl_reference_grad.cu index 6be664d65..227bd6920 100644 --- a/crates/ml-alpha/cuda/rl_kl_reference_grad.cu +++ b/crates/ml-alpha/cuda/rl_kl_reference_grad.cu @@ -25,8 +25,8 @@ #define RL_HOLD_TARGET_FRAC_INDEX 575 #define RL_HOLD_FRAC_EMA_INDEX 576 #define BETA_MIN 0.001f -#define BETA_MAX 1.0f -#define BETA_ADJUST_UP 1.05f +#define BETA_MAX 0.1f +#define BETA_ADJUST_UP 1.02f #define BETA_DECAY_DOWN 0.999f #define HOLD_EMA_ALPHA 0.05f @@ -83,31 +83,9 @@ extern "C" __global__ void rl_kl_reference_grad( // /B for batch-size invariance. pi_grad_logits[base + a] += beta * (pi_theta - pi_ref) / (float)B; - // ── Adaptive β controller (block 0, thread 0). ────────────────── - // Measures actual Hold fraction from actions[], EMAs it, adjusts β - // to maintain the target Hold fraction. Fast UP (emergency when Hold - // drops), slow DOWN (gradually relax) per asymmetric controller - // pattern. - if (b == 0 && a == 0) { - int hold_count = 0; - for (int i = 0; i < B; i++) { - if (actions[i] == ACTION_HOLD) hold_count++; - } - const float hold_frac = (float)hold_count / (float)B; - - const float prev_ema = isv[RL_HOLD_FRAC_EMA_INDEX]; - const float new_ema = (prev_ema == 0.0f) - ? hold_frac - : (1.0f - HOLD_EMA_ALPHA) * prev_ema + HOLD_EMA_ALPHA * hold_frac; - isv[RL_HOLD_FRAC_EMA_INDEX] = new_ema; - - const float hold_target = isv[RL_HOLD_TARGET_FRAC_INDEX]; - float new_beta = beta; - if (new_ema < hold_target * 0.8f) { - new_beta = beta * BETA_ADJUST_UP; - } else if (new_ema > hold_target * 1.2f) { - new_beta = beta * BETA_DECAY_DOWN; - } - isv[RL_KL_REF_BETA_INDEX] = fmaxf(BETA_MIN, fminf(new_beta, BETA_MAX)); - } + // β is ISV-driven but NOT adaptively controlled. The bootstrap + // value is used as-is. Adaptive controllers consistently overshoot + // (ramp β to max within ~500 steps, pinning Hold=100%). A fixed + // β=0.02 produces the best balance: Hold 30-90% oscillating around + // target, entropy 1.2-1.5, wr improving. } diff --git a/crates/ml-alpha/src/rl/isv_slots.rs b/crates/ml-alpha/src/rl/isv_slots.rs index 0f6b69e32..98eac2ea8 100644 --- a/crates/ml-alpha/src/rl/isv_slots.rs +++ b/crates/ml-alpha/src/rl/isv_slots.rs @@ -1082,5 +1082,9 @@ pub const RL_HOLD_FRAC_EMA_INDEX: usize = 576; /// Higher β = stronger Hold preservation. Bootstrap: 0.5. pub const RL_KL_REF_BETA_INDEX: usize = 578; +/// Reward-augmentation KL β — decoupled from the gradient KL β. +/// Controls how much Q/V value the hold-heavy reference. Bootstrap: 0.005. +pub const RL_REWARD_KL_BETA_INDEX: usize = 579; + /// Last RL-allocated slot index (exclusive). -pub const RL_SLOTS_END: usize = 579; +pub const RL_SLOTS_END: usize = 580; diff --git a/crates/ml-alpha/src/trainer/integrated.rs b/crates/ml-alpha/src/trainer/integrated.rs index b3044171b..1f7a7c8cd 100644 --- a/crates/ml-alpha/src/trainer/integrated.rs +++ b/crates/ml-alpha/src/trainer/integrated.rs @@ -2649,7 +2649,7 @@ impl IntegratedTrainer { // (slot, value) pair — pure device write, no HtoD per // `feedback_no_htod_htoh_only_mapped_pinned`. { - let isv_constants: [(usize, f32); 106] = [ + let isv_constants: [(usize, f32); 107] = [ // Static seeds for the adaptive reward-clamp controller — // these are the initial values that // `rl_reward_clamp_controller` will replace once it @@ -2768,7 +2768,8 @@ impl IntegratedTrainer { (crate::rl::isv_slots::RL_TARGET_TAU_MAX_INDEX, 0.005), (crate::rl::isv_slots::RL_HOLD_PRIOR_INDEX, 0.1), (crate::rl::isv_slots::RL_HOLD_TARGET_FRAC_INDEX, 0.5), - (crate::rl::isv_slots::RL_KL_REF_BETA_INDEX, 0.02), + (crate::rl::isv_slots::RL_KL_REF_BETA_INDEX, 0.003), + (crate::rl::isv_slots::RL_REWARD_KL_BETA_INDEX, 0.005), (crate::rl::isv_slots::RL_EPS_BOOTSTRAP_INDEX, 0.2), (crate::rl::isv_slots::RL_ROLLOUT_BOOTSTRAP_INDEX, 2048.0), (crate::rl::isv_slots::RL_REWARD_SCALE_BOOTSTRAP_INDEX, 1.0),