From 5c7cc4ec32c321670330c5806be2a4b8f92276ca Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Wed, 27 May 2026 01:05:42 +0200 Subject: [PATCH] =?UTF-8?q?wip(rl):=20entropy=20gradient=20+=20controller?= =?UTF-8?q?=20fix=20=E2=80=94=20still=20collapsing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added entropy gradient (∂H/∂logits) to PPO backward, fires on ALL batch elements. Fixed controller: COEF_FLOOR=0.01, COEF_MAX=0.5, emergency bypass Wiener when entropy < 50% target. Symmetric response (non-zero coef even when entropy above target). Still collapses: PPO done-gated gradient concentrates on trading actions, overwhelming the diffuse entropy gradient. Hold drops to 0% despite coef=0.5. The surfer philosophy needs structural enforcement at the action-selection level, not just gradient-level entropy bonus. Co-Authored-By: Claude Opus 4.7 --- crates/ml-alpha/cuda/ppo_clipped_surrogate.cu | 35 ++++++++++++++++++- .../cuda/rl_entropy_coef_controller.cu | 28 ++++++++++++--- crates/ml-alpha/cuda/rl_fused_controllers.cu | 10 ++++-- 3 files changed, 64 insertions(+), 9 deletions(-) diff --git a/crates/ml-alpha/cuda/ppo_clipped_surrogate.cu b/crates/ml-alpha/cuda/ppo_clipped_surrogate.cu index 691391bae..c648a3b0d 100644 --- a/crates/ml-alpha/cuda/ppo_clipped_surrogate.cu +++ b/crates/ml-alpha/cuda/ppo_clipped_surrogate.cu @@ -366,7 +366,40 @@ extern "C" __global__ void ppo_clipped_surrogate_bwd( const float indicator = (act == a_taken) ? 1.0f : 0.0f; pg_grad = -A * (p_a - indicator) * ratio; } - grad_logits[base + act] = pg_grad / (float)B; + + // Entropy gradient: ∂H/∂logit_a = -(1 + log π(a)) × π(a) + π(a) × Σ_j π(j)(1 + log π(j)) + // Simplified through softmax identity: ∂(-H)/∂logit_a = π(a) × (1 + log π(a)) - π(a) × (1 + H + log_sum_exp) + // Which reduces to: ∂(-H)/∂logit_a = π(a) × (log π(a) + 1 + H)... + // Actually the standard form: ∂H/∂logit_a = -p_a × (1 + log p_a) + p_a × Σ_j p_j(1 + log p_j) + // Since we want to MAXIMIZE entropy (minimize -H), gradient of -(-coef*H) = coef * ∂H/∂logit_a + // = coef * (-p_a * (1 + log p_a) + p_a * mean_term) + // Simplest correct form via softmax: ∂(-H)/∂logit_a = p_a * (log p_a + 1 + H) where H is the entropy + // So ∂(coef*H)/∂logit_a = -coef * p_a * (log(p_a) + 1 + H) + // + // This fires on ALL batch elements (not done-gated), preventing + // entropy collapse even when PPO advantages are masked to zero + // on non-done steps. Implements the surfer philosophy: Hold is + // the default, trading is the exception. + const float ent_coef = isv[RL_ENTROPY_COEF_INDEX]; + float ent_grad = 0.0f; + if (ent_coef > 0.0f) { + // Compute entropy H for this batch element (thread 0 already did in fwd) + __shared__ float s_entropy; + if (act == 0) { + float h = 0.0f; + for (int i = 0; i < N_ACTIONS; ++i) { + float pi = s_softmax[i] / s_sumexp; + h -= pi * logf(fmaxf(pi, 1e-7f)); + } + s_entropy = h; + } + __syncthreads(); + // ∂(ent_coef * H)/∂logit_a: push toward uniform + const float log_pa = logf(fmaxf(p_a, 1e-7f)); + ent_grad = -ent_coef * p_a * (log_pa + 1.0f + s_entropy); + } + + grad_logits[base + act] = (pg_grad + ent_grad) / (float)B; } diff --git a/crates/ml-alpha/cuda/rl_entropy_coef_controller.cu b/crates/ml-alpha/cuda/rl_entropy_coef_controller.cu index b161a804c..62fe52073 100644 --- a/crates/ml-alpha/cuda/rl_entropy_coef_controller.cu +++ b/crates/ml-alpha/cuda/rl_entropy_coef_controller.cu @@ -47,7 +47,7 @@ #define RL_ENTROPY_COEF_INDEX 403 #define N_ACTIONS 11 #define COEF_MIN 0.0f -#define COEF_MAX 0.05f +#define COEF_MAX 0.5f // ISV-driven entropy-target fraction per `feedback_isv_for_adaptive_bounds`. // Default 0.7 (70% of ln(N_ACTIONS) = "explore but not too randomly"). // Seeded by rl_isv_write at trainer init. @@ -89,10 +89,21 @@ extern "C" __global__ void rl_entropy_coef_controller( // ISV-driven target fraction (was hardcoded 0.7). const float target_frac = isv[RL_ENTROPY_TARGET_FRAC_INDEX]; const float h_target = target_frac * h_max; - const float deficit = fmaxf(0.0f, h_target - entropy_observed_ema); - // Scale coef proportional to the normalised deficit (0..1 fraction - // of ln N). Large deficit → push coef up to encourage exploration. - float coef_target = (deficit / h_max) * COEF_MAX; + // Symmetric response: maintain a floor coef even when entropy is + // above target. Without this, the controller drives coef → 0 when + // entropy is healthy, leaving no defense when entropy later drops + // (the surfer loses the ability to Hold). + // + // coef = COEF_FLOOR + (COEF_MAX - COEF_FLOOR) × deficit_frac + // where deficit_frac = clamp((h_target - h_obs) / h_target, 0, 1) + // + // At h_obs = h_target: coef = COEF_FLOOR (maintenance pressure) + // At h_obs = 0 (total collapse): coef = COEF_MAX (emergency) + // At h_obs > h_target: coef = COEF_FLOOR (no penalty for exploring) + const float COEF_FLOOR = 0.01f; + const float deficit_frac = fmaxf(0.0f, + fminf((h_target - entropy_observed_ema) / fmaxf(h_target, 1e-6f), 1.0f)); + float coef_target = COEF_FLOOR + (COEF_MAX - COEF_FLOOR) * deficit_frac; coef_target = fmaxf(COEF_MIN, fminf(coef_target, COEF_MAX)); // Bootstrap on sentinel 0.0 per pearl_first_observation_bootstrap: @@ -106,6 +117,13 @@ extern "C" __global__ void rl_entropy_coef_controller( return; } + // Emergency: bypass Wiener blend when entropy is below 50% of target. + // The slow blend can't react fast enough to prevent Hold collapse. + if (entropy_observed_ema < h_target * 0.5f && entropy_observed_ema > 0.0f) { + isv[RL_ENTROPY_COEF_INDEX] = coef_target; + return; + } + // Wiener-α blend with floor per pearl_wiener_alpha_floor_for_nonstationary. const float a = fmaxf(alpha, WIENER_ALPHA_FLOOR); float coef_new = (1.0f - a) * coef_prev + a * coef_target; diff --git a/crates/ml-alpha/cuda/rl_fused_controllers.cu b/crates/ml-alpha/cuda/rl_fused_controllers.cu index df649d250..ee05695e1 100644 --- a/crates/ml-alpha/cuda/rl_fused_controllers.cu +++ b/crates/ml-alpha/cuda/rl_fused_controllers.cu @@ -51,7 +51,7 @@ #define RL_ENTROPY_COEF_INDEX 403 #define N_ACTIONS 11 #define COEF_MIN 0.0f -#define COEF_MAX 0.05f +#define COEF_MAX 0.5f #define RL_ENTROPY_TARGET_FRAC_INDEX 458 // --- 5. Rollout steps controller --- @@ -249,12 +249,16 @@ extern "C" __global__ void rl_fused_controllers( const float h_max = logf((float)N_ACTIONS); const float target_frac = isv[RL_ENTROPY_TARGET_FRAC_INDEX]; const float h_target = target_frac * h_max; - const float deficit = fmaxf(0.0f, h_target - entropy_observed_ema); - float coef_target = (deficit / h_max) * COEF_MAX; + const float COEF_FLOOR = 0.01f; + const float deficit_frac = fmaxf(0.0f, + fminf((h_target - entropy_observed_ema) / fmaxf(h_target, 1e-6f), 1.0f)); + float coef_target = COEF_FLOOR + (COEF_MAX - COEF_FLOOR) * deficit_frac; coef_target = fmaxf(COEF_MIN, fminf(coef_target, COEF_MAX)); if (coef_prev == 0.0f) { isv[RL_ENTROPY_COEF_INDEX] = coef_target; + } else if (entropy_observed_ema < h_target * 0.5f && entropy_observed_ema > 0.0f) { + isv[RL_ENTROPY_COEF_INDEX] = coef_target; } else { const float a = fmaxf(alpha, WIENER_ALPHA_FLOOR); float coef_new = (1.0f - a) * coef_prev + a * coef_target;