gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

wip(rl): entropy gradient + controller fix — still collapsing

Browse Source 
Added entropy gradient (∂H/∂logits) to PPO backward, fires on ALL
batch elements. Fixed controller: COEF_FLOOR=0.01, COEF_MAX=0.5,
emergency bypass Wiener when entropy < 50% target. Symmetric response
(non-zero coef even when entropy above target).

Still collapses: PPO done-gated gradient concentrates on trading
actions, overwhelming the diffuse entropy gradient. Hold drops to 0%
despite coef=0.5. The surfer philosophy needs structural enforcement
at the action-selection level, not just gradient-level entropy bonus.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-05-27 01:05:42 +02:00
parent eeb0a829bf
commit 5c7cc4ec32
3 changed files with 64 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
crates/ml-alpha/cuda/ppo_clipped_surrogate.cu
View File

@@ -366,7 +366,40 @@ extern "C" __global__ void ppo_clipped_surrogate_bwd(
const float indicator = (act == a_taken) ? 1.0f : 0.0f;
pg_grad = -A * (p_a - indicator) * ratio;
}
grad_logits[base + act] = pg_grad / (float)B;
// Entropy gradient: ∂H/∂logit_a = -(1 + log π(a)) × π(a) + π(a) × Σ_j π(j)(1 + log π(j))
// Simplified through softmax identity: ∂(-H)/∂logit_a = π(a) × (1 + log π(a)) - π(a) × (1 + H + log_sum_exp)
// Which reduces to: ∂(-H)/∂logit_a = π(a) × (log π(a) + 1 + H)...
// Actually the standard form: ∂H/∂logit_a = -p_a × (1 + log p_a) + p_a × Σ_j p_j(1 + log p_j)
// Since we want to MAXIMIZE entropy (minimize -H), gradient of -(-coef*H) = coef * ∂H/∂logit_a
// = coef * (-p_a * (1 + log p_a) + p_a * mean_term)
// Simplest correct form via softmax: ∂(-H)/∂logit_a = p_a * (log p_a + 1 + H) where H is the entropy
// So ∂(coef*H)/∂logit_a = -coef * p_a * (log(p_a) + 1 + H)
//
// This fires on ALL batch elements (not done-gated), preventing
// entropy collapse even when PPO advantages are masked to zero
// on non-done steps. Implements the surfer philosophy: Hold is
// the default, trading is the exception.
const float ent_coef = isv[RL_ENTROPY_COEF_INDEX];
float ent_grad = 0.0f;
if (ent_coef > 0.0f) {
// Compute entropy H for this batch element (thread 0 already did in fwd)
__shared__ float s_entropy;
if (act == 0) {
float h = 0.0f;
for (int i = 0; i < N_ACTIONS; ++i) {
float pi = s_softmax[i] / s_sumexp;
h -= pi * logf(fmaxf(pi, 1e-7f));
}
s_entropy = h;
}
__syncthreads();
// ∂(ent_coef * H)/∂logit_a: push toward uniform
const float log_pa = logf(fmaxf(p_a, 1e-7f));
ent_grad = -ent_coef * p_a * (log_pa + 1.0f + s_entropy);
}
grad_logits[base + act] = (pg_grad + ent_grad) / (float)B;
}

28
crates/ml-alpha/cuda/rl_entropy_coef_controller.cu
View File

@@ -47,7 +47,7 @@
#define RL_ENTROPY_COEF_INDEX 403
#define N_ACTIONS 11
#define COEF_MIN 0.0f
#define COEF_MAX 0.05f
#define COEF_MAX 0.5f
// ISV-driven entropy-target fraction per `feedback_isv_for_adaptive_bounds`.
// Default 0.7 (70% of ln(N_ACTIONS) = "explore but not too randomly").
// Seeded by rl_isv_write at trainer init.
@@ -89,10 +89,21 @@ extern "C" __global__ void rl_entropy_coef_controller(
// ISV-driven target fraction (was hardcoded 0.7).
const float target_frac = isv[RL_ENTROPY_TARGET_FRAC_INDEX];
const float h_target = target_frac * h_max;
const float deficit = fmaxf(0.0f, h_target - entropy_observed_ema);
// Scale coef proportional to the normalised deficit (0..1 fraction
// of ln N). Large deficit → push coef up to encourage exploration.
float coef_target = (deficit / h_max) * COEF_MAX;
// Symmetric response: maintain a floor coef even when entropy is
// above target. Without this, the controller drives coef → 0 when
// entropy is healthy, leaving no defense when entropy later drops
// (the surfer loses the ability to Hold).
//
// coef = COEF_FLOOR + (COEF_MAX - COEF_FLOOR) × deficit_frac
// where deficit_frac = clamp((h_target - h_obs) / h_target, 0, 1)
//
// At h_obs = h_target: coef = COEF_FLOOR (maintenance pressure)
// At h_obs = 0 (total collapse): coef = COEF_MAX (emergency)
// At h_obs > h_target: coef = COEF_FLOOR (no penalty for exploring)
const float COEF_FLOOR = 0.01f;
const float deficit_frac = fmaxf(0.0f,
fminf((h_target - entropy_observed_ema) / fmaxf(h_target, 1e-6f), 1.0f));
float coef_target = COEF_FLOOR + (COEF_MAX - COEF_FLOOR) * deficit_frac;
coef_target = fmaxf(COEF_MIN, fminf(coef_target, COEF_MAX));
// Bootstrap on sentinel 0.0 per pearl_first_observation_bootstrap:
@@ -106,6 +117,13 @@ extern "C" __global__ void rl_entropy_coef_controller(
return;
}
// Emergency: bypass Wiener blend when entropy is below 50% of target.
// The slow blend can't react fast enough to prevent Hold collapse.
if (entropy_observed_ema < h_target * 0.5f && entropy_observed_ema > 0.0f) {
isv[RL_ENTROPY_COEF_INDEX] = coef_target;
return;
}
// Wiener-α blend with floor per pearl_wiener_alpha_floor_for_nonstationary.
const float a = fmaxf(alpha, WIENER_ALPHA_FLOOR);
float coef_new = (1.0f - a) * coef_prev + a * coef_target;

10
crates/ml-alpha/cuda/rl_fused_controllers.cu
View File

@@ -51,7 +51,7 @@
#define RL_ENTROPY_COEF_INDEX 403
#define N_ACTIONS 11
#define COEF_MIN 0.0f
#define COEF_MAX 0.05f
#define COEF_MAX 0.5f
#define RL_ENTROPY_TARGET_FRAC_INDEX 458
// --- 5. Rollout steps controller ---
@@ -249,12 +249,16 @@ extern "C" __global__ void rl_fused_controllers(
const float h_max = logf((float)N_ACTIONS);
const float target_frac = isv[RL_ENTROPY_TARGET_FRAC_INDEX];
const float h_target = target_frac * h_max;
const float deficit = fmaxf(0.0f, h_target - entropy_observed_ema);
float coef_target = (deficit / h_max) * COEF_MAX;
const float COEF_FLOOR = 0.01f;
const float deficit_frac = fmaxf(0.0f,
fminf((h_target - entropy_observed_ema) / fmaxf(h_target, 1e-6f), 1.0f));
float coef_target = COEF_FLOOR + (COEF_MAX - COEF_FLOOR) * deficit_frac;
coef_target = fmaxf(COEF_MIN, fminf(coef_target, COEF_MAX));
if (coef_prev == 0.0f) {
isv[RL_ENTROPY_COEF_INDEX] = coef_target;
} else if (entropy_observed_ema < h_target * 0.5f && entropy_observed_ema > 0.0f) {
isv[RL_ENTROPY_COEF_INDEX] = coef_target;
} else {
const float a = fmaxf(alpha, WIENER_ALPHA_FLOOR);
float coef_new = (1.0f - a) * coef_prev + a * coef_target;