foxhunt/crates/ml-alpha/cuda/rl_entropy_coef_controller.cu

// rl_entropy_coef_controller.cu — emits entropy-bonus weight to
// ISV[RL_ENTROPY_COEF_INDEX=403].
//
// Phase D of the integrated RL trainer
// (docs/superpowers/plans/2026-05-22-integrated-rl-trainer.md).
//
// The entropy bonus in the PPO surrogate is `L_entropy = -coef × H(π_new)`.
// Per `pearl_controller_anchors_isv_driven` and
// `feedback_isv_for_adaptive_bounds`, `coef` is NOT a hardcoded constant:
// it adapts to keep the policy's action entropy ≥ 70% of ln(N_ACTIONS).
//
// Controller input: `entropy_observed_ema`, the EMA of the per-batch
// entropy reported by `ppo_clipped_surrogate_fwd`. Target entropy is
// `0.7 × ln(N_ACTIONS)` ≈ 1.538 for N_ACTIONS=9.
//
// Logic:
//   * Entropy deficit > 0 (policy too sharp) → raise coef to encourage
//     exploration.
//   * Deficit ≈ 0 (policy maintains exploration) → shrink coef so the
//     policy can sharpen on real signal as training progresses.
//
// Bootstrap discipline (per `pearl_first_observation_bootstrap`): ISV
// slot starts at 0.0 sentinel. First emit DERIVES coef from the
// current input slot via the same target formula the per-step path
// uses, instead of writing a hardcoded `COEF_BOOTSTRAP = 0.01`. At
// sentinel input (entropy_observed_ema = 0) deficit = h_target =
// 1.538, target = (1.538 / 2.197) × 0.05 = 0.035 — the cold-start
// coef is high (push exploration hard when no entropy data is in yet).
// This eliminates the dead-zone where target(h_obs ≈ 1.099) = 0.01 =
// hardcoded bootstrap froze the Wiener blend at any realistic
// mid-range entropy observation.
//
// Trade-off: cold-start coef = 0.035 (3.5× the previous canonical
// 0.01) lifts the entropy bonus's relative weight in early training.
// Acceptable cost — early exploration is desirable, and once
// entropy_observed_ema stabilises the controller drifts coef back
// toward the target driven by actual observed entropy.
//
// Subsequent emits Wiener-α blend with floor 0.4 (per
// `pearl_wiener_alpha_floor_for_nonstationary` — the policy
// entropy co-adapts with training, breaking stationarity).
//
// Bounds: coef ∈ [0.0, 0.05]. The upper bound prevents the entropy
// bonus from dominating the surrogate; the lower bound is permitted to
// hit zero (a fully-trained policy may need no exploration pressure).

#define RL_ENTROPY_COEF_INDEX 403
#define N_ACTIONS               11
#define COEF_MIN              0.0f
#define COEF_MAX              0.05f
// ISV-driven entropy-target fraction per `feedback_isv_for_adaptive_bounds`.
// Default 0.7 (70% of ln(N_ACTIONS) = "explore but not too randomly").
// Seeded by rl_isv_write at trainer init.
#define RL_ENTROPY_TARGET_FRAC_INDEX 458
#define WIENER_ALPHA_FLOOR    0.4f


// ─────────────────────────────────────────────────────────────────────
// rl_entropy_coef_controller:
//   Single-thread controller — writes ONE float to
//   isv[RL_ENTROPY_COEF_INDEX].
//
// Inputs:
//   isv                    [≥ RL_ENTROPY_COEF_INDEX+1] — ISV bus
//   alpha                  — Wiener-α (floored at WIENER_ALPHA_FLOOR)
//   entropy_observed_ema   — EMA of H(π) reported by surrogate fwd
//
// Outputs:
//   isv[RL_ENTROPY_COEF_INDEX] — coef ∈ [COEF_MIN, COEF_MAX]
// ─────────────────────────────────────────────────────────────────────
// Phase R5: scalar input arg replaced with `input_slot` ISV index so the
// EMA producer (Phase R3 ema_update_per_step targeting
// ISV[RL_ENTROPY_OBSERVED_EMA_INDEX=420]) feeds this controller without
// any host roundtrip per `feedback_cpu_is_read_only`.
extern "C" __global__ void rl_entropy_coef_controller(
    float* __restrict__ isv,
    float          alpha,
    int            input_slot
) {
    if (threadIdx.x != 0 || blockIdx.x != 0) return;

    const float coef_prev = isv[RL_ENTROPY_COEF_INDEX];

    // Compute target from the current input EMA. Shared between
    // bootstrap and per-step paths so the dead-zone coincidence with a
    // hardcoded bootstrap cannot recur.
    const float entropy_observed_ema = isv[input_slot];
    const float h_max    = logf((float)N_ACTIONS);          // ln(9) ≈ 2.197
    // ISV-driven target fraction (was hardcoded 0.7).
    const float target_frac = isv[RL_ENTROPY_TARGET_FRAC_INDEX];
    const float h_target = target_frac * h_max;
    const float deficit  = fmaxf(0.0f, h_target - entropy_observed_ema);
    // Scale coef proportional to the normalised deficit (0..1 fraction
    // of ln N). Large deficit → push coef up to encourage exploration.
    float coef_target = (deficit / h_max) * COEF_MAX;
    coef_target = fmaxf(COEF_MIN, fminf(coef_target, COEF_MAX));

    // Bootstrap on sentinel 0.0 per pearl_first_observation_bootstrap:
    // first emit replaces directly with the computed target. At cold
    // start (input EMA also sentinel-zero), deficit = h_target,
    // target = 0.035 (high exploration weight). Distinct from any
    // target value the formula can emit for non-sentinel input so the
    // per-step Wiener blend always moves.
    if (coef_prev == 0.0f) {
        isv[RL_ENTROPY_COEF_INDEX] = coef_target;
        return;
    }

    // Wiener-α blend with floor per pearl_wiener_alpha_floor_for_nonstationary.
    const float a        = fmaxf(alpha, WIENER_ALPHA_FLOOR);
    float       coef_new = (1.0f - a) * coef_prev + a * coef_target;

    coef_new = fmaxf(COEF_MIN, fminf(coef_new, COEF_MAX));
    isv[RL_ENTROPY_COEF_INDEX] = coef_new;
}