// rl_entropy_coef_controller.cu — emits entropy-bonus weight to
// ISV[RL_ENTROPY_COEF_INDEX=403].
//
// Phase D of the integrated RL trainer
// (docs/superpowers/plans/2026-05-22-integrated-rl-trainer.md).
//
// The entropy bonus in the PPO surrogate is `L_entropy = -coef × H(π_new)`.
// Per `pearl_controller_anchors_isv_driven` and
// `feedback_isv_for_adaptive_bounds`, `coef` is NOT a hardcoded constant:
// it adapts to keep the policy's action entropy ≥ 70% of ln(N_ACTIONS).
//
// Controller input: `entropy_observed_ema`, the EMA of the per-batch
// entropy reported by `ppo_clipped_surrogate_fwd`. Target entropy is
// `0.7 × ln(N_ACTIONS)` ≈ 1.538 for N_ACTIONS=9.
//
// Logic:
//   * Entropy deficit > 0 (policy too sharp) → raise coef to encourage
//     exploration.
//   * Deficit ≈ 0 (policy maintains exploration) → shrink coef so the
//     policy can sharpen on real signal as training progresses.
//
// Bootstrap discipline (per `pearl_first_observation_bootstrap`): ISV
// slot starts at 0.0 sentinel. First emit DERIVES coef from the
// current input slot via the same target formula the per-step path
// uses, instead of writing a hardcoded `COEF_BOOTSTRAP = 0.01`. At
// sentinel input (entropy_observed_ema = 0) deficit = h_target =
// 1.538, target = (1.538 / 2.197) × 0.05 = 0.035 — the cold-start
// coef is high (push exploration hard when no entropy data is in yet).
// This eliminates the dead-zone where target(h_obs ≈ 1.099) = 0.01 =
// hardcoded bootstrap froze the Wiener blend at any realistic
// mid-range entropy observation.
//
// Trade-off: cold-start coef = 0.035 (3.5× the previous canonical
// 0.01) lifts the entropy bonus's relative weight in early training.
// Acceptable cost — early exploration is desirable, and once
// entropy_observed_ema stabilises the controller drifts coef back
// toward the target driven by actual observed entropy.
//
// Subsequent emits Wiener-α blend with floor 0.4 (per
// `pearl_wiener_alpha_floor_for_nonstationary` — the policy
// entropy co-adapts with training, breaking stationarity).
//
// Bounds: coef ∈ [0.0, 0.05]. The upper bound prevents the entropy
// bonus from dominating the surrogate; the lower bound is permitted to
// hit zero (a fully-trained policy may need no exploration pressure).

#define RL_ENTROPY_COEF_INDEX 403
#define N_ACTIONS               11
// COEF MIN/MAX clamp bounds are now ISV-driven per the 2026-05-30
// clamp-bound extension. Runtime-tunable + visible in diag.
#define RL_ENTROPY_COEF_MIN_INDEX     645
#define RL_ENTROPY_COEF_MAX_INDEX     646
// ISV-driven entropy-target fraction per `feedback_isv_for_adaptive_bounds`.
// Default 0.7 (70% of ln(N_ACTIONS) = "explore but not too randomly").
// Seeded by rl_isv_write at trainer init.
#define RL_ENTROPY_TARGET_FRAC_INDEX 458
// Wiener-α floor — shared across 9 controllers (slot 659).
#define RL_WIENER_ALPHA_FLOOR_INDEX  659

// Adaptive controller floors (spec 2026-05-30-adaptive-controller-floor-design):
// noise floor derives from observed entropy_observed_ema variance via
// Welford triples. Replaces the hardcoded `0.5f` emergency-bypass fraction
// with `max(h_target × 0.5, h_target − 2σ)` — under Phase 4.5 the entropy
// distribution narrows around the SAC target and the hardcoded 0.5×h_target
// emergency gate never fires, leaving the Wiener blend to drag coef to MIN
// (alpha-rl-... fold 0 confirmed entropy_coef stuck at 0.01).
// Asymmetric Schulman semantics: coef RAISES on a single below-target
// observation (entropy collapse = safety signal, act fast); coef DROPS
// only after N consecutive above-target observations (healthy entropy =
// drift coef down patiently).
#define RL_ENTROPY_OBS_VAR_COUNT_INDEX     600
#define RL_ENTROPY_OBS_VAR_M2_INDEX        602
#define RL_ENTROPY_OBS_BELOW_COUNT_INDEX   603
#define RL_SCHULMAN_TOLERANCE_INDEX        468
#define NOISE_FLOOR_TARGET_FRAC            0.5f  // floor ≥ 50% of target
#define NOISE_FLOOR_STD_MULTIPLIER         2.0f  // floor ≥ 2σ of observed signal
#define WIDEN_PATIENCE_CONSECUTIVE         3.0f  // descent requires N above-band steps


// ─────────────────────────────────────────────────────────────────────
// rl_entropy_coef_controller:
//   Single-thread controller — writes ONE float to
//   isv[RL_ENTROPY_COEF_INDEX].
//
// Inputs:
//   isv                    [≥ RL_ENTROPY_COEF_INDEX+1] — ISV bus
//   alpha                  — Wiener-α (floored at WIENER_ALPHA_FLOOR)
//   entropy_observed_ema   — EMA of H(π) reported by surrogate fwd
//
// Outputs:
//   isv[RL_ENTROPY_COEF_INDEX] — coef ∈ [COEF_MIN, COEF_MAX]
// ─────────────────────────────────────────────────────────────────────
// Phase R5: scalar input arg replaced with `input_slot` ISV index so the
// EMA producer (Phase R3 ema_update_per_step targeting
// ISV[RL_ENTROPY_OBSERVED_EMA_INDEX=420]) feeds this controller without
// any host roundtrip per `feedback_cpu_is_read_only`.
extern "C" __global__ void rl_entropy_coef_controller(
    float* __restrict__ isv,
    float          alpha,
    int            input_slot
) {
    if (threadIdx.x != 0 || blockIdx.x != 0) return;

    const float coef_prev = isv[RL_ENTROPY_COEF_INDEX];

    // Compute target from the current input EMA. Shared between
    // bootstrap and per-step paths so the dead-zone coincidence with a
    // hardcoded bootstrap cannot recur.
    const float entropy_observed_ema = isv[input_slot];
    const float h_max    = logf((float)N_ACTIONS);          // ln(9) ≈ 2.197
    // ISV-driven target fraction (was hardcoded 0.7).
    const float target_frac = isv[RL_ENTROPY_TARGET_FRAC_INDEX];
    const float h_target = target_frac * h_max;
    // Symmetric response: maintain a floor coef even when entropy is
    // above target. Without this, the controller drives coef → 0 when
    // entropy is healthy, leaving no defense when entropy later drops
    // (the surfer loses the ability to Hold).
    //
    // coef = COEF_FLOOR + (COEF_MAX - COEF_FLOOR) × deficit_frac
    // where deficit_frac = clamp((h_target - h_obs) / h_target, 0, 1)
    //
    // At h_obs = h_target: coef = COEF_FLOOR (maintenance pressure)
    // At h_obs = 0 (total collapse): coef = COEF_MAX (emergency)
    // At h_obs > h_target: coef = COEF_FLOOR (no penalty for exploring)
    const float COEF_FLOOR = 0.01f;
    const float coef_min = isv[RL_ENTROPY_COEF_MIN_INDEX];
    const float coef_max = isv[RL_ENTROPY_COEF_MAX_INDEX];
    const float deficit_frac = fmaxf(0.0f,
        fminf((h_target - entropy_observed_ema) / fmaxf(h_target, 1e-6f), 1.0f));
    float coef_target = COEF_FLOOR + (coef_max - COEF_FLOOR) * deficit_frac;
    coef_target = fmaxf(coef_min, fminf(coef_target, coef_max));

    // Bootstrap on sentinel 0.0 per pearl_first_observation_bootstrap:
    // first emit replaces directly with the computed target. At cold
    // start (input EMA also sentinel-zero), deficit = h_target,
    // target = 0.035 (high exploration weight). Distinct from any
    // target value the formula can emit for non-sentinel input so the
    // per-step Wiener blend always moves.
    if (coef_prev == 0.0f) {
        isv[RL_ENTROPY_COEF_INDEX] = coef_target;
        return;
    }

    // Adaptive noise floor — signal-driven per
    // `pearl_zscore_normalization_for_magnitude_asymmetric_signals` and
    // `feedback_adaptive_not_tuned`. Replaces hardcoded `h_target × 0.5`
    // emergency-bypass threshold with adaptive `max(h_target × 0.5,
    // h_target − 2σ_observed)` so the emergency path still fires when
    // entropy drops more than 2σ below its observed mean even if the
    // distribution is so narrow that 50% × h_target is unreachable.
    // Welford sample variance = M² / (count − 1) when count > 1.
    const float h_count = isv[RL_ENTROPY_OBS_VAR_COUNT_INDEX];
    const float h_var = (h_count > 1.0f)
        ? isv[RL_ENTROPY_OBS_VAR_M2_INDEX] / (h_count - 1.0f)
        : 0.0f;
    const float h_std = sqrtf(h_var);
    const float emergency_floor = fmaxf(h_target * NOISE_FLOOR_TARGET_FRAC,
                                         h_target - h_std * NOISE_FLOOR_STD_MULTIPLIER);

    // Asymmetric Schulman (spec 2026-05-30): RAISE coef on a single
    // below-emergency-floor observation — entropy collapse is a safety
    // signal we act on fast. DROP coef only after N consecutive
    // above-band observations so a single noisy spike of healthy
    // entropy can't drag coef toward MIN. Below-counter is on the
    // "healthy entropy" direction here (entropy_observed > h_target ×
    // tolerance), distinct from the canonical Schulman semantics where
    // below-counter means "input below band."
    if (entropy_observed_ema > 0.0f && entropy_observed_ema < emergency_floor) {
        isv[RL_ENTROPY_COEF_INDEX] = coef_target;
        isv[RL_ENTROPY_OBS_BELOW_COUNT_INDEX] = 0.0f;
        return;
    }

    const float tolerance = isv[RL_SCHULMAN_TOLERANCE_INDEX];
    const float wiener_a  = fmaxf(alpha, isv[RL_WIENER_ALPHA_FLOOR_INDEX]);
    float       coef_new  = (1.0f - wiener_a) * coef_prev + wiener_a * coef_target;
    coef_new = fmaxf(coef_min, fminf(coef_new, coef_max));

    if (coef_new < coef_prev && entropy_observed_ema > h_target * tolerance) {
        // Healthy-entropy direction — require N consecutive observations
        // before letting the Wiener blend drag coef downward. Single noisy
        // high-entropy spike must not erode the entropy bonus.
        const float new_count = isv[RL_ENTROPY_OBS_BELOW_COUNT_INDEX] + 1.0f;
        isv[RL_ENTROPY_OBS_BELOW_COUNT_INDEX] = new_count;
        if (new_count < WIDEN_PATIENCE_CONSECUTIVE) {
            // Hold coef at previous value until patience accumulates.
            isv[RL_ENTROPY_COEF_INDEX] = coef_prev;
            return;
        }
    } else {
        // Either entropy unhealthy (coef rising) or in-band — reset patience.
        isv[RL_ENTROPY_OBS_BELOW_COUNT_INDEX] = 0.0f;
    }

    isv[RL_ENTROPY_COEF_INDEX] = coef_new;
}
