foxhunt/crates/ml-alpha/cuda/rl_per_alpha_controller.cu

// rl_per_alpha_controller.cu — emits PER (Prioritized Experience Replay)
// priority exponent to ISV[RL_PER_ALPHA_INDEX=405].
//
// Phase E.3b of the integrated RL trainer
// (docs/superpowers/plans/2026-05-22-integrated-rl-trainer.md).
//
// PER's priority exponent α controls how aggressively the replay buffer
// concentrates sampling on high-TD-error transitions. Per
// `pearl_controller_anchors_isv_driven` and
// `feedback_isv_for_adaptive_bounds`, α is NOT a hardcoded constant:
// it adapts to the tail thickness of the TD-error distribution.
//
// Controller logic:
//   * High TD-error kurtosis (heavy tails) → a few transitions
//     dominate the loss → raise α to sharpen the priority distribution
//     and concentrate sampling on the informative tails.
//   * Low kurtosis (light tails, ≈ Gaussian) → TD-errors are uniform →
//     keep α near the standard PER default (0.6) — over-sharpening on a
//     light-tailed distribution wastes samples on noise.
//
// Bootstrap discipline (per `pearl_first_observation_bootstrap`): the
// ISV slot starts at 0.0 sentinel. First emit DERIVES the bootstrap
// from the current `input_slot` (td_kurtosis EMA) via the same target
// formula the per-step path uses, instead of writing a hardcoded
// "canonical PER 0.6 default". The earlier hardcoded-0.6 pattern
// coincided with `target(kurt=10) = 0.6` — the canonical
// heavy-tailed-market kurtosis — creating a Wiener-blend fixed point
// where `prev = target = 0.6` froze the controller in a dead-zone
// for any input near canonical. The derive-from-input pattern
// eliminates the coincidence: bootstrap is whatever the target formula
// emits for the current input (= 0.4 at sentinel-zero input, = 0.6
// when input has already EMA'd to canonical 10, = 0.886 at kurt=20,
// etc.) so the controller's per-step Wiener blend always sees a real
// `prev` vs `target` delta on subsequent inputs.
//
// Subsequent emits Wiener-α blend with floor 0.4 (per
// `pearl_wiener_alpha_floor_for_nonstationary` — the TD-error
// distribution drifts as Q co-adapts, breaking stationarity).
//
// Bounds: α ∈ [0.3, 1.0]. Below 0.3 the priority distribution
// approaches uniform (PER degenerates into vanilla replay); at 1.0
// sampling is perfectly proportional to priority.

#define RL_PER_ALPHA_INDEX 405
// PER α MIN/MAX clamp bounds are now ISV-driven per the 2026-05-30
// clamp-bound extension. Runtime-tunable + visible in diag.
#define RL_PER_ALPHA_MIN_INDEX  647
#define RL_PER_ALPHA_MAX_INDEX  648
// Kurtosis of a standard normal = 3.0 ("excess kurtosis 0" with the
// alternative convention). Used as the breakpoint above which we start
// raising α.
// ISV-driven kurtosis interpretation constants per
// `feedback_isv_for_adaptive_bounds`.
#define RL_KURT_GAUSSIAN_INDEX    471
#define RL_KURT_NOISE_FLOOR_INDEX 472
#define RL_KURT_LIFT_SCALE_INDEX  459
// Schulman tolerance from the shared global slot.
#define RL_SCHULMAN_TOLERANCE_INDEX        468
// Noise-floor gate: if the streaming kurtosis estimator emits a value
// below this magnitude, treat it as "no signal" (sentinel-zero proxy)
// and hold α at the prior value. Without this, on the first few steps
// after the streaming estimator initialises, a sub-Gaussian kurtosis
// reading (e.g. 0.5 — near-uniform per-step batch means before tails
// develop) would yield target = 0.4 = PER_ALPHA_MIN, dragging α toward
// MIN despite zero real signal. Matches the defensive noise-floor
// pattern on the other controllers (per
// pearl_multiplicative_controllers_need_bounded_step_and_noise_floor).
// (KURT_NOISE_FLOOR — was 1.0f #define — now isv[RL_KURT_NOISE_FLOOR_INDEX])
// Wiener-α floor — shared across 9 controllers (slot 659).
#define RL_WIENER_ALPHA_FLOOR_INDEX  659

// Adaptive controller floors (spec 2026-05-30-adaptive-controller-floor-design):
// noise floor augmented with Welford-derived adaptive component (replaces
// reliance on the ISV-driven RL_KURT_NOISE_FLOOR_INDEX absolute floor alone).
// Asymmetric Schulman semantics: α RISES on a single above-band kurtosis
// observation (heavy tails = safety signal, sharpen PER fast); α FALLS only
// after N consecutive below-band observations (light tails = patient drift).
// Replaces the regime where the controller stuck at MIN 0.4 because every
// step's kurtosis read landed just above the absolute floor but well below
// the band — the Wiener blend dragged α down even on single observations.
#define RL_TD_KURT_VAR_COUNT_INDEX         604
#define RL_TD_KURT_VAR_M2_INDEX            606
#define RL_TD_KURT_BELOW_COUNT_INDEX       607
#define NOISE_FLOOR_STD_MULTIPLIER         2.0f  // floor ≥ 2σ of observed kurtosis
#define WIDEN_PATIENCE_CONSECUTIVE         3.0f  // α descent requires N below-band steps

// ─────────────────────────────────────────────────────────────────────
// rl_per_alpha_controller:
//   Single-thread controller — writes ONE float to
//   isv[RL_PER_ALPHA_INDEX].
//
// Inputs:
//   isv              [≥ RL_PER_ALPHA_INDEX+1] — ISV bus
//   alpha_step       — Wiener-α from the controller's signal stats;
//                      floored at WIENER_ALPHA_FLOOR before the blend.
//                      Named `alpha_step` to disambiguate from the
//                      output (PER's α).
//   td_kurtosis_ema  — caller-computed kurtosis EMA of the per-
//                      transition |TD-error| series. Phase F produces
//                      this via a 4-moment reduce over the replay
//                      priority column.
//
// Outputs:
//   isv[RL_PER_ALPHA_INDEX] — PER priority exponent α ∈
//                              [PER_ALPHA_MIN, PER_ALPHA_MAX]
// ─────────────────────────────────────────────────────────────────────
// Phase R5: scalar input arg replaced with `input_slot` ISV index so the
// EMA producer (Phase R3 ema_update_on_done targeting
// ISV[RL_TD_KURTOSIS_EMA_INDEX=422]) feeds this controller without any
// host roundtrip per `feedback_cpu_is_read_only`.
extern "C" __global__ void rl_per_alpha_controller(
    float* __restrict__ isv,
    float           alpha_step,
    int             input_slot
) {
    if (threadIdx.x != 0 || blockIdx.x != 0) return;

    const float prev = isv[RL_PER_ALPHA_INDEX];

    // Noise-floor gate: kurtosis below the adaptive noise floor is
    // dominated by streaming-estimator startup noise (per-step batch-mean
    // differences before tails accumulate). Hold α at prev to avoid
    // dragging toward PER_ALPHA_MIN on cold-start.
    //
    // Adaptive component (spec 2026-05-30): floor scales with observed
    // kurtosis std. Replaces reliance on the absolute ISV floor alone.
    // Welford sample variance = M² / (count − 1) when count > 1.
    const float td_kurtosis_ema = isv[input_slot];
    const float kurt_count = isv[RL_TD_KURT_VAR_COUNT_INDEX];
    const float kurt_var = (kurt_count > 1.0f)
        ? isv[RL_TD_KURT_VAR_M2_INDEX] / (kurt_count - 1.0f)
        : 0.0f;
    const float kurt_std = sqrtf(kurt_var);
    const float adaptive_noise_floor = fmaxf(isv[RL_KURT_NOISE_FLOOR_INDEX],
                                              kurt_std * NOISE_FLOOR_STD_MULTIPLIER);
    if (td_kurtosis_ema > 0.0f && td_kurtosis_ema < adaptive_noise_floor) {
        // Real signal present but below the noise floor — hold prev.
        // (Strict sentinel zero handled by the prev==0 bootstrap path
        // below, which derives target from the current EMA so the
        // first-non-zero observation still seeds correctly.)
        if (prev != 0.0f) return;
    }

    // Compute target from the current input EMA via a piecewise linear
    // lift. Shared between bootstrap and per-step paths so the dead-zone
    // coincidence with a hardcoded bootstrap value cannot recur.
    //   kurt ≤ KURT_GAUSSIAN          → target = 0.4 (PER_ALPHA_MIN + 0.1)
    //   kurt = KURT_GAUSSIAN + KURT_LIFT_SCALE (≈10) → target = 0.6 (canonical PER)
    //   kurt → ∞                      → target → PER_ALPHA_MAX
    //
    // The 0.4-0.6 baseline keeps the steady-state output near PER's
    // canonical 0.6 (when input EMA stabilises at kurt=10) while
    // leaving headroom to lift toward 1.0 under heavy tails.
    const float per_alpha_min = isv[RL_PER_ALPHA_MIN_INDEX];
    const float per_alpha_max = isv[RL_PER_ALPHA_MAX_INDEX];
    const float kurt_excess = fmaxf(0.0f, td_kurtosis_ema - isv[RL_KURT_GAUSSIAN_INDEX]);
    const float kurt_lift_scale = isv[RL_KURT_LIFT_SCALE_INDEX];
    float       target      = 0.4f + 0.2f * (kurt_excess / kurt_lift_scale);
    target = fmaxf(per_alpha_min, fminf(target, per_alpha_max));

    // Bootstrap on sentinel 0.0 per pearl_first_observation_bootstrap:
    // first emit replaces directly with the computed target. At
    // construction time when input EMA is also sentinel-zero, target =
    // 0.4 (min), which is distinct from EVERY non-sentinel target
    // value the formula can emit (≥ 0.4). The next per-step call with
    // any real input will see prev=0.4 vs target≥0.4 and Wiener-blend
    // away from the sentinel.
    if (prev == 0.0f) {
        isv[RL_PER_ALPHA_INDEX] = target;
        return;
    }

    // Wiener-α blend with floor per pearl_wiener_alpha_floor_for_nonstationary.
    const float a   = fmaxf(alpha_step, isv[RL_WIENER_ALPHA_FLOOR_INDEX]);
    float       out = (1.0f - a) * prev + a * target;

    out = fmaxf(per_alpha_min, fminf(out, per_alpha_max));

    // Asymmetric Schulman patience on the DESCENT direction (spec 2026-05-30):
    // PER α rises on a single above-band kurtosis observation (heavy tails
    // = act fast to concentrate sampling on informative tails). PER α
    // falls only after N consecutive below-band observations — a single
    // light-tailed step must not drag α toward MIN.
    const float kurt_gaussian = isv[RL_KURT_GAUSSIAN_INDEX];
    const float tolerance     = isv[RL_SCHULMAN_TOLERANCE_INDEX];
    if (out < prev && td_kurtosis_ema < kurt_gaussian / tolerance) {
        const float new_count = isv[RL_TD_KURT_BELOW_COUNT_INDEX] + 1.0f;
        isv[RL_TD_KURT_BELOW_COUNT_INDEX] = new_count;
        if (new_count < WIDEN_PATIENCE_CONSECUTIVE) {
            // Hold at prev until patience accumulates.
            isv[RL_PER_ALPHA_INDEX] = prev;
            return;
        }
    } else {
        isv[RL_TD_KURT_BELOW_COUNT_INDEX] = 0.0f;
    }

    isv[RL_PER_ALPHA_INDEX] = out;
}