foxhunt/crates/ml-alpha/cuda/rl_rollout_steps_controller.cu

// rl_rollout_steps_controller.cu — emits PPO rollout buffer flush size
// to ISV[RL_N_ROLLOUT_STEPS_INDEX=404].
//
// Phase E.3b of the integrated RL trainer
// (docs/superpowers/plans/2026-05-22-integrated-rl-trainer.md).
//
// The rollout length sets how many on-policy transitions PPO accumulates
// before flushing into an update batch. Per
// `pearl_controller_anchors_isv_driven` and
// `feedback_isv_for_adaptive_bounds`, this is NOT a hardcoded constant:
// it adapts to the noise level of the advantage estimator.
//
// Controller logic (Schulman-style bounded step):
//   * input > ADV_VAR_RATIO_TARGET × TOLERANCE → noisy → widen rollout
//     by ADJUST_RATE.
//   * input < ADV_VAR_RATIO_TARGET / TOLERANCE → stable → shrink rollout
//     by ADJUST_RATE.
//   * in-band → hold.
//
// CRITICAL: the prior design used `scale = clamp(input/target, 0.5, 2.0)`
// which saturated to ±2× on the SIGN of (input − target), not the
// magnitude. With typical streaming input 1–10 and target=0.1, every
// step doubled until the controller hit ROLLOUT_MAX in ≤4 steps
// (canonical: alpha-rl-kc2h9 fold0 had n_rollout pegged at MAX 100% of
// 50000 steps despite no actual signal change). Schulman-style bounded
// discrete adjustment converges smoothly to MAX/MIN when signal is
// consistently out-of-band, but doesn't slam there from one
// observation.
//
// Noise-floor gate holds the controller when input is below
// ADV_VAR_RATIO_NOISE_FLOOR = ADV_VAR_RATIO_TARGET × 0.01 — at that
// magnitude the streaming kernel's signal is dominated by numerical
// noise / cold-start, not a real "advantages are clean" indication
// worth shrinking the rollout for.
//
// Bootstrap discipline (per `pearl_first_observation_bootstrap`): the
// ISV slot starts at 0.0 sentinel. First emit writes
// `ROLLOUT_BOOTSTRAP = 2048` directly (the canonical PPO default).
// Subsequent emits Wiener-α blend with floor 0.4 (per
// `pearl_wiener_alpha_floor_for_nonstationary` — the advantage
// distribution drifts as π_new co-adapts, breaking stationarity).
//
// Bounds: rollout ∈ [256, 8192]. Below 256 PPO has too little data per
// update and the surrogate is dominated by sampling noise; above 8192
// updates lag the data so far behind that the importance ratios blow
// past the clip band.

#define RL_N_ROLLOUT_STEPS_INDEX           404
#define ROLLOUT_MIN                        256.0f
#define ROLLOUT_MAX                        8192.0f
// ISV-driven bootstrap + target.
#define RL_ROLLOUT_BOOTSTRAP_INDEX         475
#define RL_ADV_VAR_RATIO_TARGET_INDEX      449
// Schulman params from shared global slots (same as ppo_clip, target_tau).
#define RL_SCHULMAN_TOLERANCE_INDEX        468
#define RL_SCHULMAN_ADJUST_RATE_INDEX      469
#define ADV_VAR_RATIO_NOISE_FLOOR_FRAC     0.01f
#define WIENER_ALPHA_FLOOR                 0.4f

// ─────────────────────────────────────────────────────────────────────
// rl_rollout_steps_controller:
//   Single-thread controller — writes ONE float to
//   isv[RL_N_ROLLOUT_STEPS_INDEX].
//
// Inputs:
//   isv                          [≥ RL_N_ROLLOUT_STEPS_INDEX+1] — ISV bus
//   alpha                        — Wiener-α from controller stats;
//                                  floored at WIENER_ALPHA_FLOOR before
//                                  the blend.
//   advantage_var_over_abs_mean  — caller-computed
//                                  `var(A) / max(|mean A|, ε)` EMA;
//                                  Phase F produces this via a small
//                                  reduce kernel over the rollout
//                                  buffer's advantage column.
//
// Outputs:
//   isv[RL_N_ROLLOUT_STEPS_INDEX] — rollout length ∈ [ROLLOUT_MIN, ROLLOUT_MAX]
// ─────────────────────────────────────────────────────────────────────
// Phase R5: scalar input arg replaced with `input_slot` ISV index so the
// EMA producer (Phase R3 ema_update_per_step targeting
// ISV[RL_ADVANTAGE_VAR_RATIO_EMA_INDEX=421]) feeds this controller
// without any host roundtrip per `feedback_cpu_is_read_only`.
extern "C" __global__ void rl_rollout_steps_controller(
    float* __restrict__ isv,
    float           alpha,
    int             input_slot
) {
    if (threadIdx.x != 0 || blockIdx.x != 0) return;

    const float prev = isv[RL_N_ROLLOUT_STEPS_INDEX];
    // Bootstrap on sentinel 0.0 per pearl_first_observation_bootstrap.
    if (prev == 0.0f) {
        isv[RL_N_ROLLOUT_STEPS_INDEX] = isv[RL_ROLLOUT_BOOTSTRAP_INDEX];
        return;
    }

    // Cold-start gate: sentinel-zero EMA (streaming kernel hasn't
    // fired yet).
    const float advantage_var_over_abs_mean = isv[input_slot];
    if (advantage_var_over_abs_mean == 0.0f) return;

    // ISV-driven regression target — read from
    // RL_ADV_VAR_RATIO_TARGET_INDEX (seeded at trainer init).
    const float adv_var_target = isv[RL_ADV_VAR_RATIO_TARGET_INDEX];

    // Noise-floor gate: input below target × NOISE_FLOOR_FRAC is
    // dominated by numerical noise — hold rollout unchanged.
    // Mirrors the ppo_clip / target_tau controllers' design after the
    // alpha-rl-mjzfk multiplicative-blow-up incident.
    const float adv_var_noise_floor =
        adv_var_target * ADV_VAR_RATIO_NOISE_FLOOR_FRAC;
    if (advantage_var_over_abs_mean < adv_var_noise_floor) return;

    // Bounded multiplicative adjustment (Schulman-style adaptive).
    // At most ADV_VAR_RATIO_ADJUST_RATE × shift per step regardless of
    // how far input is from target. After several consecutive
    // out-of-band observations the output drifts smoothly toward
    // MIN/MAX, but a single observation can't slam it there.
    const float tolerance   = isv[RL_SCHULMAN_TOLERANCE_INDEX];
    const float adjust_rate = isv[RL_SCHULMAN_ADJUST_RATE_INDEX];
    float scale;
    if (advantage_var_over_abs_mean > adv_var_target * tolerance) {
        scale = adjust_rate;
    } else if (advantage_var_over_abs_mean < adv_var_target / tolerance) {
        scale = 1.0f / adjust_rate;
    } else {
        scale = 1.0f;
    }
    float target = prev * scale;
    target = fmaxf(ROLLOUT_MIN, fminf(target, ROLLOUT_MAX));

    // First-observation replace-directly per
    // `pearl_first_observation_bootstrap`.
    if (prev == isv[RL_ROLLOUT_BOOTSTRAP_INDEX]) {
        isv[RL_N_ROLLOUT_STEPS_INDEX] = target;
        return;
    }

    // Wiener-α blend with floor per pearl_wiener_alpha_floor_for_nonstationary.
    const float a   = fmaxf(alpha, WIENER_ALPHA_FLOOR);
    float       out = (1.0f - a) * prev + a * target;

    out = fmaxf(ROLLOUT_MIN, fminf(out, ROLLOUT_MAX));
    isv[RL_N_ROLLOUT_STEPS_INDEX] = out;
}