foxhunt/crates/ml-alpha/cuda/rl_reward_scale_controller.cu

// rl_reward_scale_controller.cu — emits per-trade reward standardisation
// scale to ISV[RL_REWARD_SCALE_INDEX=406].
//
// Phase F of the integrated RL trainer
// (docs/superpowers/plans/2026-05-22-integrated-rl-trainer.md).
//
// The C51 distributional Q-head uses a fixed atom support [-1, +1] with
// N_ATOMS=21. For the categorical Bellman projection to work well, scaled
// rewards must sit comfortably within that support — a typical realized
// |PnL| trade should map to a magnitude ≈ 1.0. Without standardisation,
// a strategy trading 1-tick ES moves (≈$12.50 per contract) would stuff
// nearly all probability mass into a handful of atoms near zero, starving
// the distributional representation.
//
// This controller emits the inverse of the mean absolute realized PnL:
//
//   target_scale = 1.0 / max(mean_abs_pnl_ema, EPS_PNL)
//
// so that reward_scaled = reward_raw * isv[RL_REWARD_SCALE_INDEX] keeps
// a typical trade's realized PnL ≈ 1.0 (in atom-support units). The EMA
// is maintained on the Rust side (Phase F.3) from the LobEnv adapter's
// reward stream over the last ~100 closes, and passed in as a scalar.
//
// Bootstrap discipline (per `pearl_first_observation_bootstrap`): the ISV
// slot starts at 0.0 sentinel. First emit writes REWARD_SCALE_BOOTSTRAP=1.0
// directly — raw passthrough until the EMA stabilises. Subsequent emits
// Wiener-α blend with floor 0.4 (per `pearl_wiener_alpha_floor_for_nonstationary`
// — the trade-magnitude distribution drifts as the policy co-adapts,
// breaking stationarity).
//
// Bounds: scale ∈ [1e-3, 1e3]. Five orders of magnitude either side of
// unit scale handles micro-PnL noise scenarios at the low end and
// catastrophic-loss-magnitude outliers at the high end, while preventing
// runaway. If the typical |PnL| stabilises at e.g. 50 USD, the controller
// converges to scale ≈ 0.02 (within bounds); at $0.01 noise it caps at
// 1e3 rather than exploding.

#define RL_REWARD_SCALE_INDEX   406
#define REWARD_SCALE_MIN        1e-3f
#define REWARD_SCALE_MAX        1e3f
// ISV-driven bootstrap (was 1.0f #define).
#define RL_REWARD_SCALE_BOOTSTRAP_INDEX   476
// Floor for the mean_abs_pnl_ema denominator to guard against div-by-zero
// when the EMA hasn't accumulated any closed trades yet.
#define EPS_PNL                 1e-3f
#define WIENER_ALPHA_FLOOR      0.4f

// ─────────────────────────────────────────────────────────────────────
// rl_reward_scale_controller:
//   Single-thread controller — writes ONE float to
//   isv[RL_REWARD_SCALE_INDEX].
//
// Inputs:
//   isv                  [≥ RL_REWARD_SCALE_INDEX+1] — ISV bus
//   alpha_step           — Wiener-α from the controller's signal stats;
//                          floored at WIENER_ALPHA_FLOOR before the blend.
//                          Named `alpha_step` to disambiguate from PER's α
//                          (ISV[405]) and from the ISV slot name itself.
//   mean_abs_pnl_ema     — caller-computed EMA of |realized_pnl_usd| over
//                          the last ~100 closed trades. Phase F.3 maintains
//                          this from the LobEnv adapter reward stream.
//
// Outputs:
//   isv[RL_REWARD_SCALE_INDEX] — reward standardisation scale ∈
//                                 [REWARD_SCALE_MIN, REWARD_SCALE_MAX]
// ─────────────────────────────────────────────────────────────────────
// Phase R5: scalar input arg replaced with `input_slot` ISV index so the
// EMA producer (Phase R3 ema_update_on_done targeting
// ISV[RL_MEAN_ABS_PNL_EMA_INDEX=423]) feeds this controller without
// any host roundtrip per `feedback_cpu_is_read_only`.
extern "C" __global__ void rl_reward_scale_controller(
    float* __restrict__ isv,
    float           alpha_step,
    int             input_slot
) {
    if (threadIdx.x != 0 || blockIdx.x != 0) return;

    const float prev = isv[RL_REWARD_SCALE_INDEX];
    // Bootstrap on sentinel 0.0 per pearl_first_observation_bootstrap.
    if (prev == 0.0f) {
        isv[RL_REWARD_SCALE_INDEX] = isv[RL_REWARD_SCALE_BOOTSTRAP_INDEX];
        return;
    }

    // Cold-start gate: when mean_abs_pnl_ema is
    // sentinel-zero (no closed trade yet — typical for the first
    // several steps before any trail-stop fires), the reciprocal
    // target `1 / max(0, EPS_PNL) = 1/EPS_PNL ≈ 1000` slams to
    // REWARD_SCALE_MAX. The Wiener α-floor=0.4 then drags scale
    // toward MAX every step: by step 6 scale = 972, by step 7's
    // first closed trade the scale was 583× — multiplying a real
    // $832 trade close into a 485,256-magnitude reward fed straight
    // into Q/V backward. l_v jumped to 15.9 on that step, and the
    // overshoot took ~30 steps to settle.
    //
    // Hold at bootstrap (1.0 = raw passthrough) until the first
    // closed trade pushes mean_abs_pnl_ema off sentinel. This is
    // exactly what `pearl_first_observation_bootstrap` mandates:
    // "sentinel = 0" means "no signal" — adapt against signal, not
    // noise.
    const float mean_abs_pnl_ema = isv[input_slot];
    if (mean_abs_pnl_ema == 0.0f) return;
    // target_scale = 1.0 / max(mean_abs_pnl_ema, EPS_PNL).
    // Larger typical PnL → smaller scale (reward is divided down toward ±1).
    // Smaller typical PnL → larger scale (reward is amplified toward ±1).
    const float denom  = fmaxf(mean_abs_pnl_ema, EPS_PNL);
    float       target = 1.0f / denom;
    target = fmaxf(REWARD_SCALE_MIN, fminf(target, REWARD_SCALE_MAX));

    // First-observation replace-directly per
    // `pearl_first_observation_bootstrap`. With prev=1.0 (bootstrap)
    // and a target far from 1.0 on the first real PnL signal, the
    // Wiener blend at α=0.4 would leave 60% of the bootstrap weight
    // in the output — a single $832 trade close × scale=0.6 = $499
    // feeds V regression at magnitude 500× the atom support's
    // ±1 expectation. Replacing directly with target eliminates
    // this cold-start contamination.
    if (prev == isv[RL_REWARD_SCALE_BOOTSTRAP_INDEX]) {
        isv[RL_REWARD_SCALE_INDEX] = target;
        return;
    }

    // Wiener-α blend with floor per pearl_wiener_alpha_floor_for_nonstationary.
    const float a   = fmaxf(alpha_step, WIENER_ALPHA_FLOOR);
    float       out = (1.0f - a) * prev + a * target;

    out = fmaxf(REWARD_SCALE_MIN, fminf(out, REWARD_SCALE_MAX));
    isv[RL_REWARD_SCALE_INDEX] = out;
}