// rl_reward_clamp_controller.cu — emits adaptive reward-clamp bounds to
// ISV[RL_REWARD_CLAMP_WIN_INDEX=452] and ISV[RL_REWARD_CLAMP_LOSS_INDEX=453].
//
// Audit 2026-05-24: the static clamp `[-3.0, +1.0]` set up by
// `rl_isv_write` at trainer init was load-bearing on 85% of steps in
// alpha-rl-rmgm5 — pre-clamp p95 = 15.5× WIN, max = 2830× WIN. The
// adaptive bound below tracks the EMA of the per-step positive-tail
// max (published by `apply_reward_scale` into slot 478) so the C51
// Bellman target receives the actual winning-trade magnitude
// distribution rather than a near-constant +1.0 clipped signal.
//
// Reads:
//   isv[RL_POS_SCALED_REWARD_MAX_INDEX = 478]  — per-step max(scaled, 0)
//   isv[RL_POS_SCALED_REWARD_MAX_EMA_INDEX = 479] — controller's EMA state
//   isv[RL_REWARD_CLAMP_MARGIN_INDEX = 480]    — k multiplier (default 1.5)
//   isv[RL_REWARD_CLAMP_RATIO_INDEX = 481]     — loss/win ratio (default 3.0)
//
// Writes:
//   isv[RL_POS_SCALED_REWARD_MAX_EMA_INDEX = 479] — updated EMA
//   isv[RL_REWARD_CLAMP_WIN_INDEX  = 452]      — adaptive WIN bound
//   isv[RL_REWARD_CLAMP_LOSS_INDEX = 453]      — adaptive LOSS = ratio × WIN
//
// EMA discipline: Wiener-α blend with caller-supplied α (host-floored
// at WIENER_ALPHA_FLOOR per `pearl_wiener_alpha_floor_for_nonstationary`).
// Bootstrap on sentinel 0 per `pearl_first_observation_bootstrap` —
// first non-zero raw observation replaces the EMA slot directly so the
// blend always has a real prev value to mix with.
//
// Bounds: WIN ∈ [MIN_WIN=1.0, +∞). MIN_WIN preserves the floor from
// the static design — clamp never tightens below the C51 V_MAX. The
// upper bound was REMOVED 2026-05-24 follow-up — a hardcoded MAX_WIN
// can't be reconciled with adaptive control: rdgzl hit MAX_WIN=20 in
// every window because the positive-tail distribution has p99=76.9
// and max=2230; the cap turned the "adaptive" controller back into
// a static one.
//
// Safety reasoning for removing the cap: WIN = MARGIN × pos_max_ema.
// MARGIN is bounded [1.0, 5.0]. pos_max_ema is bounded by what
// apply_reward_scale produces, which is raw_reward × reward_scale,
// where reward_scale is itself bounded by its controller at [1e-3, 1e3].
// Raw realized PnL is finite per-trade (bounded by position size ×
// price-tick × ticks-moved). So WIN cannot diverge unboundedly in
// practice; the EMA + MARGIN cap is the structural bound.
//
// NOTE: C51's atom support in `bellman_target_projection.cu` is
// hardcoded `V_MAX=1.0`. Any Bellman target > 1.0 is categorically
// projected to atom 20 regardless of clamp. So lifting WIN beyond
// 1.0 helps V regression + PPO advantage (which see real magnitude)
// but NOT Q's distributional learning beyond +1.0. A follow-up that
// also lifts V_MAX/V_MIN would unlock the Q ceiling.
//
// LOSS = RATIO × WIN preserves the loss-aversion asymmetry from the
// `pearl_audit_unboundedness_for_implicit_asymmetry` design. Driven
// from a separate ISV slot so the asymmetry ratio is itself tunable
// without recompile.
//
// Per `feedback_no_atomicadd`: single-thread kernel. Per
// `feedback_cpu_is_read_only`: all state in ISV. Per
// `feedback_isv_for_adaptive_bounds`: hardcoded MIN/MAX retained per
// the user-stated "floors and clamp bounds" exemption (2026-05-24).

#define RL_POS_SCALED_REWARD_MAX_INDEX            478
#define RL_POS_SCALED_REWARD_MAX_EMA_INDEX        479
#define RL_REWARD_CLAMP_MARGIN_INDEX              480
#define RL_REWARD_CLAMP_RATIO_INDEX               481
#define RL_REWARD_CLAMP_WIN_INDEX                 452
#define RL_REWARD_CLAMP_LOSS_INDEX                453
// MARGIN adaptation slots (audit 2026-05-24 follow-up).
#define RL_REWARD_CLAMP_CLIP_RATE_EMA_INDEX       482
#define RL_REWARD_CLAMP_CLIP_RATE_TARGET_INDEX    483
// C51 atom span slots (audit 2026-05-24 second follow-up).
// Initial design: ratchet (monotone-grow). wwcsz followup
// 2026-05-24: replaced with slow EWMA (α=0.001, half-life ~700
// steps) to focus atom resolution on the ACTIVE reward range.
// Static ratchet wasted resolution on rare tails — with avg
// rewards in [-5, +5] but atom span at [-60, +20], Δz=4 meant Q
// couldn't differentiate "slightly winning" from "slightly losing"
// actions. Slow EWMA lets the span shrink toward the current
// active range while floors at [-1, +1] preserve the original C51
// baseline. The slow α gives Q's atom mapping time to be valid
// across encoder/head co-adaptation.
#define RL_C51_V_MAX_INDEX                        484
#define RL_C51_V_MIN_INDEX                        485

// Adaptive RATIO slots (wwcsz followup) — negative tail signal that
// drives RATIO = LOSS/WIN bound asymmetry. The static RATIO=3.0
// over-emphasised loss-aversion vs observed |loss|/|win| ≈ 0.83.
#define RL_NEG_SCALED_REWARD_MAX_INDEX            489
#define RL_NEG_SCALED_REWARD_MAX_EMA_INDEX        490

#define MIN_WIN                  1.0f
#define MIN_RATIO                1.0f      // no inverted asymmetry (loss never < win)
#define MAX_RATIO                3.0f      // no worse than original loss-aversion
#define V_BOUND_FLOOR            1.0f      // |V_MIN| and V_MAX both ≥ 1.0
#define V_BOUND_EWMA_ALPHA       0.001f    // slow — half-life ~700 steps
#define WIENER_ALPHA_FLOOR       0.4f

// MARGIN Schulman bounded-step bounds + adjust rate per
// `pearl_multiplicative_controllers_need_bounded_step_and_noise_floor`.
// TOLERANCE=1.5 → dead-zone is [target/1.5, target×1.5] = [0.033, 0.075];
// ADJUST_RATE=1.2 → MARGIN moves by ≤20% per controller call (one per
// trainer step). Bounds [MIN_MARGIN=1.0, MAX_MARGIN=5.0] — MIN preserves
// the initial passthrough behaviour; MAX × (mean pos_max_ema≈3-5) →
// WIN_eff up to ~15-25 which is well within MAX_WIN=50 ceiling.
#define MIN_MARGIN               1.0f
#define MAX_MARGIN               5.0f
#define MARGIN_TOLERANCE         1.5f
#define MARGIN_ADJUST_RATE       1.2f
#define CLIP_RATE_EMA_ALPHA      0.05f

extern "C" __global__ void rl_reward_clamp_controller(
    float* __restrict__ isv,
    float               alpha
) {
    if (threadIdx.x != 0 || blockIdx.x != 0) return;

    const float pos_max = isv[RL_POS_SCALED_REWARD_MAX_INDEX];
    const float ema_prev = isv[RL_POS_SCALED_REWARD_MAX_EMA_INDEX];

    // EMA maintenance — sparse-sample-aware. Only update the EMA on
    // steps that ACTUALLY observed a positive reward (pos_max > 0).
    //
    // Audit 2026-05-24 (vj5f6 follow-up): the prior implementation
    // blended pos_max=0 into the EMA via the Wiener-α step, which
    // exponentially decayed the EMA toward 0 during dry-spell windows
    // (no closed winning trades). With α=0.4, ten consecutive
    // dry steps decay EMA by 0.6^10 ≈ 0.006, collapsing the WIN
    // bound back to the MIN_WIN floor. WIN_clamp was observed
    // oscillating 1.0 ↔ 67.0 across 40k steps in vj5f6.
    //
    // Fix: pos_max=0 means "no signal this step," not "win magnitude
    // is zero." Skip EMA updates during dry spells; the EMA retains
    // the last winning-period estimate until the next observation.
    float ema_new = ema_prev;
    if (pos_max > 0.0f) {
        if (ema_prev == 0.0f) {
            ema_new = pos_max;  // first-observation bootstrap
        } else {
            const float a = fmaxf(alpha, WIENER_ALPHA_FLOOR);
            ema_new = (1.0f - a) * ema_prev + a * pos_max;
        }
        isv[RL_POS_SCALED_REWARD_MAX_EMA_INDEX] = ema_new;
    }
    // If ema_prev was 0 AND pos_max was 0 → ema_new still 0 here →
    // WIN_eff below = max(MIN_WIN, MARGIN × 0) = MIN_WIN. That's
    // the correct "no data yet" behavior (statically-seeded WIN=1.0
    // remains effective).

    // ── MARGIN adaptation from clip-rate EMA (audit follow-up). ───
    //
    // Step 1: compute clip indicator for THIS step — did the pos_max
    // we just observed exceed the WIN that was active during the
    // apply_reward_scale call? slot 452 still holds the WIN that was
    // used (we'll overwrite it below with the new WIN). Only counts
    // when there WAS a positive reward (pos_max > 0) — clip-rate is
    // a fraction of positive-reward steps, not all steps.
    const float win_active = isv[RL_REWARD_CLAMP_WIN_INDEX];
    float clip_indicator = 0.0f;
    if (pos_max > 0.0f) {
        clip_indicator = (pos_max > win_active) ? 1.0f : 0.0f;
    }

    // Step 2: blend clip indicator into EMA. Sparse-aware — only
    // update when pos_max > 0 (matches the pos_max EMA discipline).
    // A step with pos_max=0 isn't "0% clipped"; it's "no data." Mixing
    // dry steps in would bias clip_rate_ema downward and the MARGIN
    // controller would shrink WIN unnecessarily.
    const float clip_rate_prev = isv[RL_REWARD_CLAMP_CLIP_RATE_EMA_INDEX];
    float clip_rate_new = clip_rate_prev;
    if (pos_max > 0.0f) {
        if (clip_rate_prev == 0.0f) {
            clip_rate_new = clip_indicator;  // bootstrap
        } else {
            clip_rate_new = (1.0f - CLIP_RATE_EMA_ALPHA) * clip_rate_prev
                          + CLIP_RATE_EMA_ALPHA * clip_indicator;
        }
        isv[RL_REWARD_CLAMP_CLIP_RATE_EMA_INDEX] = clip_rate_new;
    }

    // Step 3: Schulman bounded step on MARGIN — only adjust when we
    // actually have a fresh clip-rate observation. If pos_max=0 this
    // step, MARGIN retains its prior value (consistent with the EMA
    // behavior above; the controller doesn't ratchet on stale data).
    float margin = isv[RL_REWARD_CLAMP_MARGIN_INDEX];
    if (pos_max > 0.0f) {
        const float clip_target = isv[RL_REWARD_CLAMP_CLIP_RATE_TARGET_INDEX];
        const float upper = clip_target * MARGIN_TOLERANCE;
        const float lower = clip_target / MARGIN_TOLERANCE;
        if (clip_rate_new > upper) {
            margin = fminf(MAX_MARGIN, margin * MARGIN_ADJUST_RATE);
        } else if (clip_rate_new < lower) {
            margin = fmaxf(MIN_MARGIN, margin / MARGIN_ADJUST_RATE);
        }
        isv[RL_REWARD_CLAMP_MARGIN_INDEX] = margin;
    }

    // ── Adaptive RATIO from observed neg/pos EMAs (wwcsz followup). ──
    //
    // The static RATIO=3.0 baked 3:1 loss-aversion into Q's value
    // representation. wwcsz showed observed |loss|/|win| ≈ 0.83,
    // making Q over-cautious. Track per-step max(-scaled, 0) via a
    // sparse-aware EMA (same discipline as pos_max_ema) and compute
    // RATIO = clamp(MIN=1.0, neg_ema/pos_ema, MAX=3.0). Floor 1.0
    // prevents inverted asymmetry; ceiling 3.0 preserves original
    // loss-aversion as the worst case.
    const float neg_max  = isv[RL_NEG_SCALED_REWARD_MAX_INDEX];
    const float neg_prev = isv[RL_NEG_SCALED_REWARD_MAX_EMA_INDEX];
    float       neg_new  = neg_prev;
    if (neg_max > 0.0f) {
        if (neg_prev == 0.0f) {
            neg_new = neg_max;
        } else {
            const float a = fmaxf(alpha, WIENER_ALPHA_FLOOR);
            neg_new = (1.0f - a) * neg_prev + a * neg_max;
        }
        isv[RL_NEG_SCALED_REWARD_MAX_EMA_INDEX] = neg_new;
    }
    // Adaptive RATIO. Defer if EMAs not warmed up yet — keep
    // statically-seeded RATIO=3.0 until both EMAs are non-zero.
    if (ema_new > 0.0f && neg_new > 0.0f) {
        const float ratio_target = neg_new / ema_new;
        const float ratio_clamped = fmaxf(MIN_RATIO,
                                          fminf(MAX_RATIO, ratio_target));
        isv[RL_REWARD_CLAMP_RATIO_INDEX] = ratio_clamped;
    }

    // Step 4 (G.2): WIN / LOSS clamp bounds remain STRUCTURAL — the
    // G.2 feedback loop required BOTH clamp widening AND span widening.
    // With clamp frozen here, only the atom span adapts (Step 5 below).
    // Suppress unused warnings on diagnostic-only state.
    (void) margin;

    // Adaptive LOSS clamp from observed neg/pos ratio. WIN stays
    // structural at 1.0; LOSS adapts to actual loss magnitude to
    // give Q accurate resolution without over-allocating to rare tails.
    if (ema_new > 0.0f && neg_new > 0.0f) {
        const float observed_ratio = neg_new / ema_new;
        const float adaptive_loss = fmaxf(1.0f, fminf(observed_ratio * 1.1f, 3.0f));
        isv[RL_REWARD_CLAMP_LOSS_INDEX] = adaptive_loss;
    }

    // ── Step 5: C51 atom span adaptation from observed reward EMAs. ──
    //
    // G.2 disabled this because atom-span growth + clamp-bound growth
    // created a positive feedback loop. With clamp bounds FROZEN (Step
    // 4), the loop can't form — the clamp caps the reward magnitude
    // regardless of atom span.
    //
    // The span slowly tracks the observed reward tail EMAs so Q's
    // distributional resolution covers the ACTUAL reward range. Without
    // this, rewards exceeding the static span project to edge atoms and
    // Q loses magnitude discrimination (wr plateaus at ~0.46).
    //
    // Floor at V_BOUND_FLOOR (1.0) preserves baseline resolution.
    // Ceiling at clamp bounds (WIN=1.0, LOSS=3.0) prevents runaway.
    // Slow EWMA (α=0.001, half-life ~700 steps) lets Q's atom mapping
    // adapt gradually without whipsawing.
    // Atom span anchors on the CLAMP bounds — the post-clamp reward
    // range that Q's Bellman targets actually see. Pre-clamp EMAs
    // (pos_ema ≈ 50) would blow atoms to [-50, +50] with Δ_z=5,
    // destroying resolution. The clamp bounds (WIN, LOSS) are the
    // structural ceiling on what rewards enter Q.
    const float win_bound  = isv[RL_REWARD_CLAMP_WIN_INDEX];   // 1.0
    const float loss_bound = isv[RL_REWARD_CLAMP_LOSS_INDEX];  // 3.0
    {
        const float v_max_prev = isv[RL_C51_V_MAX_INDEX];
        const float v_max_target = fmaxf(V_BOUND_FLOOR, win_bound);
        const float v_max_new = (v_max_prev == 0.0f)
            ? v_max_target
            : (1.0f - V_BOUND_EWMA_ALPHA) * v_max_prev
              + V_BOUND_EWMA_ALPHA * v_max_target;
        isv[RL_C51_V_MAX_INDEX] = fmaxf(V_BOUND_FLOOR, v_max_new);
    }
    {
        const float v_min_prev = isv[RL_C51_V_MIN_INDEX];
        const float v_min_target = fminf(-V_BOUND_FLOOR, -loss_bound);
        const float v_min_new = (v_min_prev == 0.0f)
            ? v_min_target
            : (1.0f - V_BOUND_EWMA_ALPHA) * v_min_prev
              + V_BOUND_EWMA_ALPHA * v_min_target;
        isv[RL_C51_V_MIN_INDEX] = fminf(-V_BOUND_FLOOR, v_min_new);
    }
}