// rl_reward_clamp_controller.cu — emits adaptive reward-clamp bounds to
// ISV[RL_REWARD_CLAMP_WIN_INDEX=452] and ISV[RL_REWARD_CLAMP_LOSS_INDEX=453].
//
// Audit 2026-05-24: the static clamp `[-3.0, +1.0]` set up by
// `rl_isv_write` at trainer init was load-bearing on 85% of steps in
// alpha-rl-rmgm5 — pre-clamp p95 = 15.5× WIN, max = 2830× WIN. The
// adaptive bound below tracks the EMA of the per-step positive-tail
// max (published by `apply_reward_scale` into slot 478) so the C51
// Bellman target receives the actual winning-trade magnitude
// distribution rather than a near-constant +1.0 clipped signal.
//
// Reads:
//   isv[RL_POS_SCALED_REWARD_MAX_INDEX = 478]  — per-step max(scaled, 0)
//   isv[RL_POS_SCALED_REWARD_MAX_EMA_INDEX = 479] — controller's EMA state
//   isv[RL_REWARD_CLAMP_MARGIN_INDEX = 480]    — k multiplier (default 1.5)
//   isv[RL_REWARD_CLAMP_RATIO_INDEX = 481]     — loss/win ratio (default 3.0)
//
// Writes:
//   isv[RL_POS_SCALED_REWARD_MAX_EMA_INDEX = 479] — updated EMA
//   isv[RL_REWARD_CLAMP_WIN_INDEX  = 452]      — adaptive WIN bound
//   isv[RL_REWARD_CLAMP_LOSS_INDEX = 453]      — adaptive LOSS = ratio × WIN
//
// EMA discipline: Wiener-α blend with caller-supplied α (host-floored
// at WIENER_ALPHA_FLOOR per `pearl_wiener_alpha_floor_for_nonstationary`).
// Bootstrap on sentinel 0 per `pearl_first_observation_bootstrap` —
// first non-zero raw observation replaces the EMA slot directly so the
// blend always has a real prev value to mix with.
//
// Bounds: WIN ∈ [MIN_WIN=1.0, +∞). MIN_WIN preserves the floor from
// the static design — clamp never tightens below the C51 V_MAX. The
// upper bound was REMOVED 2026-05-24 follow-up — a hardcoded MAX_WIN
// can't be reconciled with adaptive control: rdgzl hit MAX_WIN=20 in
// every window because the positive-tail distribution has p99=76.9
// and max=2230; the cap turned the "adaptive" controller back into
// a static one.
//
// Safety reasoning for removing the cap: WIN = MARGIN × pos_max_ema.
// MARGIN is bounded [1.0, 5.0]. pos_max_ema is bounded by what
// apply_reward_scale produces, which is raw_reward × reward_scale,
// where reward_scale is itself bounded by its controller at [1e-3, 1e3].
// Raw realized PnL is finite per-trade (bounded by position size ×
// price-tick × ticks-moved). So WIN cannot diverge unboundedly in
// practice; the EMA + MARGIN cap is the structural bound.
//
// NOTE: C51's atom support in `bellman_target_projection.cu` is
// hardcoded `V_MAX=1.0`. Any Bellman target > 1.0 is categorically
// projected to atom 20 regardless of clamp. So lifting WIN beyond
// 1.0 helps V regression + PPO advantage (which see real magnitude)
// but NOT Q's distributional learning beyond +1.0. A follow-up that
// also lifts V_MAX/V_MIN would unlock the Q ceiling.
//
// LOSS = RATIO × WIN preserves the loss-aversion asymmetry from the
// `pearl_audit_unboundedness_for_implicit_asymmetry` design. Driven
// from a separate ISV slot so the asymmetry ratio is itself tunable
// without recompile.
//
// Per `feedback_no_atomicadd`: single-thread kernel. Per
// `feedback_cpu_is_read_only`: all state in ISV. Per
// `feedback_isv_for_adaptive_bounds`: hardcoded MIN/MAX retained per
// the user-stated "floors and clamp bounds" exemption (2026-05-24).

#define RL_POS_SCALED_REWARD_MAX_INDEX            478
#define RL_POS_SCALED_REWARD_MAX_EMA_INDEX        479
#define RL_REWARD_CLAMP_MARGIN_INDEX              480
#define RL_REWARD_CLAMP_RATIO_INDEX               481
#define RL_REWARD_CLAMP_WIN_INDEX                 452
#define RL_REWARD_CLAMP_LOSS_INDEX                453
// MARGIN adaptation slots (audit 2026-05-24 follow-up).
#define RL_REWARD_CLAMP_CLIP_RATE_EMA_INDEX       482
#define RL_REWARD_CLAMP_CLIP_RATE_TARGET_INDEX    483
// C51 atom span slots (audit 2026-05-24 second follow-up).
// Initial design: ratchet (monotone-grow). wwcsz followup
// 2026-05-24: replaced with slow EWMA (α=0.001, half-life ~700
// steps) to focus atom resolution on the ACTIVE reward range.
// Static ratchet wasted resolution on rare tails — with avg
// rewards in [-5, +5] but atom span at [-60, +20], Δz=4 meant Q
// couldn't differentiate "slightly winning" from "slightly losing"
// actions. Slow EWMA lets the span shrink toward the current
// active range while floors at [-1, +1] preserve the original C51
// baseline. The slow α gives Q's atom mapping time to be valid
// across encoder/head co-adaptation.
#define RL_C51_V_MAX_INDEX                        484
#define RL_C51_V_MIN_INDEX                        485

// Adaptive RATIO slots (wwcsz followup) — negative tail signal that
// drives RATIO = LOSS/WIN bound asymmetry. The static RATIO=3.0
// over-emphasised loss-aversion vs observed |loss|/|win| ≈ 0.83.
#define RL_NEG_SCALED_REWARD_MAX_INDEX            489
#define RL_NEG_SCALED_REWARD_MAX_EMA_INDEX        490

// Adaptive controller floors (spec 2026-05-30-adaptive-controller-floor-design):
// every prior `#define` clamp-bound constant becomes ISV-driven so it can
// be tuned at runtime via re-seed instead of recompile. Defaults preserve
// the historic behaviour exactly.
#define RL_REWARD_CLAMP_V_BOUND_FLOOR_INDEX          633
#define RL_REWARD_CLAMP_V_BOUND_EWMA_ALPHA_INDEX     634
#define RL_REWARD_CLAMP_MIN_WIN_INDEX                635
#define RL_REWARD_CLAMP_MIN_RATIO_INDEX              636
#define RL_REWARD_CLAMP_MAX_RATIO_INDEX              637
#define RL_REWARD_CLAMP_MIN_MARGIN_INDEX             654
#define RL_REWARD_CLAMP_MAX_MARGIN_INDEX             655
#define RL_REWARD_CLAMP_MARGIN_TOLERANCE_INDEX       656
#define RL_REWARD_CLAMP_MARGIN_ADJUST_RATE_INDEX     657
#define RL_REWARD_CLAMP_CLIP_RATE_EMA_ALPHA_INDEX    658
#define RL_WIENER_ALPHA_FLOOR_INDEX                  659

// pos_max_ema cold-start redesign (2026-05-31 B-2 spec). Growth cap is
// ISV-driven (was `#define POS_MAX_EMA_MAX_GROWTH_PER_STEP 1.5f`) per
// `feedback_isv_for_adaptive_bounds`. Two slots:
//   718: base cap value (√1.5 ≈ 1.225 nets ~1.5×/step under twice-per-step invocation)
//   719: adaptive CV-gain (0 = disabled; >0 enables Welford-CV-driven cap relaxation)
#define RL_POS_MAX_EMA_GROWTH_CAP_BASE_INDEX    718
#define RL_POS_MAX_EMA_GROWTH_CAP_CV_GAIN_INDEX 719
// Welford triplet for RL_REWARD_MAGNITUDE_EMA_INDEX (used to compute signal CV
// for adaptive layer). Slots from rl_signal_variance_update wiring.
#define RL_REWARD_MAG_VAR_COUNT_INDEX  615
#define RL_REWARD_MAG_VAR_MEAN_INDEX   616
#define RL_REWARD_MAG_VAR_M2_INDEX     617

extern "C" __global__ void rl_reward_clamp_controller(
    float* __restrict__ isv,
    float               alpha
) {
    if (threadIdx.x != 0 || blockIdx.x != 0) return;

    // ── ISV-driven design constants (spec 2026-05-30) ──
    // Loaded once at the top of the kernel so they're consistent across
    // all sub-controllers (EMA → MARGIN → RATIO → C51 span) within this
    // step. Default values preserve historic behaviour exactly.
    const float v_bound_floor       = isv[RL_REWARD_CLAMP_V_BOUND_FLOOR_INDEX];
    const float v_bound_ewma_alpha  = isv[RL_REWARD_CLAMP_V_BOUND_EWMA_ALPHA_INDEX];
    const float min_ratio_isv       = isv[RL_REWARD_CLAMP_MIN_RATIO_INDEX];
    const float max_ratio_isv       = isv[RL_REWARD_CLAMP_MAX_RATIO_INDEX];
    const float min_margin_isv      = isv[RL_REWARD_CLAMP_MIN_MARGIN_INDEX];
    const float max_margin_isv      = isv[RL_REWARD_CLAMP_MAX_MARGIN_INDEX];
    const float margin_tolerance    = isv[RL_REWARD_CLAMP_MARGIN_TOLERANCE_INDEX];
    const float margin_adjust_rate  = isv[RL_REWARD_CLAMP_MARGIN_ADJUST_RATE_INDEX];
    const float clip_rate_ema_alpha = isv[RL_REWARD_CLAMP_CLIP_RATE_EMA_ALPHA_INDEX];
    const float wiener_floor        = isv[RL_WIENER_ALPHA_FLOOR_INDEX];

    const float pos_max = isv[RL_POS_SCALED_REWARD_MAX_INDEX];
    const float ema_prev = isv[RL_POS_SCALED_REWARD_MAX_EMA_INDEX];

    // EMA maintenance — sparse-sample-aware. Only update the EMA on
    // steps that ACTUALLY observed a positive reward (pos_max > 0).
    //
    // Audit 2026-05-24 (vj5f6 follow-up): the prior implementation
    // blended pos_max=0 into the EMA via the Wiener-α step, which
    // exponentially decayed the EMA toward 0 during dry-spell windows
    // (no closed winning trades). With α=0.4, ten consecutive
    // dry steps decay EMA by 0.6^10 ≈ 0.006, collapsing the WIN
    // bound back to the MIN_WIN floor. WIN_clamp was observed
    // oscillating 1.0 ↔ 67.0 across 40k steps in vj5f6.
    //
    // Fix: pos_max=0 means "no signal this step," not "win magnitude
    // is zero." Skip EMA updates during dry spells; the EMA retains
    // the last winning-period estimate until the next observation.
    // ── Adaptive growth cap (2026-05-31 B-2 spec, ISV-driven per
    //    feedback_isv_for_adaptive_bounds). When cv_gain=0, falls back to
    //    fixed base. Non-zero cv_gain enables Welford-CV-driven relaxation:
    //      stable regime (CV→0): growth_cap = base (tight)
    //      volatile regime (CV→1): growth_cap = base + cv_gain (loose)
    const float growth_base    = isv[RL_POS_MAX_EMA_GROWTH_CAP_BASE_INDEX];
    const float growth_cv_gain = isv[RL_POS_MAX_EMA_GROWTH_CAP_CV_GAIN_INDEX];
    float growth_cap = growth_base;
    if (growth_cv_gain > 0.0f) {
        const float wf_count = isv[RL_REWARD_MAG_VAR_COUNT_INDEX];
        if (wf_count > 1.0f) {
            const float wf_m2   = isv[RL_REWARD_MAG_VAR_M2_INDEX];
            const float wf_mean = isv[RL_REWARD_MAG_VAR_MEAN_INDEX];
            const float wf_var  = wf_m2 / (wf_count - 1.0f);
            const float wf_cv   = (wf_mean > 1e-6f) ? sqrtf(wf_var) / wf_mean : 0.0f;
            growth_cap = growth_base + growth_cv_gain * fminf(wf_cv, 1.0f);
        }
    }

    // ── Single uniform update rule (2026-05-31 B-2 spec §3.3). The
    //    first-observation branch `if (ema_prev == 0.0f) ema_new = pos_max;`
    //    was REMOVED — it let a single cold-start fat-tail observation seed
    //    pos_max_ema at unbounded magnitude (jnct8 step 5: pos_ema=879
    //    directly from first positive reward). EMA slots are now bootstrapped
    //    to MIN_WIN=1.0 in `with_controllers_bootstrapped`, so ema_prev is
    //    never 0 at run start. Cold-start adaptation is geometric at
    //    growth_cap per invocation (~1.5×/step under twice-per-step invocation
    //    → ~30 steps to reach 11500× initial = saturates any realistic
    //    magnitude). Dry-spell skip preserved via the outer pos_max>0 check.
    float ema_new = ema_prev;
    if (pos_max > 0.0f) {
        const float a = fmaxf(alpha, wiener_floor);
        const float ema_raw = (1.0f - a) * ema_prev + a * pos_max;
        const float max_grow = ema_prev * growth_cap;
        ema_new = fminf(ema_raw, max_grow);
        isv[RL_POS_SCALED_REWARD_MAX_EMA_INDEX] = ema_new;
    }
    // If ema_prev was 0 AND pos_max was 0 → ema_new still 0 here →
    // WIN_eff below = max(MIN_WIN, MARGIN × 0) = MIN_WIN. That's
    // the correct "no data yet" behavior (statically-seeded WIN=1.0
    // remains effective).

    // ── MARGIN adaptation from clip-rate EMA (audit follow-up). ───
    //
    // Step 1: compute clip indicator for THIS step — did the pos_max
    // we just observed exceed the WIN that was active during the
    // apply_reward_scale call? slot 452 still holds the WIN that was
    // used (we'll overwrite it below with the new WIN). Only counts
    // when there WAS a positive reward (pos_max > 0) — clip-rate is
    // a fraction of positive-reward steps, not all steps.
    const float win_active = isv[RL_REWARD_CLAMP_WIN_INDEX];
    float clip_indicator = 0.0f;
    if (pos_max > 0.0f) {
        clip_indicator = (pos_max > win_active) ? 1.0f : 0.0f;
    }

    // Step 2: blend clip indicator into EMA. Sparse-aware — only
    // update when pos_max > 0 (matches the pos_max EMA discipline).
    // A step with pos_max=0 isn't "0% clipped"; it's "no data." Mixing
    // dry steps in would bias clip_rate_ema downward and the MARGIN
    // controller would shrink WIN unnecessarily.
    const float clip_rate_prev = isv[RL_REWARD_CLAMP_CLIP_RATE_EMA_INDEX];
    float clip_rate_new = clip_rate_prev;
    if (pos_max > 0.0f) {
        if (clip_rate_prev == 0.0f) {
            clip_rate_new = clip_indicator;  // bootstrap
        } else {
            clip_rate_new = (1.0f - clip_rate_ema_alpha) * clip_rate_prev
                          + clip_rate_ema_alpha * clip_indicator;
        }
        isv[RL_REWARD_CLAMP_CLIP_RATE_EMA_INDEX] = clip_rate_new;
    }

    // Step 3: Schulman bounded step on MARGIN — only adjust when we
    // actually have a fresh clip-rate observation. If pos_max=0 this
    // step, MARGIN retains its prior value (consistent with the EMA
    // behavior above; the controller doesn't ratchet on stale data).
    float margin = isv[RL_REWARD_CLAMP_MARGIN_INDEX];
    if (pos_max > 0.0f) {
        const float clip_target = isv[RL_REWARD_CLAMP_CLIP_RATE_TARGET_INDEX];
        const float upper = clip_target * margin_tolerance;
        const float lower = clip_target / margin_tolerance;
        if (clip_rate_new > upper) {
            margin = fminf(max_margin_isv, margin * margin_adjust_rate);
        } else if (clip_rate_new < lower) {
            margin = fmaxf(min_margin_isv, margin / margin_adjust_rate);
        }
        isv[RL_REWARD_CLAMP_MARGIN_INDEX] = margin;
    }

    // ── Adaptive RATIO from observed neg/pos EMAs (wwcsz followup). ──
    //
    // The static RATIO=3.0 baked 3:1 loss-aversion into Q's value
    // representation. wwcsz showed observed |loss|/|win| ≈ 0.83,
    // making Q over-cautious. Track per-step max(-scaled, 0) via a
    // sparse-aware EMA (same discipline as pos_max_ema) and compute
    // RATIO = clamp(MIN=1.0, neg_ema/pos_ema, MAX=3.0). Floor 1.0
    // prevents inverted asymmetry; ceiling 3.0 preserves original
    // loss-aversion as the worst case.
    // Single uniform update (B-2 spec §3.3 — same as pos_max_ema above).
    // neg_prev is bootstrapped to cold_start=1.0; no first-observation special case.
    // Same growth_cap (computed above) applies — neg-side cascade is mirror-image
    // of pos-side, same protection needed.
    const float neg_max  = isv[RL_NEG_SCALED_REWARD_MAX_INDEX];
    const float neg_prev = isv[RL_NEG_SCALED_REWARD_MAX_EMA_INDEX];
    float       neg_new  = neg_prev;
    if (neg_max > 0.0f) {
        const float a = fmaxf(alpha, wiener_floor);
        const float neg_raw = (1.0f - a) * neg_prev + a * neg_max;
        const float neg_max_grow = neg_prev * growth_cap;
        neg_new = fminf(neg_raw, neg_max_grow);
        isv[RL_NEG_SCALED_REWARD_MAX_EMA_INDEX] = neg_new;
    }
    // Adaptive RATIO. Defer if EMAs not warmed up yet — keep
    // statically-seeded RATIO=3.0 until both EMAs are non-zero.
    if (ema_new > 0.0f && neg_new > 0.0f) {
        const float ratio_target = neg_new / ema_new;
        const float ratio_clamped = fmaxf(min_ratio_isv,
                                          fminf(max_ratio_isv, ratio_target));
        isv[RL_REWARD_CLAMP_RATIO_INDEX] = ratio_clamped;
    }

    // ── Step 4: Adaptive WIN / LOSS clamp writeback. ──────────────
    //
    // History: a 2026-05-24 G.2 patch froze the clamp bounds at the
    // trainer-seeded WIN=1.0, LOSS=3.0 over concern that simultaneously
    // adapting clamps + atom span could form a positive feedback loop.
    //
    // Empirical refutation (alpha-rl-gwmkf step 4083, 2026-05-30):
    // with R-multiple 43 trade distributions (avg_win $7062, avg_loss
    // $163) the frozen WIN=1.0 caused clip_rate_ema = 0.9999 —
    // essentially every win gets clamped, V regression sees identical
    // targets for +$500 wins and +$42k wins, and PPO advantage loses
    // magnitude signal. Diagnosis: starved policy gradient.
    //
    // Safety argument: the MARGIN controller above already saturates
    // at MAX_MARGIN_ISV (bounded ≤ 5.0). pos_max_ema is bounded by
    // `reward_scale × raw_PnL`, both finite. So WIN ≤ MAX_MARGIN ×
    // MAX_REWARD_SCALE × MAX_RAW_PNL is structurally bounded.
    //
    // Atom span (Step 5) follows the new WIN via slow EWMA (α=0.001,
    // half-life ~700 steps), so resolution adjusts gradually as the
    // reward distribution evolves rather than whipsawing.
    if (ema_new > 0.0f) {
        const float min_win_isv = isv[RL_REWARD_CLAMP_MIN_WIN_INDEX];
        const float win_new     = fmaxf(min_win_isv, margin * ema_new);
        const float ratio       = isv[RL_REWARD_CLAMP_RATIO_INDEX];
        const float loss_new    = fmaxf(min_win_isv, win_new * ratio);
        isv[RL_REWARD_CLAMP_WIN_INDEX]  = win_new;
        isv[RL_REWARD_CLAMP_LOSS_INDEX] = loss_new;
    }

    // ── Step 5: C51 atom span adaptation from observed reward EMAs. ──
    //
    // G.2 disabled this because atom-span growth + clamp-bound growth
    // created a positive feedback loop. With clamp bounds FROZEN (Step
    // 4), the loop can't form — the clamp caps the reward magnitude
    // regardless of atom span.
    //
    // The span slowly tracks the observed reward tail EMAs so Q's
    // distributional resolution covers the ACTUAL reward range. Without
    // this, rewards exceeding the static span project to edge atoms and
    // Q loses magnitude discrimination (wr plateaus at ~0.46).
    //
    // Floor at V_BOUND_FLOOR (1.0) preserves baseline resolution.
    // Ceiling at clamp bounds (WIN=1.0, LOSS=3.0) prevents runaway.
    // Slow EWMA (α=0.001, half-life ~700 steps) lets Q's atom mapping
    // adapt gradually without whipsawing.
    // Atom span anchors on the CLAMP bounds — the post-clamp reward
    // range that Q's Bellman targets actually see. Pre-clamp EMAs
    // (pos_ema ≈ 50) would blow atoms to [-50, +50] with Δ_z=5,
    // destroying resolution. The clamp bounds (WIN, LOSS) are the
    // structural ceiling on what rewards enter Q.
    const float win_bound  = isv[RL_REWARD_CLAMP_WIN_INDEX];   // 1.0
    const float loss_bound = isv[RL_REWARD_CLAMP_LOSS_INDEX];  // 3.0
    {
        const float v_max_prev = isv[RL_C51_V_MAX_INDEX];
        const float v_max_target = fmaxf(v_bound_floor, win_bound);
        const float v_max_new = (v_max_prev == 0.0f)
            ? v_max_target
            : (1.0f - v_bound_ewma_alpha) * v_max_prev
              + v_bound_ewma_alpha * v_max_target;
        isv[RL_C51_V_MAX_INDEX] = fmaxf(v_bound_floor, v_max_new);
    }
    {
        const float v_min_prev = isv[RL_C51_V_MIN_INDEX];
        const float v_min_target = fminf(-v_bound_floor, -loss_bound);
        const float v_min_new = (v_min_prev == 0.0f)
            ? v_min_target
            : (1.0f - v_bound_ewma_alpha) * v_min_prev
              + v_bound_ewma_alpha * v_min_target;
        isv[RL_C51_V_MIN_INDEX] = fminf(-v_bound_floor, v_min_new);
    }
}
