diff --git a/crates/ml-alpha/cuda/rl_reward_clamp_controller.cu b/crates/ml-alpha/cuda/rl_reward_clamp_controller.cu index 1a8640e39..dd43cade7 100644 --- a/crates/ml-alpha/cuda/rl_reward_clamp_controller.cu +++ b/crates/ml-alpha/cuda/rl_reward_clamp_controller.cu @@ -224,47 +224,51 @@ extern "C" __global__ void rl_reward_clamp_controller( isv[RL_REWARD_CLAMP_RATIO_INDEX] = ratio_clamped; } - // Step 4: compute adaptive WIN/LOSS bounds: MARGIN × EMA, floored. - // No upper cap on WIN per the header note — the MARGIN ∈ [1, 5] - // × EMA composition is the structural bound. LOSS = RATIO × WIN - // with RATIO now adaptive. - const float win_eff = fmaxf(MIN_WIN, margin * ema_new); - const float ratio = isv[RL_REWARD_CLAMP_RATIO_INDEX]; - const float loss_eff = ratio * win_eff; - - isv[RL_REWARD_CLAMP_WIN_INDEX] = win_eff; - isv[RL_REWARD_CLAMP_LOSS_INDEX] = loss_eff; - - // ── C51 atom span — slow EWMA (was ratchet). ────────────────── + // Step 4 (G.2, 2026-05-24): WIN / LOSS bounds are STRUCTURAL — they + // stay at their trainer-seeded values (WIN=1.0, LOSS=3.0) matching + // the C51 atom span × 3:1 loss-aversion asymmetry. The previous + // adaptive widening was anti-correct: when clip_rate exceeded the + // 5% target, the controller WIDENED the clamp to accommodate the + // distribution, defeating the bound's structural purpose. The + // resulting positive feedback loop (large scaled rewards → wider + // clamp → larger V/Q targets → C51 atom support ratchets up → even + // larger scaled rewards permitted) was diagnosed in the F.5 200-step + // local smoke: WIN drifted 1.0 → 41.3 (41×) in 150 steps, scaled + // rewards of 14.71 flowed through, V regression spiked to l_v=104. // - // wwcsz followup: replace monotone-grow with slow EWMA so atom - // resolution focuses on the active reward range. Static ratchet - // wasted Δz on rare tails (V_MAX=20 with rewards mostly in - // [-5, +5] → Δz=2 wasted on extremes). Slow α=0.001 - // (half-life ~700 steps) gives Q's atom mapping time to be valid - // across encoder/head co-adaptation. Floors at [-1, +1] - // preserve the original C51 baseline as the worst case. - const float v_max_prev = isv[RL_C51_V_MAX_INDEX]; - const float v_min_prev = isv[RL_C51_V_MIN_INDEX]; - const float v_max_target = fmaxf(V_BOUND_FLOOR, win_eff); - const float v_min_target = fminf(-V_BOUND_FLOOR, -loss_eff); - // Bootstrap on sentinel 0 — first emit replaces directly (per - // pearl_first_observation_bootstrap). - float v_max_new, v_min_new; - if (v_max_prev == 0.0f) { - v_max_new = v_max_target; - } else { - v_max_new = (1.0f - V_BOUND_EWMA_ALPHA) * v_max_prev - + V_BOUND_EWMA_ALPHA * v_max_target; - v_max_new = fmaxf(V_BOUND_FLOOR, v_max_new); - } - if (v_min_prev == 0.0f) { - v_min_new = v_min_target; - } else { - v_min_new = (1.0f - V_BOUND_EWMA_ALPHA) * v_min_prev - + V_BOUND_EWMA_ALPHA * v_min_target; - v_min_new = fminf(-V_BOUND_FLOOR, v_min_new); - } - isv[RL_C51_V_MAX_INDEX] = v_max_new; - isv[RL_C51_V_MIN_INDEX] = v_min_new; + // The diagnostic-only state (pos_max_ema, neg_max_ema, RATIO, + // clip_rate_ema, MARGIN) is still maintained above so the diag can + // surface what the system *would* adapt to under an unbounded + // policy — useful for understanding reward-distribution drift. But + // the LOAD-BEARING WIN_INDEX / LOSS_INDEX slots are now write-once + // (trainer's isv_constants seed). Per + // pearl_audit_unboundedness_for_implicit_asymmetry: structural + // bounds must NOT adapt in response to the very signal they're + // meant to bound. + // + // Suppress unused-variable warnings on the diagnostic-only blocks. + (void) margin; + (void) ema_new; + + // ── C51 atom span — STRUCTURAL (G.2, 2026-05-24). ───────────── + // + // The atom-span EWMA was the SECOND half of the positive feedback + // loop disabled in Step 4: with WIN/LOSS adapting to fit large + // scaled rewards, V_MAX/V_MIN ratcheted up to match (e.g., F.5 + // smoke saw V_MAX=2.66 by step 149 from a seed of 1.0). Larger + // atom span then permitted V to predict larger values, amplifying + // advantage magnitude and PPO/V losses. + // + // Fix: the C51 atom span stays at its trainer-seeded value + // (V_MAX=1.0, V_MIN=-1.0). Q's distributional representation has + // FIXED structural resolution matching the seeded WIN/LOSS bounds + // — any reward signal outside that range is clipped at + // apply_reward_scale rather than absorbed by widening atoms. + // Per pearl_c51_atom_span_must_track_clamp_range: atom span tracks + // the SEED clamp range, not the runtime-adapted one. + // + // V_BOUND_FLOOR no longer used; keep the constant for any future + // re-introduction. + (void) V_BOUND_FLOOR; + (void) V_BOUND_EWMA_ALPHA; }