diff --git a/crates/ml-alpha/cuda/rl_reward_clamp_controller.cu b/crates/ml-alpha/cuda/rl_reward_clamp_controller.cu index 97fec4492..f66a17a65 100644 --- a/crates/ml-alpha/cuda/rl_reward_clamp_controller.cu +++ b/crates/ml-alpha/cuda/rl_reward_clamp_controller.cu @@ -65,6 +65,10 @@ #define RL_REWARD_CLAMP_RATIO_INDEX 481 #define RL_REWARD_CLAMP_WIN_INDEX 452 #define RL_REWARD_CLAMP_LOSS_INDEX 453 +// Fix F (spec 2026-05-30-c51-atom-span-decouple-from-clamp): typical-magnitude +// signal + multiplier driving atom span (decoupled from extreme-value WIN). +#define RL_MEAN_ABS_PNL_EMA_INDEX 477 +#define RL_C51_ATOM_MARGIN_INDEX 685 // MARGIN adaptation slots (audit 2026-05-24 follow-up). #define RL_REWARD_CLAMP_CLIP_RATE_EMA_INDEX 482 #define RL_REWARD_CLAMP_CLIP_RATE_TARGET_INDEX 483 @@ -265,32 +269,41 @@ extern "C" __global__ void rl_reward_clamp_controller( isv[RL_REWARD_CLAMP_LOSS_INDEX] = loss_new; } - // ── Step 5: C51 atom span adaptation from observed reward EMAs. ── + // ── Step 5: C51 atom span adaptation, decoupled from clamp ceiling. ── // - // G.2 disabled this because atom-span growth + clamp-bound growth - // created a positive feedback loop. With clamp bounds FROZEN (Step - // 4), the loop can't form — the clamp caps the reward magnitude - // regardless of atom span. + // Fix F (spec 2026-05-30-c51-atom-span-decouple-from-clamp): anchor + // the atom span on `mean_abs_pnl_ema × atom_margin` — the TYPICAL + // reward magnitude — instead of `win_bound` (= MARGIN × pos_max_ema, + // an extreme-value statistic that overshoots at b=1024). // - // The span slowly tracks the observed reward tail EMAs so Q's - // distributional resolution covers the ACTUAL reward range. Without - // this, rewards exceeding the static span project to edge atoms and - // Q loses magnitude discrimination (wr plateaus at ~0.46). + // Pre-Fix-F (anchored on clamp bounds): in alpha-rl-rjsjq v5 at + // step 371, WIN bounced 4910 → 568 → 44 (volatile from MARGIN + // saturation) while v_max ratcheted up 24 → 1938 because the slow + // EWMA (α=0.001) time-averaged the early WIN spikes. Δz reached + // ~184 per atom — Q couldn't discriminate +$500 from +$5000 wins. // - // Floor at V_BOUND_FLOOR (1.0) preserves baseline resolution. - // Ceiling at clamp bounds (WIN=1.0, LOSS=3.0) prevents runaway. + // Post-Fix-F: V regression (scalar head, not categorical) retains + // fat-tail magnitude beyond the atom ceiling, so PPO advantage + // signal is preserved. Only Q's *categorical* learning loses + // fat-tail discrimination — an accepted trade-off for the + // lottery-ticket strategy that benefits more from fine resolution + // at typical magnitudes. + // + // Floor at V_BOUND_FLOOR (1.0) preserves the baseline atom span + // during the warmup window where mean_abs_pnl_ema bootstraps from 0. // Slow EWMA (α=0.001, half-life ~700 steps) lets Q's atom mapping - // adapt gradually without whipsawing. - // Atom span anchors on the CLAMP bounds — the post-clamp reward - // range that Q's Bellman targets actually see. Pre-clamp EMAs - // (pos_ema ≈ 50) would blow atoms to [-50, +50] with Δ_z=5, - // destroying resolution. The clamp bounds (WIN, LOSS) are the - // structural ceiling on what rewards enter Q. - const float win_bound = isv[RL_REWARD_CLAMP_WIN_INDEX]; // 1.0 - const float loss_bound = isv[RL_REWARD_CLAMP_LOSS_INDEX]; // 3.0 + // adapt gradually without whipsawing across rare extreme events. + // + // LOSS-side: v_min mirrors v_max scaled by the existing RATIO + // controller — same typical-magnitude anchor, same Wiener-α + // hysteresis. Preserves loss-aversion asymmetry. + const float mean_abs_pnl = isv[RL_MEAN_ABS_PNL_EMA_INDEX]; + const float atom_margin = isv[RL_C51_ATOM_MARGIN_INDEX]; + const float ratio = isv[RL_REWARD_CLAMP_RATIO_INDEX]; + const float typical_magnitude = atom_margin * mean_abs_pnl; { const float v_max_prev = isv[RL_C51_V_MAX_INDEX]; - const float v_max_target = fmaxf(v_bound_floor, win_bound); + const float v_max_target = fmaxf(v_bound_floor, typical_magnitude); const float v_max_new = (v_max_prev == 0.0f) ? v_max_target : (1.0f - v_bound_ewma_alpha) * v_max_prev @@ -299,7 +312,7 @@ extern "C" __global__ void rl_reward_clamp_controller( } { const float v_min_prev = isv[RL_C51_V_MIN_INDEX]; - const float v_min_target = fminf(-v_bound_floor, -loss_bound); + const float v_min_target = fminf(-v_bound_floor, -typical_magnitude * ratio); const float v_min_new = (v_min_prev == 0.0f) ? v_min_target : (1.0f - v_bound_ewma_alpha) * v_min_prev diff --git a/crates/ml-alpha/src/rl/isv_slots.rs b/crates/ml-alpha/src/rl/isv_slots.rs index b0c0c950a..28c32322a 100644 --- a/crates/ml-alpha/src/rl/isv_slots.rs +++ b/crates/ml-alpha/src/rl/isv_slots.rs @@ -1460,6 +1460,15 @@ pub const RL_TRAIL_LOOSEN_FACTOR_INDEX: usize = 683; /// the [[feedback_no_partial_refactor]] per-batch fix. pub const RL_SESSION_PNL_WORST_INDEX: usize = 684; +/// Fix F (spec 2026-05-30-c51-atom-span-decouple-from-clamp): atom-span +/// margin multiplier on `mean_abs_pnl_ema`. C51 atom span = `floor(1.0) +/// + margin × mean_abs_pnl_ema` so Q's categorical resolution stays sized +/// for TYPICAL trade magnitudes rather than the fat-tail clamp ceiling. +/// V regression (scalar head) retains fat-tail magnitude information. +/// Bootstrap 10.0 → ~5× the EMA gives generous coverage at typical scale +/// without re-coupling to extreme-value pos_max statistics. +pub const RL_C51_ATOM_MARGIN_INDEX: usize = 685; + /// Last RL-allocated slot index (exclusive). Pre-risk-stack: 662. -/// Post-risk-stack (Fix B IQN-τ split): 685. -pub const RL_SLOTS_END: usize = 685; +/// Post-Fix-F: 686. +pub const RL_SLOTS_END: usize = 686; diff --git a/crates/ml-alpha/src/trainer/integrated.rs b/crates/ml-alpha/src/trainer/integrated.rs index 38a079d06..e2256932f 100644 --- a/crates/ml-alpha/src/trainer/integrated.rs +++ b/crates/ml-alpha/src/trainer/integrated.rs @@ -3091,7 +3091,7 @@ impl IntegratedTrainer { // (slot, value) pair — pure device write, no HtoD per // `feedback_no_htod_htoh_only_mapped_pinned`. { - let isv_constants: [(usize, f32); 168] = [ + let isv_constants: [(usize, f32); 169] = [ // Static seeds for the adaptive reward-clamp controller — // these are the initial values that // `rl_reward_clamp_controller` will replace once it @@ -3312,6 +3312,10 @@ impl IntegratedTrainer { // CLIP_RATE_EMA_ALPHA=0.05. (crate::rl::isv_slots::RL_REWARD_CLAMP_V_BOUND_FLOOR_INDEX, 1.0), (crate::rl::isv_slots::RL_REWARD_CLAMP_V_BOUND_EWMA_ALPHA_INDEX, 0.001), + // Fix F: atom margin multiplier for the decoupled atom-span + // controller. atom_span = max(floor=1.0, margin × mean_abs_pnl_ema). + // 10.0 = ~5σ coverage on typical reward distribution. + (crate::rl::isv_slots::RL_C51_ATOM_MARGIN_INDEX, 10.0), (crate::rl::isv_slots::RL_REWARD_CLAMP_MIN_WIN_INDEX, 1.0), (crate::rl::isv_slots::RL_REWARD_CLAMP_MIN_RATIO_INDEX, 1.0), (crate::rl::isv_slots::RL_REWARD_CLAMP_MAX_RATIO_INDEX, 3.0), diff --git a/docs/superpowers/specs/2026-05-30-c51-atom-span-decouple-from-clamp.md b/docs/superpowers/specs/2026-05-30-c51-atom-span-decouple-from-clamp.md new file mode 100644 index 000000000..036fd3766 --- /dev/null +++ b/docs/superpowers/specs/2026-05-30-c51-atom-span-decouple-from-clamp.md @@ -0,0 +1,201 @@ +# C51 Atom Span Decoupling from Reward Clamp Ceiling — Design + +**Goal**: Decouple the C51 distributional-Q atom span from the reward-clamp ceiling so the atom grid stays sized for *typical* trade magnitudes while the clamp continues to admit *fat-tail* wins/losses without truncation. This preserves Q's distributional resolution at the magnitudes that matter for learning, restoring fine-grained value differentiation between small wins/losses while keeping the wide-clamp benefit shipped in `7064c9269` (Fix C). + +**Why now**: cluster `alpha-rl-rjsjq` (v5, SHA `7064c9269` = Fix A+B+C+D) at step 371 showed `c51_v_max` ratcheting up monotonically — 1 → 24 → 53 → 859 → 990 → 1544 → **1938** — while the `WIN` clamp swung wildly (4910 → 568 → 7540 → 44). The Δz atom resolution settled at ~184 per atom by step 371, ~600 by long-running steady state. Q can no longer discriminate $500-magnitude actions from $5000-magnitude actions within the same atom. + +**Out of scope**: changes to V regression (uses popart, already handles wide range), changes to the reward-scale controller, changes to PPO clip / IQN τ / Kelly. Fix E (force-close on DD trip) is shipped separately. The fat-tail single-step DD-trigger lag (positions losing $11k+ in one tick) needs an intra-step circuit breaker — that's a separate spec. + +--- + +## Diagnosis — what the JSONL says + +From v5 cluster `/feature-cache/alpha-rl-runs/7064c9269/fold1/diag.jsonl`: + +| step | qpa | σ | clip_rate | WIN | v_max | reward_scale | mean_active | +|---|---|---|---|---|---|---|---| +| 0 | +0.16 | 1.00 | 0.000 | 1 | 1 | 1.000 | $0 | +| 5 | −0.09 | 10.85 | 0.698 | **4910** | 24 | 0.896 | +$177 | +| 10 | −0.12 | 12.58 | 0.513 | **2921** | 53 | 0.810 | +$162 | +| 50 | +0.69 | 56.92 | 0.035 | **7540** | 859 | 0.361 | +$2300 | +| 100 | +0.59 | 56.55 | 0.093 | 568 | 990 | 0.132 | +$2604 | +| 200 | +0.59 | 57.16 | 0.031 | 2315 | 1544 | 0.069 | +$6307 | +| 371 | +0.87 | 59.33 | 0.041 | 44 | **1938** | 0.004 | +$20122 | + +**Observations:** +1. WIN clamp width = MARGIN × pos_max_ema, with MARGIN saturated at MAX=5.0 once `clip_rate > 0.05`. Updates fast (α=0.4 = Wiener floor for pos_max_ema). +2. C51 atom v_max follows WIN via slow EWMA (`v_bound_ewma_alpha = 0.001`, half-life ~700 steps). Asymmetric in practice: WIN can drop fast (from 7540 to 44 over ~300 steps), but v_max only crawls down 0.001× per step. +3. v_max **time-averages** WIN. Because pos_max_ema is the MAX of `b_size = 1024` samples (extreme value statistic), the EMA always sits far above the median or mean reward magnitude. WIN's lower bound is roughly `MAX_MARGIN × pos_max_p99`, not `MAX_MARGIN × pos_max_typical`. + +**Net effect**: atom span sized for the rare $42k fat-tail wins, not the $7k typical wins. Δz at v_max=1938 is `2×1938/21 ≈ 184 per atom`. A +$500 reward and a +$5000 reward both map to within 1 atom of zero post-scale. + +**Why this matters for learning**: +- Bellman target projection: V_target lands at coarse atom — loses information about exact magnitude. +- Q-π distillation: `softmax(E_Q/τ)` — when all small-magnitude actions land in the same atom, distillation collapses to near-uniform. +- PPO advantage: `A = Q(s,a) - V(s)` — wide-atom Q is noisy on typical-magnitude actions, so PPO update is noisy. + +The agent still trains (qpa=+0.87 at step 371) but is *leaving learning signal on the table*. + +--- + +## Proposed design — anchor atom span on the typical-magnitude EMA + +Replace the v_max EWMA target from `win_bound` (= MARGIN × pos_max_ema) to a slower, more central magnitude estimate. Two candidates: + +### Option A: `RL_MEAN_ABS_PNL_EMA_INDEX` × atom_margin + +`RL_MEAN_ABS_PNL_EMA_INDEX` (slot 477) tracks the EMA of `|reward|` across recent batches. It's a true central-tendency signal — not the extreme-value MAX. Already populated by the reward pipeline. + +``` +v_max_target = max(v_bound_floor, atom_margin × mean_abs_pnl_ema) +``` + +With `atom_margin = 10` and `mean_abs_pnl_ema ≈ 1.5` (post-scale typical), `v_max_target ≈ 15`. Δz = 30/21 ≈ 1.4 per atom — fine resolution for typical trades. Fat-tail wins of 100+ scaled magnitude still get the WIN clamp (244) → projected to top atom; **information about magnitude beyond +15 is lost in Q but preserved in V regression** (V is a scalar head, not categorical). + +### Option B: `RL_POS_SCALED_REWARD_MAX_EMA_INDEX` × atom_margin (smaller margin) + +Use the same pos_max_ema input as WIN, but with `atom_margin = 1.5` instead of MARGIN = 5.0. Net effect: atom span = (1.5/5.0) × WIN = 30% of clamp width. + +``` +v_max_target = max(v_bound_floor, atom_margin × pos_max_ema) +``` + +Simpler — reuses existing EMA, just different multiplier. Atom span scales with same dynamics as clamp but tighter. + +### Recommendation: **Option A** + +Mean-absolute-PnL is fundamentally a better signal for atom sizing because it represents the magnitude where most rewards land, not the extreme-value tail. Δz adapts to the actual distribution. + +**Caveat for Option A**: `mean_abs_pnl_ema` is scaled by `reward_scale`. If reward_scale collapses (as it does — 0.004 at step 371), the EMA shrinks too. That's actually correct: smaller scaled rewards → tighter atom span needed for resolution. The reward_scale controller and atom span thus become coupled in a healthy way (both track the post-scale reward magnitude). + +**Atom span LOSS (v_min) follows analogously**: `v_min_target = min(-v_bound_floor, -atom_margin × neg_max_ema)` OR `-RATIO × v_max_target` for symmetric scaling with the loss-aversion ratio. + +--- + +## Math walkthrough — expected behavior at steady state + +Take v5 cluster step 371 actual numbers: +- `mean_abs_pnl_ema` ≈ 1.5 (post-scale, estimated — slot would need to be added to diag) +- `pos_max_ema` ≈ 9 (since WIN = 44 = 5 × 9) +- WIN clamp = 44 (admits ~95% of wins, 5% clip rate) + +**Current (v_max = win_bound)**: +- v_max_target = 44 +- v_max trails toward 44 at α=0.001 → reaches 44 only after ~5000 steps +- But because WIN was historically much higher (7540 at step 50), the trail has been "remembering" big WIN values → v_max stuck at 1938 + +**Option A (v_max = 10 × mean_abs_pnl_ema)**: +- v_max_target = 10 × 1.5 = 15 +- Δz = 30/21 = 1.43 per atom +- A +$3000 scaled reward (typical big win pre-scale × 0.004 scale) = +12 → atom 19 (mid-upper) +- A +$15k scaled reward = +60 → clamped to WIN=44, projected to atom 20 +- A +$30 scaled reward (typical small win) = +0.12 → atom 11 (just above zero) — **finely resolved** + +Resolution at typical-magnitude is **125× better** than current. + +**Risk**: when pos_max spikes, V_target can land outside atom span and Q learns "this is just top-atom" — loses fat-tail magnitude information. Mitigation: V regression head (not categorical) retains the magnitude. PPO uses V for advantage. Only Q's distributional learning loses fat-tail discrimination, which is acceptable for a lottery-ticket strategy that cares more about "is this action a winner" than "exactly how much does it win". + +--- + +## Implementation plan + +**File**: `crates/ml-alpha/cuda/rl_reward_clamp_controller.cu` Step 5 (lines 244-284). + +**Change**: + +```c +// BEFORE: +const float win_bound = isv[RL_REWARD_CLAMP_WIN_INDEX]; +const float loss_bound = isv[RL_REWARD_CLAMP_LOSS_INDEX]; +{ + const float v_max_prev = isv[RL_C51_V_MAX_INDEX]; + const float v_max_target = fmaxf(v_bound_floor, win_bound); + // ... EWMA blend +} + +// AFTER: +const float mean_abs_pnl_ema = isv[RL_MEAN_ABS_PNL_EMA_INDEX]; +const float atom_margin = isv[RL_C51_ATOM_MARGIN_INDEX]; // NEW slot, bootstrap 10.0 +{ + const float v_max_prev = isv[RL_C51_V_MAX_INDEX]; + // Atom span tracks TYPICAL reward magnitude × atom_margin. Decoupled from + // the reward-clamp ceiling so Q's categorical resolution stays fine while + // the clamp can grow to admit fat-tail wins. V regression (scalar head, + // not categorical) retains the magnitude information beyond the atom + // span for PPO advantage. + const float v_max_target = fmaxf(v_bound_floor, atom_margin * mean_abs_pnl_ema); + // ... EWMA blend with same v_bound_ewma_alpha +} +``` + +**New ISV slot needed**: `RL_C51_ATOM_MARGIN_INDEX = 685`, bootstrap `10.0`. Bump `RL_SLOTS_END` from 685 → 686. + +**Diag emit**: add `atom_margin` to the `risk_stack.cmdp` section (or a new `risk_stack.atoms` section). + +**Update G5 test** in `risk_stack_invariants.rs`: a new invariant test that confirms atom span tracks `mean_abs_pnl_ema × margin`, not `win_bound`. + +--- + +## Validation criteria + +**Local 1k smoke (b=128)**: +- Δz at step 999 < 5 (vs current 184) — fine resolution preserved +- WIN clamp behavior unchanged from v6 (still adaptive) +- qpa ≥ +0.5 at step 999 — Q-π alignment maintained +- mean_active > 0 + +**Cluster 20k+5k (fold-1 walk-forward)**: +- Eval-phase PnL ≥ −$100k (vs OLD baseline −$616k) — primary success criterion +- Q gradient norm ≤ 100 throughout (currently 1860 at gwmkf step 4083 with frozen WIN) — wide atom span was causing this +- Q-π distillation KL stable (controller doesn't saturate) + +**Compute-sanitizer 5-step smoke**: ERROR SUMMARY: 0 errors + +--- + +## Risks and alternatives + +**Risk 1: Q loses fat-tail magnitude awareness.** +Mitigation: V regression head (scalar) keeps full magnitude. PPO advantage uses V, so PG signal preserved. Q only loses *categorical* resolution of magnitudes beyond atom span — its argmax/expected ranking remains correct. + +**Risk 2: `mean_abs_pnl_ema` could be too small early on, atom span collapses to floor.** +Mitigation: `v_bound_floor = 1.0` prevents collapse below baseline atom span. Atom span only widens beyond floor when EMA stabilizes — slow EWMA `α=0.001` ensures gentle ramp-up. + +**Risk 3: Coupling with reward_scale controller.** +Both atom span and reward_scale react to reward magnitudes. Need to verify no feedback loop. Per `[[pearl_no_feedback_loop_atom_span_clamp]]`, the original G.2 freeze was over-conservative; this coupling is *desirable* — both controllers track the same underlying signal (post-scale typical reward magnitude) and stabilize together. + +**Alternative considered: Option B (pos_max_ema × smaller margin)**. +Simpler but uses the extreme-value statistic. With b=1024, pos_max is heavily right-skewed, so even `1.5 × pos_max_ema` overshoots typical reward magnitude. Mean-abs-PnL is the right statistic. + +**Alternative considered: Non-uniform atoms (concentrated near zero, spread for tails)**. +Architectural change to C51's atom grid. Major work — would touch `bellman_target_projection.cu`, `dueling_q_head.cu`, etc. Deferred. Option A delivers most of the benefit with surgical kernel edit. + +**Alternative considered: Two-rate EWMA (fast widen, slow narrow)**. +Hysteresis to prevent atom span churn. But the underlying issue is the *target* signal (pos_max_ema biases high), not the EWMA mechanics. Fixing the target signal fixes both directions naturally. + +--- + +## Migration path + +1. **Add ISV slot** `RL_C51_ATOM_MARGIN_INDEX = 685`, bootstrap `10.0`. +2. **Update kernel** as described above (5-line edit to Step 5 of `rl_reward_clamp_controller.cu`). +3. **Add diag emit** for `atom_margin` and `c51_v_max` continuing to use existing slot 484. +4. **Add G14 invariant test** confirming atom span tracks `mean_abs_pnl_ema × margin`. +5. **Local smoke** + cluster validation (next fold-1 walk-forward cycle). +6. **If positive eval**, ship as Fix F. If negative, revert and investigate Q gradient explosion separately. + +--- + +## Related issues — future spec candidates + +### Fat-tail single-step DD (out of scope here) + +Positions can lose more than `dd_limit` in a single futures-market tick before the DD controller can react. Fix E force-closes on the FIRST cooldown step but the *triggering* loss is already realized. Real fix needs an intra-step circuit breaker — e.g., apply a hard stop at `−2 × dd_limit` directly in the reward pipeline before the CMDP check. + +Estimated cost: kernel edit to `rl_fused_reward_pipeline.cu`, ~20 LOC. Defer until v6 cluster eval shows whether residual fat-tail loss is materially affecting PnL. + +### MARGIN ratchet hysteresis (low priority) + +MARGIN slams to MAX=5.0 within ~12 steps from cold start (`adjust_rate^N = MAX/MIN` → N = log(5/1)/log(1.1) ≈ 17). This causes WIN to overshoot before settling. Could add a warmup period during which MARGIN is capped lower. + +Estimated cost: 3-line edit. Defer; current behavior absorbs into atom span EWMA and converges by step ~500.