diff --git a/crates/ml-alpha/cuda/rl_reward_clamp_controller.cu b/crates/ml-alpha/cuda/rl_reward_clamp_controller.cu index c4232cbeb..2a5cf0beb 100644 --- a/crates/ml-alpha/cuda/rl_reward_clamp_controller.cu +++ b/crates/ml-alpha/cuda/rl_reward_clamp_controller.cu @@ -26,10 +26,28 @@ // first non-zero raw observation replaces the EMA slot directly so the // blend always has a real prev value to mix with. // -// Bounds: WIN ∈ [MIN_WIN=1.0, MAX_WIN=20.0]. MIN_WIN preserves the -// floor from the static design — clamp never tightens below the C51 -// V_MAX. MAX_WIN caps runaway during early training before the EMA -// stabilises (atom support × 20 still well within float headroom). +// Bounds: WIN ∈ [MIN_WIN=1.0, +∞). MIN_WIN preserves the floor from +// the static design — clamp never tightens below the C51 V_MAX. The +// upper bound was REMOVED 2026-05-24 follow-up — a hardcoded MAX_WIN +// can't be reconciled with adaptive control: rdgzl hit MAX_WIN=20 in +// every window because the positive-tail distribution has p99=76.9 +// and max=2230; the cap turned the "adaptive" controller back into +// a static one. +// +// Safety reasoning for removing the cap: WIN = MARGIN × pos_max_ema. +// MARGIN is bounded [1.0, 5.0]. pos_max_ema is bounded by what +// apply_reward_scale produces, which is raw_reward × reward_scale, +// where reward_scale is itself bounded by its controller at [1e-3, 1e3]. +// Raw realized PnL is finite per-trade (bounded by position size × +// price-tick × ticks-moved). So WIN cannot diverge unboundedly in +// practice; the EMA + MARGIN cap is the structural bound. +// +// NOTE: C51's atom support in `bellman_target_projection.cu` is +// hardcoded `V_MAX=1.0`. Any Bellman target > 1.0 is categorically +// projected to atom 20 regardless of clamp. So lifting WIN beyond +// 1.0 helps V regression + PPO advantage (which see real magnitude) +// but NOT Q's distributional learning beyond +1.0. A follow-up that +// also lifts V_MAX/V_MIN would unlock the Q ceiling. // // LOSS = RATIO × WIN preserves the loss-aversion asymmetry from the // `pearl_audit_unboundedness_for_implicit_asymmetry` design. Driven @@ -41,17 +59,32 @@ // `feedback_isv_for_adaptive_bounds`: hardcoded MIN/MAX retained per // the user-stated "floors and clamp bounds" exemption (2026-05-24). -#define RL_POS_SCALED_REWARD_MAX_INDEX 478 -#define RL_POS_SCALED_REWARD_MAX_EMA_INDEX 479 -#define RL_REWARD_CLAMP_MARGIN_INDEX 480 -#define RL_REWARD_CLAMP_RATIO_INDEX 481 -#define RL_REWARD_CLAMP_WIN_INDEX 452 -#define RL_REWARD_CLAMP_LOSS_INDEX 453 +#define RL_POS_SCALED_REWARD_MAX_INDEX 478 +#define RL_POS_SCALED_REWARD_MAX_EMA_INDEX 479 +#define RL_REWARD_CLAMP_MARGIN_INDEX 480 +#define RL_REWARD_CLAMP_RATIO_INDEX 481 +#define RL_REWARD_CLAMP_WIN_INDEX 452 +#define RL_REWARD_CLAMP_LOSS_INDEX 453 +// MARGIN adaptation slots (audit 2026-05-24 follow-up). +#define RL_REWARD_CLAMP_CLIP_RATE_EMA_INDEX 482 +#define RL_REWARD_CLAMP_CLIP_RATE_TARGET_INDEX 483 #define MIN_WIN 1.0f -#define MAX_WIN 20.0f #define WIENER_ALPHA_FLOOR 0.4f +// MARGIN Schulman bounded-step bounds + adjust rate per +// `pearl_multiplicative_controllers_need_bounded_step_and_noise_floor`. +// TOLERANCE=1.5 → dead-zone is [target/1.5, target×1.5] = [0.033, 0.075]; +// ADJUST_RATE=1.2 → MARGIN moves by ≤20% per controller call (one per +// trainer step). Bounds [MIN_MARGIN=1.0, MAX_MARGIN=5.0] — MIN preserves +// the initial passthrough behaviour; MAX × (mean pos_max_ema≈3-5) → +// WIN_eff up to ~15-25 which is well within MAX_WIN=50 ceiling. +#define MIN_MARGIN 1.0f +#define MAX_MARGIN 5.0f +#define MARGIN_TOLERANCE 1.5f +#define MARGIN_ADJUST_RATE 1.2f +#define CLIP_RATE_EMA_ALPHA 0.05f + extern "C" __global__ void rl_reward_clamp_controller( float* __restrict__ isv, float alpha @@ -82,9 +115,56 @@ extern "C" __global__ void rl_reward_clamp_controller( } isv[RL_POS_SCALED_REWARD_MAX_EMA_INDEX] = ema_new; - // Compute adaptive WIN bound: MARGIN × EMA, clamped to [MIN, MAX]. - const float margin = isv[RL_REWARD_CLAMP_MARGIN_INDEX]; - const float win_eff = fmaxf(MIN_WIN, fminf(MAX_WIN, margin * ema_new)); + // ── MARGIN adaptation from clip-rate EMA (audit follow-up). ─── + // + // Step 1: compute clip indicator for THIS step — did the pos_max + // we just observed exceed the WIN that was active during the + // apply_reward_scale call? slot 452 still holds the WIN that was + // used (we'll overwrite it below with the new WIN). Only counts + // when there WAS a positive reward (pos_max > 0) — clip-rate is + // a fraction of positive-reward steps, not all steps. + const float win_active = isv[RL_REWARD_CLAMP_WIN_INDEX]; + float clip_indicator = 0.0f; + if (pos_max > 0.0f) { + clip_indicator = (pos_max > win_active) ? 1.0f : 0.0f; + } + + // Step 2: blend clip indicator into EMA. Fixed α=0.05 here (not + // Wiener-driven — the indicator is bimodal 0/1 so variance-based α + // would saturate; constant α gives ~20-step half-life which + // matches the policy-update cadence). + const float clip_rate_prev = isv[RL_REWARD_CLAMP_CLIP_RATE_EMA_INDEX]; + float clip_rate_new; + if (clip_rate_prev == 0.0f) { + // Bootstrap on sentinel — set to current indicator (avoids + // long warmup where MARGIN stays static). + clip_rate_new = clip_indicator; + } else { + clip_rate_new = (1.0f - CLIP_RATE_EMA_ALPHA) * clip_rate_prev + + CLIP_RATE_EMA_ALPHA * clip_indicator; + } + isv[RL_REWARD_CLAMP_CLIP_RATE_EMA_INDEX] = clip_rate_new; + + // Step 3: Schulman bounded step on MARGIN — if clip rate > target + // × TOLERANCE, raise MARGIN (widen WIN, capture more tail); if + // clip rate < target / TOLERANCE, lower MARGIN (tighten WIN, + // sharper regression). Dead-zone in between prevents oscillation. + const float clip_target = isv[RL_REWARD_CLAMP_CLIP_RATE_TARGET_INDEX]; + float margin = isv[RL_REWARD_CLAMP_MARGIN_INDEX]; + const float upper = clip_target * MARGIN_TOLERANCE; + const float lower = clip_target / MARGIN_TOLERANCE; + if (clip_rate_new > upper) { + margin = fminf(MAX_MARGIN, margin * MARGIN_ADJUST_RATE); + } else if (clip_rate_new < lower) { + margin = fmaxf(MIN_MARGIN, margin / MARGIN_ADJUST_RATE); + } + isv[RL_REWARD_CLAMP_MARGIN_INDEX] = margin; + + // Step 4: compute adaptive WIN bound: MARGIN × EMA, floored. + // No upper cap per the header note — the MARGIN ∈ [1, 5] × EMA + // composition is the structural bound; downstream C51 projection + // saturates beyond V_MAX without needing this kernel to enforce. + const float win_eff = fmaxf(MIN_WIN, margin * ema_new); // LOSS = RATIO × WIN preserves asymmetry. RATIO seeded to 3.0. const float ratio = isv[RL_REWARD_CLAMP_RATIO_INDEX]; diff --git a/crates/ml-alpha/examples/alpha_rl_train.rs b/crates/ml-alpha/examples/alpha_rl_train.rs index 9b2744dc0..2f427e752 100644 --- a/crates/ml-alpha/examples/alpha_rl_train.rs +++ b/crates/ml-alpha/examples/alpha_rl_train.rs @@ -66,6 +66,7 @@ use ml_alpha::rl::isv_slots::{ RL_MAX_ABS_SCALED_REWARD_PRE_CLAMP_INDEX, RL_MEAN_ABS_PNL_EMA_INDEX, RL_POS_SCALED_REWARD_MAX_INDEX, RL_POS_SCALED_REWARD_MAX_EMA_INDEX, RL_REWARD_CLAMP_MARGIN_INDEX, RL_REWARD_CLAMP_RATIO_INDEX, + RL_REWARD_CLAMP_CLIP_RATE_EMA_INDEX, RL_REWARD_CLAMP_CLIP_RATE_TARGET_INDEX, RL_MEAN_TRADE_DURATION_EMA_INDEX, RL_N_ROLLOUT_STEPS_INDEX, RL_PER_ALPHA_INDEX, RL_PI_GRAD_NORM_EMA_INDEX, RL_PPO_CLIP_INDEX, RL_PPO_LOG_RATIO_ABS_MAX_INDEX, RL_PPO_RATIO_CLAMP_MAX_INDEX, RL_Q_DIVERGENCE_EMA_INDEX, RL_Q_GRAD_NORM_EMA_INDEX, @@ -670,6 +671,11 @@ fn main() -> Result<()> { isv[RL_POS_SCALED_REWARD_MAX_INDEX], "pos_scaled_max_ema": isv[RL_POS_SCALED_REWARD_MAX_EMA_INDEX], + // MARGIN controller state — clip rate vs target tells + // us whether the controller is happy (rate ≈ target) + // or fighting the tail (rate > target × tolerance). + "clip_rate_ema": + isv[RL_REWARD_CLAMP_CLIP_RATE_EMA_INDEX], }, // audit — PPO importance-ratio clamp diagnostics. // @@ -780,6 +786,7 @@ fn main() -> Result<()> { "ppo_ratio_clamp_bootstrap": isv[RL_PPO_RATIO_CLAMP_BOOTSTRAP_INDEX], "reward_clamp_margin": isv[RL_REWARD_CLAMP_MARGIN_INDEX], "reward_clamp_ratio": isv[RL_REWARD_CLAMP_RATIO_INDEX], + "reward_clamp_clip_rate_target": isv[RL_REWARD_CLAMP_CLIP_RATE_TARGET_INDEX], }, // audit — Q vs π action agreement EMA at slot 407 // (previously dead). 1.0 = perfect ranking consistency, diff --git a/crates/ml-alpha/src/rl/isv_slots.rs b/crates/ml-alpha/src/rl/isv_slots.rs index 43f9237aa..f02c8d769 100644 --- a/crates/ml-alpha/src/rl/isv_slots.rs +++ b/crates/ml-alpha/src/rl/isv_slots.rs @@ -39,8 +39,9 @@ //! | 462-467 | 6 LR-controller numerics + aux λ | rl_isv_write (seed loop) | //! | 468-477 | 10 design constants (Schulman + bootstraps + streaming α + kurt refs) | rl_isv_write | //! | 478-481 | 4 adaptive reward-clamp slots (raw + EMA + margin + ratio) | apply_reward_scale + rl_reward_clamp_controller | +//! | 482-483 | 2 MARGIN-adaptation slots (clip-rate EMA + target) | rl_reward_clamp_controller (Schulman step on MARGIN) | //! -//! Total: 82 slots; `RL_SLOTS_END = 482`. +//! Total: 84 slots; `RL_SLOTS_END = 484`. /// Discount factor γ. Controller input: mean trade duration / anchor. /// Bootstrap 0.99. @@ -595,6 +596,29 @@ pub const RL_REWARD_CLAMP_MARGIN_INDEX: usize = 480; /// `rl_isv_write`. pub const RL_REWARD_CLAMP_RATIO_INDEX: usize = 481; +// ─── MARGIN adaptation (audit 2026-05-24, chain follow-up) ──── +// +// alpha-rl-rdgzl evidence: static MARGIN=1.5 left 25-40% of wins +// clipped because the positive-tail distribution is brutally +// heavy: p50=1.85, p90=10.1, p99=76.9, max=2230. EMA-driven WIN +// captures the median but misses the tail. MARGIN must adapt to +// keep clip rate near a target (~5%) so the controller chases +// the tail rather than the median. + +/// EMA of per-step indicator `(pos_max > current_WIN) ? 1 : 0`. The +/// MARGIN controller uses this to decide whether to expand +/// (clip-rate > target × TOLERANCE → raise) or contract (clip-rate +/// < target / TOLERANCE → lower) the MARGIN multiplier. +pub const RL_REWARD_CLAMP_CLIP_RATE_EMA_INDEX: usize = 482; + +/// Target clip rate for the MARGIN controller. Default 0.05 — allow +/// 5% of positive-reward steps to exceed the WIN bound (tail +/// outliers ride the clamp; mode-and-shoulder of the distribution +/// passes through). Seeded by `rl_isv_write`. Tuning lower (e.g. +/// 0.02) forces wider WIN bounds (captures more tail); higher +/// (e.g. 0.10) accepts more clipping for tighter regression scale. +pub const RL_REWARD_CLAMP_CLIP_RATE_TARGET_INDEX: usize = 483; + /// Last RL-allocated slot index (exclusive). The integrated trainer /// extends `ISV_TOTAL_DIM` to at least this value at trainer init time. -pub const RL_SLOTS_END: usize = 482; +pub const RL_SLOTS_END: usize = 484; diff --git a/crates/ml-alpha/src/trainer/integrated.rs b/crates/ml-alpha/src/trainer/integrated.rs index 871892df9..4880ff8eb 100644 --- a/crates/ml-alpha/src/trainer/integrated.rs +++ b/crates/ml-alpha/src/trainer/integrated.rs @@ -1199,17 +1199,22 @@ impl IntegratedTrainer { // (slot, value) pair — pure device write, no HtoD per // `feedback_no_htod_htoh_only_mapped_pinned`. { - let isv_constants: [(usize, f32); 28] = [ + let isv_constants: [(usize, f32); 29] = [ // Static seeds for the adaptive reward-clamp controller — // these are the initial values that // `rl_reward_clamp_controller` will replace once it // observes the first positive reward. Until then, kernels // reading slots 452/453 see the documented [-3, +1] - // defaults. + // defaults. MARGIN (480) starts at 1.5 then adapts via + // the controller's Schulman bounded-step from clip-rate + // EMA (482) vs target (483). CLIP_RATE_TARGET seeded to + // 0.05 = "let 5% of wins ride the clamp as tail + // outliers." (crate::rl::isv_slots::RL_REWARD_CLAMP_WIN_INDEX, 1.0), (crate::rl::isv_slots::RL_REWARD_CLAMP_LOSS_INDEX, 3.0), (crate::rl::isv_slots::RL_REWARD_CLAMP_MARGIN_INDEX, 1.5), (crate::rl::isv_slots::RL_REWARD_CLAMP_RATIO_INDEX, 3.0), + (crate::rl::isv_slots::RL_REWARD_CLAMP_CLIP_RATE_TARGET_INDEX, 0.05), (crate::rl::isv_slots::RL_KL_TARGET_INDEX, 0.01), (crate::rl::isv_slots::RL_IMPROVEMENT_THRESHOLD_INDEX, 0.99), (crate::rl::isv_slots::RL_PLATEAU_PATIENCE_INDEX, 1000.0),