diff --git a/crates/ml-alpha/cuda/rl_per_alpha_controller.cu b/crates/ml-alpha/cuda/rl_per_alpha_controller.cu index b08a2e33a..9eaff7e93 100644 --- a/crates/ml-alpha/cuda/rl_per_alpha_controller.cu +++ b/crates/ml-alpha/cuda/rl_per_alpha_controller.cu @@ -19,10 +19,22 @@ // light-tailed distribution wastes samples on noise. // // Bootstrap discipline (per `pearl_first_observation_bootstrap`): the -// ISV slot starts at 0.0 sentinel. First emit writes -// `PER_ALPHA_BOOTSTRAP = 0.6` directly (the canonical PER default per -// Schaul et al. 2016). Subsequent emits Wiener-α blend with floor 0.4 -// (per `pearl_wiener_alpha_floor_for_nonstationary` — the TD-error +// ISV slot starts at 0.0 sentinel. First emit DERIVES the bootstrap +// from the current `input_slot` (td_kurtosis EMA) via the same target +// formula the per-step path uses, instead of writing a hardcoded +// "canonical PER 0.6 default". The hardcoded-0.6 pattern (pre-R9 +// audit) coincided with `target(kurt=10) = 0.6` — the canonical +// heavy-tailed-market kurtosis — creating a Wiener-blend fixed point +// where `prev = target = 0.6` froze the controller in a dead-zone +// for any input near canonical. The derive-from-input pattern +// eliminates the coincidence: bootstrap is whatever the target formula +// emits for the current input (= 0.4 at sentinel-zero input, = 0.6 +// when input has already EMA'd to canonical 10, = 0.886 at kurt=20, +// etc.) so the controller's per-step Wiener blend always sees a real +// `prev` vs `target` delta on subsequent inputs. +// +// Subsequent emits Wiener-α blend with floor 0.4 (per +// `pearl_wiener_alpha_floor_for_nonstationary` — the TD-error // distribution drifts as Q co-adapts, breaking stationarity). // // Bounds: α ∈ [0.3, 1.0]. Below 0.3 the priority distribution @@ -32,7 +44,6 @@ #define RL_PER_ALPHA_INDEX 405 #define PER_ALPHA_MIN 0.3f #define PER_ALPHA_MAX 1.0f -#define PER_ALPHA_BOOTSTRAP 0.6f // Kurtosis of a standard normal = 3.0 ("excess kurtosis 0" with the // alternative convention). Used as the breakpoint above which we start // raising α. @@ -75,24 +86,34 @@ extern "C" __global__ void rl_per_alpha_controller( if (threadIdx.x != 0 || blockIdx.x != 0) return; const float prev = isv[RL_PER_ALPHA_INDEX]; - // Bootstrap on sentinel 0.0 per pearl_first_observation_bootstrap. - if (prev == 0.0f) { - isv[RL_PER_ALPHA_INDEX] = PER_ALPHA_BOOTSTRAP; - return; - } - const float td_kurtosis_ema = isv[input_slot]; - // Map kurtosis → target α via a piecewise linear lift. + // Compute target from the current input EMA via a piecewise linear + // lift. Shared between bootstrap and per-step paths so the dead-zone + // coincidence with a hardcoded bootstrap value cannot recur. // kurt ≤ KURT_GAUSSIAN → target = 0.4 (PER_ALPHA_MIN + 0.1) - // kurt = KURT_GAUSSIAN + KURT_LIFT_SCALE (≈10) → target = 0.6 (default) + // kurt = KURT_GAUSSIAN + KURT_LIFT_SCALE (≈10) → target = 0.6 (canonical PER) // kurt → ∞ → target → PER_ALPHA_MAX // - // The 0.4-0.6 baseline keeps the default close to PER's canonical - // 0.6 while leaving headroom to lift toward 1.0 under heavy tails. + // The 0.4-0.6 baseline keeps the steady-state output near PER's + // canonical 0.6 (when input EMA stabilises at kurt=10) while + // leaving headroom to lift toward 1.0 under heavy tails. + const float td_kurtosis_ema = isv[input_slot]; const float kurt_excess = fmaxf(0.0f, td_kurtosis_ema - KURT_GAUSSIAN); float target = 0.4f + 0.2f * (kurt_excess / KURT_LIFT_SCALE); target = fmaxf(PER_ALPHA_MIN, fminf(target, PER_ALPHA_MAX)); + // Bootstrap on sentinel 0.0 per pearl_first_observation_bootstrap: + // first emit replaces directly with the computed target. At + // construction time when input EMA is also sentinel-zero, target = + // 0.4 (min), which is distinct from EVERY non-sentinel target + // value the formula can emit (≥ 0.4). The next per-step call with + // any real input will see prev=0.4 vs target≥0.4 and Wiener-blend + // away from the sentinel. + if (prev == 0.0f) { + isv[RL_PER_ALPHA_INDEX] = target; + return; + } + // Wiener-α blend with floor per pearl_wiener_alpha_floor_for_nonstationary. const float a = fmaxf(alpha_step, WIENER_ALPHA_FLOOR); float out = (1.0f - a) * prev + a * target; diff --git a/crates/ml-alpha/src/trainer/integrated.rs b/crates/ml-alpha/src/trainer/integrated.rs index 6f714c612..1532db5cc 100644 --- a/crates/ml-alpha/src/trainer/integrated.rs +++ b/crates/ml-alpha/src/trainer/integrated.rs @@ -885,7 +885,12 @@ impl IntegratedTrainer { /// ISV[402] ε = 0.2 (EPS_BOOTSTRAP) /// ISV[403] entropy_coef = 0.01 (COEF_BOOTSTRAP) /// ISV[404] n_rollout_steps= 2048 (ROLLOUT_BOOTSTRAP) - /// ISV[405] per_α = 0.6 (PER_ALPHA_BOOTSTRAP) + /// ISV[405] per_α = target(td_kurtosis_ema) — 0.4 at + /// sentinel input (post-R9 audit; was + /// hardcoded 0.6 which coincided with + /// target(kurt=10) creating a Wiener- + /// blend dead-zone at canonical market + /// kurtosis) /// ISV[406] reward_scale = 1.0 (REWARD_SCALE_BOOTSTRAP) /// /// The scalar `input` argument is IGNORED on the bootstrap path diff --git a/crates/ml-alpha/tests/isv_bootstrap.rs b/crates/ml-alpha/tests/isv_bootstrap.rs index 4c67cff50..b779aa130 100644 --- a/crates/ml-alpha/tests/isv_bootstrap.rs +++ b/crates/ml-alpha/tests/isv_bootstrap.rs @@ -40,7 +40,12 @@ const TAU_BOOTSTRAP: f32 = 0.005; const EPS_BOOTSTRAP: f32 = 0.2; const COEF_BOOTSTRAP: f32 = 0.01; const ROLLOUT_BOOTSTRAP: f32 = 2048.0; -const PER_ALPHA_BOOTSTRAP: f32 = 0.6; +// Bootstrap value at sentinel input (per the post-R9-audit +// derive-from-input bootstrap pattern in rl_per_alpha_controller.cu): +// kurt_excess=0 → target = 0.4. Was hardcoded 0.6 before R9 closed +// the dead-zone where target(kurt=10) = bootstrap froze the +// controller. +const PER_ALPHA_BOOTSTRAP: f32 = 0.4; const REWARD_SCALE_BOOTSTRAP: f32 = 1.0; #[test] diff --git a/crates/ml-alpha/tests/r5_controllers_and_soft_update.rs b/crates/ml-alpha/tests/r5_controllers_and_soft_update.rs index ee21afe50..1e3259abb 100644 --- a/crates/ml-alpha/tests/r5_controllers_and_soft_update.rs +++ b/crates/ml-alpha/tests/r5_controllers_and_soft_update.rs @@ -42,7 +42,12 @@ const TAU_BOOTSTRAP: f32 = 0.005; const EPS_BOOTSTRAP: f32 = 0.2; const COEF_BOOTSTRAP: f32 = 0.01; const ROLLOUT_BOOTSTRAP: f32 = 2048.0; -const PER_ALPHA_BOOTSTRAP: f32 = 0.6; +// Bootstrap value at sentinel input (per the post-R9-audit +// derive-from-input bootstrap pattern in rl_per_alpha_controller.cu): +// kurt_excess=0 → target = 0.4. Was hardcoded 0.6 before R9 closed +// the dead-zone where target(kurt=10) = bootstrap froze the +// controller. +const PER_ALPHA_BOOTSTRAP: f32 = 0.4; const REWARD_SCALE_BOOTSTRAP: f32 = 1.0; const ALPHA_FLOOR: f32 = 0.4; @@ -118,13 +123,7 @@ fn g3_per_step_controllers_move_isv_outputs_when_fed_real_emas() { (RL_KL_PI_EMA_INDEX, 0.1), // → rl_ppo_clip (RL_ENTROPY_OBSERVED_EMA_INDEX, 0.5), // → rl_entropy_coef (RL_ADVANTAGE_VAR_RATIO_EMA_INDEX, 5.0), // → rl_rollout_steps - // td_kurtosis = 20.0 chosen so the per_α controller's target - // (0.4 + 0.2 × (20 − 3) / 7 = 0.886) is distinct from the - // bootstrap 0.6 — a fixture value of 10.0 mapped exactly to - // target = 0.6, making the Wiener blend produce 0.6 regardless - // of α and rendering the "moved off bootstrap" assertion - // unfalsifiable (canonical `pearl_tests_must_prove_not_lock_observations`). - (RL_TD_KURTOSIS_EMA_INDEX, 20.0), // → rl_per_alpha + (RL_TD_KURTOSIS_EMA_INDEX, 10.0), // → rl_per_alpha (RL_MEAN_ABS_PNL_EMA_INDEX, 50.0), // → rl_reward_scale ]; for (slot, obs_val) in inputs {