diff --git a/crates/ml-alpha/cuda/rl_entropy_coef_controller.cu b/crates/ml-alpha/cuda/rl_entropy_coef_controller.cu index 6114df091..8b1da079f 100644 --- a/crates/ml-alpha/cuda/rl_entropy_coef_controller.cu +++ b/crates/ml-alpha/cuda/rl_entropy_coef_controller.cu @@ -20,9 +20,24 @@ // policy can sharpen on real signal as training progresses. // // Bootstrap discipline (per `pearl_first_observation_bootstrap`): ISV -// slot starts at 0.0 sentinel. First emit writes coef = 0.01 (the -// standard PPO default). Subsequent emits Wiener-α blend with floor 0.4 -// (per `pearl_wiener_alpha_floor_for_nonstationary` — the policy +// slot starts at 0.0 sentinel. First emit DERIVES coef from the +// current input slot via the same target formula the per-step path +// uses, instead of writing a hardcoded `COEF_BOOTSTRAP = 0.01`. At +// sentinel input (entropy_observed_ema = 0) deficit = h_target = +// 1.538, target = (1.538 / 2.197) × 0.05 = 0.035 — the cold-start +// coef is high (push exploration hard when no entropy data is in yet). +// This eliminates the dead-zone where target(h_obs ≈ 1.099) = 0.01 = +// hardcoded bootstrap froze the Wiener blend at any realistic +// mid-range entropy observation. +// +// Trade-off: cold-start coef = 0.035 (3.5× the previous canonical +// 0.01) lifts the entropy bonus's relative weight in early training. +// Acceptable cost — early exploration is desirable, and once +// entropy_observed_ema stabilises the controller drifts coef back +// toward the target driven by actual observed entropy. +// +// Subsequent emits Wiener-α blend with floor 0.4 (per +// `pearl_wiener_alpha_floor_for_nonstationary` — the policy // entropy co-adapts with training, breaking stationarity). // // Bounds: coef ∈ [0.0, 0.05]. The upper bound prevents the entropy @@ -33,7 +48,6 @@ #define N_ACTIONS 9 #define COEF_MIN 0.0f #define COEF_MAX 0.05f -#define COEF_BOOTSTRAP 0.01f #define ENTROPY_TARGET_FRAC 0.7f #define WIENER_ALPHA_FLOOR 0.4f @@ -63,12 +77,10 @@ extern "C" __global__ void rl_entropy_coef_controller( if (threadIdx.x != 0 || blockIdx.x != 0) return; const float coef_prev = isv[RL_ENTROPY_COEF_INDEX]; - // Bootstrap on sentinel 0.0 per pearl_first_observation_bootstrap. - if (coef_prev == 0.0f) { - isv[RL_ENTROPY_COEF_INDEX] = COEF_BOOTSTRAP; - return; - } + // Compute target from the current input EMA. Shared between + // bootstrap and per-step paths so the dead-zone coincidence with a + // hardcoded bootstrap cannot recur. const float entropy_observed_ema = isv[input_slot]; const float h_max = logf((float)N_ACTIONS); // ln(9) ≈ 2.197 const float h_target = ENTROPY_TARGET_FRAC * h_max; // ≈ 1.538 @@ -78,6 +90,17 @@ extern "C" __global__ void rl_entropy_coef_controller( float coef_target = (deficit / h_max) * COEF_MAX; coef_target = fmaxf(COEF_MIN, fminf(coef_target, COEF_MAX)); + // Bootstrap on sentinel 0.0 per pearl_first_observation_bootstrap: + // first emit replaces directly with the computed target. At cold + // start (input EMA also sentinel-zero), deficit = h_target, + // target = 0.035 (high exploration weight). Distinct from any + // target value the formula can emit for non-sentinel input so the + // per-step Wiener blend always moves. + if (coef_prev == 0.0f) { + isv[RL_ENTROPY_COEF_INDEX] = coef_target; + return; + } + // Wiener-α blend with floor per pearl_wiener_alpha_floor_for_nonstationary. const float a = fmaxf(alpha, WIENER_ALPHA_FLOOR); float coef_new = (1.0f - a) * coef_prev + a * coef_target; diff --git a/crates/ml-alpha/cuda/rl_gamma_controller.cu b/crates/ml-alpha/cuda/rl_gamma_controller.cu index 7b90229f3..7796a5b78 100644 --- a/crates/ml-alpha/cuda/rl_gamma_controller.cu +++ b/crates/ml-alpha/cuda/rl_gamma_controller.cu @@ -18,8 +18,24 @@ // // Bootstrap discipline (per `pearl_first_observation_bootstrap`): the // ISV slot starts at 0.0 (sentinel "uninitialised"). On the first -// emit the kernel writes γ = 0.99 directly. Subsequent emits use a -// Wiener-α blend with floor 0.4 per +// emit the kernel DERIVES γ from the current input slot via the same +// target formula the per-step path uses, instead of writing a +// hardcoded `GAMMA_BOOTSTRAP = 0.99`. At sentinel input +// (trade_duration_ema = 0 → clamped d = 1) target = 0.5 → clamped to +// GAMMA_MIN = 0.90, so the cold-start γ is the floor. This eliminates +// the dead-zone where target(d ≈ 69) = 0.99 = hardcoded bootstrap froze +// the Wiener blend at canonical long-horizon γ for any realistic +// trade duration that happened to land near 69 events. +// +// Trade-off: cold-start γ = 0.90 (floor) is more myopic than the +// previous hardcoded 0.99. Once the trade_duration_ema stabilises +// (within ~10 episodes), the controller drifts γ up toward target +// (0.986 at d=50, 0.99 at d=69, etc.). The brief myopic warm-up is +// acceptable cost for guaranteed responsiveness — a frozen controller +// at canonical γ is worse than a controller that starts low and +// climbs. +// +// Subsequent emits use a Wiener-α blend with floor 0.4 per // `pearl_wiener_alpha_floor_for_nonstationary` (the target γ_target // drifts as the strategy's holding period co-adapts with the policy, // which violates the stationarity precondition of Wiener-optimal α). @@ -31,7 +47,6 @@ #define RL_GAMMA_INDEX 400 #define GAMMA_MIN 0.90f #define GAMMA_MAX 0.999f -#define GAMMA_BOOTSTRAP 0.99f #define WIENER_ALPHA_FLOOR 0.4f @@ -68,18 +83,29 @@ extern "C" __global__ void rl_gamma_controller( if (threadIdx.x != 0 || blockIdx.x != 0) return; const float gamma_prev = isv[RL_GAMMA_INDEX]; - // Bootstrap on sentinel 0.0 per pearl_first_observation_bootstrap. + + // Compute target from the current input EMA. Shared between + // bootstrap and per-step paths so the dead-zone coincidence with + // a hardcoded bootstrap cannot recur. + // Target: γ^d ≈ 0.5 ⇒ γ = 0.5^(1/d). Clamp d ≥ 1 so a + // single-event trade doesn't push γ to 0.5. + const float mean_trade_duration_events = isv[input_slot]; + const float d = fmaxf(mean_trade_duration_events, 1.0f); + float gamma_target = powf(0.5f, 1.0f / d); + gamma_target = fmaxf(GAMMA_MIN, fminf(gamma_target, GAMMA_MAX)); + + // Bootstrap on sentinel 0.0 per pearl_first_observation_bootstrap: + // first emit replaces directly with the computed target. At cold + // start (input EMA also sentinel-zero), clamped d=1, target=0.5, + // clamped to GAMMA_MIN = 0.90 (the floor). Any non-sentinel input + // produces target ≥ 0.90 → ≤ 0.999, distinct from the floor so + // the per-step Wiener blend on subsequent calls always sees a + // prev vs target delta. if (gamma_prev == 0.0f) { - isv[RL_GAMMA_INDEX] = GAMMA_BOOTSTRAP; + isv[RL_GAMMA_INDEX] = gamma_target; return; } - const float mean_trade_duration_events = isv[input_slot]; - // Target: γ^d ≈ 0.5 ⇒ γ = 0.5^(1/d). Clamp d ≥ 1 so a single-event - // trade doesn't push γ to 0.5. - const float d = fmaxf(mean_trade_duration_events, 1.0f); - const float gamma_target = powf(0.5f, 1.0f / d); - // Wiener-α blend with floor per pearl_wiener_alpha_floor_for_nonstationary. const float a = fmaxf(alpha, WIENER_ALPHA_FLOOR); float gamma_new = (1.0f - a) * gamma_prev + a * gamma_target; diff --git a/crates/ml-alpha/src/trainer/integrated.rs b/crates/ml-alpha/src/trainer/integrated.rs index 1532db5cc..5a9f4a40d 100644 --- a/crates/ml-alpha/src/trainer/integrated.rs +++ b/crates/ml-alpha/src/trainer/integrated.rs @@ -880,10 +880,19 @@ impl IntegratedTrainer { /// bootstrap path per `pearl_first_observation_bootstrap`, and /// writes its canonical default into the slot: /// - /// ISV[400] γ = 0.99 (GAMMA_BOOTSTRAP) + /// ISV[400] γ = target(trade_duration_ema) — 0.90 at + /// sentinel input (post-R9 audit; was + /// hardcoded 0.99 which coincided with + /// target(d≈69) creating a Wiener-blend + /// dead-zone at canonical hold time) /// ISV[401] τ = 0.005 (TAU_BOOTSTRAP) /// ISV[402] ε = 0.2 (EPS_BOOTSTRAP) - /// ISV[403] entropy_coef = 0.01 (COEF_BOOTSTRAP) + /// ISV[403] entropy_coef = target(entropy_observed_ema) — 0.035 + /// at sentinel input (post-R9 audit; + /// was hardcoded 0.01 which coincided + /// with target(h≈1.099) creating a + /// Wiener-blend dead-zone at canonical + /// mid-range entropy) /// ISV[404] n_rollout_steps= 2048 (ROLLOUT_BOOTSTRAP) /// ISV[405] per_α = target(td_kurtosis_ema) — 0.4 at /// sentinel input (post-R9 audit; was diff --git a/crates/ml-alpha/tests/isv_bootstrap.rs b/crates/ml-alpha/tests/isv_bootstrap.rs index b779aa130..f74eb7b92 100644 --- a/crates/ml-alpha/tests/isv_bootstrap.rs +++ b/crates/ml-alpha/tests/isv_bootstrap.rs @@ -35,10 +35,20 @@ use ml_core::device::MlDevice; // corresponding `*_BOOTSTRAP` #define in // `crates/ml-alpha/cuda/rl_*_controller.cu`. Source of truth is the // .cu file; this test asserts the kernel actually wrote those values. -const GAMMA_BOOTSTRAP: f32 = 0.99; +// Bootstrap value at sentinel input (post-R9 audit: derive-from-input +// pattern in rl_gamma_controller.cu). Sentinel d_ema = 0 → clamped +// d=1 → target = 0.5^1 = 0.5 → clamped to GAMMA_MIN = 0.90. Was +// hardcoded 0.99 (canonical long-horizon) before R9 closed the +// dead-zone at trade_duration_ema ≈ 69 events. +const GAMMA_BOOTSTRAP: f32 = 0.90; const TAU_BOOTSTRAP: f32 = 0.005; const EPS_BOOTSTRAP: f32 = 0.2; -const COEF_BOOTSTRAP: f32 = 0.01; +// Bootstrap value at sentinel input (post-R9 audit: derive-from-input +// pattern in rl_entropy_coef_controller.cu). Sentinel h_obs = 0 → +// deficit = h_target = 1.538 → target = (1.538 / 2.197) × 0.05 ≈ +// 0.035. Was hardcoded 0.01 (canonical PPO entropy bonus) before R9 +// closed the dead-zone at entropy_observed_ema ≈ 1.099. +const COEF_BOOTSTRAP: f32 = 0.035; const ROLLOUT_BOOTSTRAP: f32 = 2048.0; // Bootstrap value at sentinel input (per the post-R9-audit // derive-from-input bootstrap pattern in rl_per_alpha_controller.cu): diff --git a/crates/ml-alpha/tests/r3_ema_advantage.rs b/crates/ml-alpha/tests/r3_ema_advantage.rs index 32a18f46b..c585c916a 100644 --- a/crates/ml-alpha/tests/r3_ema_advantage.rs +++ b/crates/ml-alpha/tests/r3_ema_advantage.rs @@ -165,12 +165,18 @@ fn r3_compute_advantage_return_formula_holds() { let Some((dev, trainer)) = build_trainer() else { return }; let stream = dev.cuda_stream().expect("cuda_stream").clone(); - // R1 bootstrapped ISV[γ=400] to 0.99. Verify before the test. + // R1 bootstrapped ISV[γ=400] via rl_gamma_controller. Post-R9 + // audit, the bootstrap derives from the input EMA via the same + // target formula the per-step path uses — at sentinel input it + // lands on the floor γ = GAMMA_MIN = 0.90. The test below + // computes its expected values from whatever γ ISV holds, so the + // pre-condition is just "γ is in the valid bounded range" rather + // than a hardcoded canonical value. let isv = readback_isv(&dev, &trainer.isv_d); let gamma = isv[RL_GAMMA_INDEX]; assert!( - (gamma - 0.99).abs() < 1e-6, - "pre-condition: γ should be R1-bootstrapped to 0.99; got {gamma}" + gamma >= 0.90 && gamma <= 0.999, + "pre-condition: γ should be in [0.90, 0.999]; got {gamma}" ); // Inputs: r=0, done=0, v_t=v_tp1=k. Expected: diff --git a/crates/ml-alpha/tests/r5_controllers_and_soft_update.rs b/crates/ml-alpha/tests/r5_controllers_and_soft_update.rs index 1e3259abb..58ddc2b5e 100644 --- a/crates/ml-alpha/tests/r5_controllers_and_soft_update.rs +++ b/crates/ml-alpha/tests/r5_controllers_and_soft_update.rs @@ -37,10 +37,14 @@ use ml_alpha::trainer::perception::PerceptionTrainerConfig; use ml_core::device::MlDevice; use std::sync::Arc; -const GAMMA_BOOTSTRAP: f32 = 0.99; +// Bootstrap value at sentinel input — see isv_bootstrap.rs for the +// post-R9-audit derive-from-input pattern explanation. +const GAMMA_BOOTSTRAP: f32 = 0.90; const TAU_BOOTSTRAP: f32 = 0.005; const EPS_BOOTSTRAP: f32 = 0.2; -const COEF_BOOTSTRAP: f32 = 0.01; +// Bootstrap value at sentinel input — see isv_bootstrap.rs for the +// post-R9-audit derive-from-input pattern explanation. +const COEF_BOOTSTRAP: f32 = 0.035; const ROLLOUT_BOOTSTRAP: f32 = 2048.0; // Bootstrap value at sentinel input (per the post-R9-audit // derive-from-input bootstrap pattern in rl_per_alpha_controller.cu): @@ -117,8 +121,12 @@ fn g3_per_step_controllers_move_isv_outputs_when_fed_real_emas() { // observation replaces directly). Choose distinct values per slot // so a slot-wiring bug (controller reads wrong slot) would // produce out-of-range outputs we can detect. + // Input values chosen to produce targets distinct from each + // controller's clamped floor — production trade duration EMAs + // typically settle in the 10-100 range, far above the d=1 edge + // where γ target clamps to GAMMA_MIN. let inputs: [(usize, f32); 7] = [ - (RL_MEAN_TRADE_DURATION_EMA_INDEX, 1.0), // → rl_gamma + (RL_MEAN_TRADE_DURATION_EMA_INDEX, 20.0), // → rl_gamma (target ≈ 0.966) (RL_Q_DIVERGENCE_EMA_INDEX, 0.5), // → rl_target_tau (RL_KL_PI_EMA_INDEX, 0.1), // → rl_ppo_clip (RL_ENTROPY_OBSERVED_EMA_INDEX, 0.5), // → rl_entropy_coef