diff --git a/crates/ml-alpha/examples/alpha_rl_train.rs b/crates/ml-alpha/examples/alpha_rl_train.rs index 8ebaa8f98..1b601a33b 100644 --- a/crates/ml-alpha/examples/alpha_rl_train.rs +++ b/crates/ml-alpha/examples/alpha_rl_train.rs @@ -39,10 +39,15 @@ use ml_alpha::data::loader::{ use ml_alpha::heads::HORIZONS; use ml_alpha::rl::isv_slots::{ RL_ADVANTAGE_VAR_RATIO_EMA_INDEX, RL_ENTROPY_COEF_INDEX, RL_ENTROPY_OBSERVED_EMA_INDEX, - RL_GAMMA_INDEX, RL_KL_PI_EMA_INDEX, RL_LR_AUX_INDEX, RL_LR_BCE_INDEX, RL_LR_PI_INDEX, - RL_LR_Q_INDEX, RL_LR_V_INDEX, RL_MEAN_ABS_PNL_EMA_INDEX, RL_MEAN_TRADE_DURATION_EMA_INDEX, - RL_N_ROLLOUT_STEPS_INDEX, RL_PER_ALPHA_INDEX, RL_PPO_CLIP_INDEX, RL_Q_DIVERGENCE_EMA_INDEX, - RL_REWARD_SCALE_INDEX, RL_TARGET_TAU_INDEX, RL_TD_KURTOSIS_EMA_INDEX, + RL_GAMMA_INDEX, RL_KL_PI_EMA_INDEX, RL_LR_AUX_INDEX, RL_LR_BCE_INDEX, RL_LR_PI_BEST_LOSS_INDEX, + RL_LR_PI_INDEX, RL_LR_PI_LOSS_EMA_INDEX, RL_LR_PI_STEPS_SINCE_BEST_INDEX, + RL_LR_Q_BEST_LOSS_INDEX, RL_LR_Q_INDEX, RL_LR_Q_LOSS_EMA_INDEX, + RL_LR_Q_STEPS_SINCE_BEST_INDEX, RL_LR_V_BEST_LOSS_INDEX, RL_LR_V_INDEX, + RL_LR_V_LOSS_EMA_INDEX, RL_LR_V_STEPS_SINCE_BEST_INDEX, RL_MEAN_ABS_PNL_EMA_INDEX, + RL_MEAN_TRADE_DURATION_EMA_INDEX, RL_N_ROLLOUT_STEPS_INDEX, RL_PER_ALPHA_INDEX, + RL_PI_GRAD_NORM_EMA_INDEX, RL_PPO_CLIP_INDEX, RL_Q_DIVERGENCE_EMA_INDEX, + RL_Q_GRAD_NORM_EMA_INDEX, RL_REWARD_SCALE_INDEX, RL_TARGET_TAU_INDEX, RL_TD_KURTOSIS_EMA_INDEX, + RL_V_GRAD_NORM_EMA_INDEX, }; use ml_alpha::trainer::integrated::{ read_slice_d_pub, read_slice_i32_d_pub, IntegratedStepStats, IntegratedTrainer, @@ -506,6 +511,31 @@ fn main() -> Result<()> { "td_kurtosis": isv[RL_TD_KURTOSIS_EMA_INDEX], "mean_abs_pnl": isv[RL_MEAN_ABS_PNL_EMA_INDEX], }, + // Grad-norm EMAs (diagnostic — no longer drive the LR + // controller as of commit 042de99e6 but still useful to + // correlate with plateau-decay events). + "grad_norm_ema": { + "q": isv[RL_Q_GRAD_NORM_EMA_INDEX], + "pi": isv[RL_PI_GRAD_NORM_EMA_INDEX], + "v": isv[RL_V_GRAD_NORM_EMA_INDEX], + }, + // Plateau-decay LR controller state (per head). loss_ema is + // the controller's internal slow EMA at α=0.05; best is the + // lowest loss_ema observed so far; steps_since_best is the + // staleness counter that fires LR halving at PLATEAU_PATIENCE + // (1000). Confirms whether the controller is seeing plateaus + // and decaying for the right reason. + "lr_plateau": { + "q": { "loss_ema": isv[RL_LR_Q_LOSS_EMA_INDEX], + "best": isv[RL_LR_Q_BEST_LOSS_INDEX], + "stale": isv[RL_LR_Q_STEPS_SINCE_BEST_INDEX] }, + "pi": { "loss_ema": isv[RL_LR_PI_LOSS_EMA_INDEX], + "best": isv[RL_LR_PI_BEST_LOSS_INDEX], + "stale": isv[RL_LR_PI_STEPS_SINCE_BEST_INDEX] }, + "v": { "loss_ema": isv[RL_LR_V_LOSS_EMA_INDEX], + "best": isv[RL_LR_V_BEST_LOSS_INDEX], + "stale": isv[RL_LR_V_STEPS_SINCE_BEST_INDEX] }, + }, "replay_len": trainer.replay.len(), "rewards": { "sum": reward_sum,