From 9e2c036c5e36aaa6aef9d22e56e625cf625cfa5c Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Wed, 27 May 2026 11:14:58 +0200 Subject: [PATCH] =?UTF-8?q?tune(rl):=20lower=20KL=20=CE=B2=3D0.0005=20+=20?= =?UTF-8?q?reward=5Fkl=3D0.001=20for=20b=3D1024?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit β=0.003 pinned Hold=100% at b=1024 despite 60 dones/step. The KL gradient overwhelms even the dense PPO signal. Try 6× lower β to find the balance point. Co-Authored-By: Claude Opus 4.7 --- crates/ml-alpha/src/trainer/integrated.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/ml-alpha/src/trainer/integrated.rs b/crates/ml-alpha/src/trainer/integrated.rs index 74724f12a..9b8cedcf3 100644 --- a/crates/ml-alpha/src/trainer/integrated.rs +++ b/crates/ml-alpha/src/trainer/integrated.rs @@ -2768,8 +2768,8 @@ impl IntegratedTrainer { (crate::rl::isv_slots::RL_TARGET_TAU_MAX_INDEX, 0.005), (crate::rl::isv_slots::RL_HOLD_PRIOR_INDEX, 0.1), (crate::rl::isv_slots::RL_HOLD_TARGET_FRAC_INDEX, 0.5), - (crate::rl::isv_slots::RL_KL_REF_BETA_INDEX, 0.003), - (crate::rl::isv_slots::RL_REWARD_KL_BETA_INDEX, 0.005), + (crate::rl::isv_slots::RL_KL_REF_BETA_INDEX, 0.0005), + (crate::rl::isv_slots::RL_REWARD_KL_BETA_INDEX, 0.001), (crate::rl::isv_slots::RL_KL_REF_TARGET_INDEX, 0.5), (crate::rl::isv_slots::RL_EPS_BOOTSTRAP_INDEX, 0.2), (crate::rl::isv_slots::RL_ROLLOUT_BOOTSTRAP_INDEX, 2048.0),