diff --git a/crates/ml-alpha/cuda/rl_lr_controller.cu b/crates/ml-alpha/cuda/rl_lr_controller.cu index 3897c2614..af3af7cbd 100644 --- a/crates/ml-alpha/cuda/rl_lr_controller.cu +++ b/crates/ml-alpha/cuda/rl_lr_controller.cu @@ -51,7 +51,16 @@ #define RL_LR_AUX_INDEX 416 #define LR_BOOTSTRAP 1e-3f -#define LR_MIN 1e-5f +// LR_MIN raised from 1e-5 to 1e-4 — at 1e-5 the per-step Adam update is +// too small to make meaningful progress, so even when the model is +// genuinely below the plateau-detected best the gradient can't move +// loss_ema. alpha-rl-frt7s fold0 (commit 95dcc4e31) confirmed: 7 +// halvings drove all 3 LRs to 1e-5 by step 7783, after which Q stayed +// stuck at loss_ema ≈ 2.82 for 42k more steps (only 7% information +// gain from uniform=3.04). At 1e-4 (still 10× below bootstrap) Q has +// 10× more learning per step at the floor, enough to escape the +// noise-locked best. +#define LR_MIN 1e-4f #define LR_MAX 1e-2f // unused in monotone-decay design (kept for clamp arithmetic safety) // Slow EMA α for the controller's internal loss tracking. Well below @@ -81,7 +90,15 @@ #define IMPROVEMENT_THRESHOLD 0.99f #define PLATEAU_PATIENCE 1000.0f #define DECAY_FACTOR 0.5f -#define LR_WARMUP_STEPS 500.0f +// LR_WARMUP_STEPS raised from 500 to 2000 — at 500, a lucky -1σ EMA +// excursion during the early-training noise window can lock `best` at +// an unreachable value. alpha-rl-frt7s fold0: Q `best=2.3230` locked +// at step ~783 from a brief downward excursion; loss_ema (mean 2.82, +// std 0.10) never dipped below it across 50k steps. 2000 steps = +// ~145 EMA half-lives at α=0.05 — well past convergence of even the +// slowest signal, and gives the noise distribution enough time to +// settle before best is locked in. +#define LR_WARMUP_STEPS 2000.0f __device__ __forceinline__ void plateau_decay_head( float* isv, diff --git a/crates/ml-alpha/examples/alpha_rl_train.rs b/crates/ml-alpha/examples/alpha_rl_train.rs index 0f189433f..877eccef4 100644 --- a/crates/ml-alpha/examples/alpha_rl_train.rs +++ b/crates/ml-alpha/examples/alpha_rl_train.rs @@ -625,6 +625,13 @@ fn main() -> Result<()> { // controller-cu file level — there's no ISV slot for these // design constants because they're fundamental to the // controller's behaviour, not adaptive). + // R9 — K-loop training-intensification iterations performed + // this step. K = clamp(isv[404]/1024, 1, 8). Surfaces the + // wire-up of n_rollout_steps as the DQN-replay/PPO multi- + // update knob: when advantages are noisy, do K Q+π+V + // updates per env step (each with fresh PER sample) so + // gradient signal isn't starved at b_size=1. + "k_updates": trainer.last_k_updates, "controller_branch": { "rollout_steps_input": isv[RL_ADVANTAGE_VAR_RATIO_EMA_INDEX], // ISV-driven target — seeded by rl_streaming_clamp_init, diff --git a/crates/ml-alpha/src/trainer/integrated.rs b/crates/ml-alpha/src/trainer/integrated.rs index 53b758bbc..53f5c17cc 100644 --- a/crates/ml-alpha/src/trainer/integrated.rs +++ b/crates/ml-alpha/src/trainer/integrated.rs @@ -242,7 +242,7 @@ impl Default for IntegratedTrainerConfig { } /// Stats returned from `step_synthetic`. Useful for tests + diagnostics. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Default)] pub struct IntegratedStepStats { pub l_bce: f32, pub l_q: f32, @@ -532,6 +532,14 @@ pub struct IntegratedTrainer { /// `last_q_loss`. Zero at construction. last_v_loss: f32, + /// R9 — number of training iterations performed in the most-recent + /// `step_with_lobsim` call. Derived from `isv[RL_N_ROLLOUT_STEPS_INDEX]` + /// via `K = clamp(isv[404] / 1024, 1, 8)`. Surfaces in the diag + /// JSONL via `pub` accessor so post-hoc analysis can see how the + /// `n_rollout_steps` controller is driving training intensity per + /// env step. Zero at construction (first step overwrites). + pub last_k_updates: usize, + /// Host-side step counter for Thompson-sampling RNG salt (Phase E.3b). /// Increments once per `step_with_lobsim` call so consecutive calls /// draw different actions from the same Q-distribution. Per @@ -1075,6 +1083,7 @@ impl IntegratedTrainer { last_pi_loss: 0.0, last_q_loss: 0.0, last_v_loss: 0.0, + last_k_updates: 0, step_counter: 0, } .with_controllers_bootstrapped()?) @@ -2736,38 +2745,58 @@ impl IntegratedTrainer { self.push_to_replay(b_size) .context("step_with_lobsim: push_to_replay")?; - let per_indices = self - .sample_and_gather(b_size) - .context("step_with_lobsim: sample_and_gather")?; - // ── Step 7b: delegate to step_synthetic. ────────────────────── - // step_synthetic now reads: - // * SAMPLED buffers (sampled_h_t_d / sampled_h_tp1_d / - // sampled_actions_d / sampled_rewards_d / sampled_dones_d / - // sampled_next_actions_d) for the off-policy Q forward + - // Bellman target build + Q backward (PER's "always on" per - // `feedback_always_per`). - // * CURRENT-step buffers (h_t via perception.h_t_view(), - // actions_d, log_pi_old_d, advantages_d, returns_d) for - // the on-policy PPO surrogate + V regression. + // ── Step 7b: K-loop training intensification driven by + // `n_rollout_steps` (ISV[404]). Wires the previously-write-only + // controller to actual training behavior. // - // The encoder backward consumes ONLY π + V grad_h_t - // contributions (R7d stop-grad on Q's encoder grad — see - // step_synthetic Step 10 commentary). - let stats = self.step_synthetic(snapshots)?; + // The controller widens its emitted value when advantages are + // noisy (var/|mean| high → "need more samples to learn"). We + // map that to the number of training iterations performed per + // env step: K = clamp(isv[404] / K_DIVISOR, 1, K_MAX). Each + // iteration is a fresh PER sample + full step_synthetic call + // (Q + π + V forward, backward, Adam) + PER priority update. + // + // * isv[404] = 256 (MIN) → K = 1 (single update per env step, current behavior) + // * isv[404] = 2048 (BOOTSTRAP) → K = 2 + // * isv[404] = 8192 (MAX) → K = 8 + // + // Adapts the training:env ratio so noisy regimes get more + // gradient samples without slowing down env stepping. b_size=1 + // means each iteration is cheap; the K-multiplier directly + // addresses the per-step gradient signal starvation that left + // l_q stuck at 2.82 (uniform=3.04, only 7% information gain) + // in alpha-rl-frt7s. + let n_rollout_steps = + self.isv_host[crate::rl::isv_slots::RL_N_ROLLOUT_STEPS_INDEX]; + let k_updates = ((n_rollout_steps / 1024.0).round() as usize) + .clamp(1, 8); + self.last_k_updates = k_updates; - // ── Step 7c (R7d): PER priority update. ─────────────────────── - // step_synthetic's Q backward populated self.td_per_sample_d - // (per-sample CE loss = |TD-error| analogue for distributional - // Q). DtoH and feed back to the replay buffer so the next - // step's sample picks the high-TD transitions per the - // canonical Schaul 2015 PER recipe. - if !per_indices.is_empty() { - let td_per_sample_host = - read_slice_d(&self.stream, &self.td_per_sample_d, b_size) - .context("step_with_lobsim: read td_per_sample_d")?; - self.replay - .update_priorities(&per_indices, &td_per_sample_host); + // Track the LAST iteration's stats for caller reporting (per- + // iteration stats are not aggregated — caller sees the loss + // after the final Adam step of the last iteration, which is + // the most-recent state of the model). + let mut stats = IntegratedStepStats::default(); + for _k_iter in 0..k_updates { + // Fresh PER sample per iteration — different transitions + // each gradient step. Without this the K-loop would just + // do K Adam steps on the same gradient, equivalent to a + // single step with K × LR (and would interact badly with + // Adam's per-parameter moment estimates). + let per_indices = self + .sample_and_gather(b_size) + .context("step_with_lobsim: sample_and_gather (k-loop)")?; + stats = self.step_synthetic(snapshots)?; + // PER priority update per iteration so the next sample + // picks transitions with updated TD errors. + if !per_indices.is_empty() { + let td_per_sample_host = + read_slice_d(&self.stream, &self.td_per_sample_d, b_size) + .context("step_with_lobsim: read td_per_sample_d (k-loop)")?; + self.replay + .update_priorities(&per_indices, &td_per_sample_host); + } } // Target-net soft update (Phase R5 + R6) — runs once per step