From 38cf5fee0b1b92da842ccc568d560f66301ca577 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 25 May 2026 20:13:26 +0200 Subject: [PATCH] =?UTF-8?q?wip(rl):=20n-step=20returns=20foundation=20?= =?UTF-8?q?=E2=80=94=20struct,=20ISV=20slot,=20buffer=20field?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds RL_N_STEP_INDEX (slot 542, default 10), n_step_gamma field to Transition, and NStepEntry ring buffer struct to IntegratedTrainer. Remaining: implement n-step accumulation in push_to_replay and modify bellman_target_projection.cu to use n_step_gamma. Co-Authored-By: Claude Opus 4.7 --- crates/ml-alpha/src/rl/common.rs | 6 ++++++ crates/ml-alpha/src/rl/isv_slots.rs | 6 +++++- crates/ml-alpha/src/trainer/integrated.rs | 18 +++++++++++++++++- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/crates/ml-alpha/src/rl/common.rs b/crates/ml-alpha/src/rl/common.rs index d7da9d7e0..e5baf7060 100644 --- a/crates/ml-alpha/src/rl/common.rs +++ b/crates/ml-alpha/src/rl/common.rs @@ -134,8 +134,14 @@ pub struct Transition { pub next_h_t: CudaSlice, /// True at trade close (position returned to 0) or end-of-stream. + /// For n-step: true if ANY step in [t, t+n) had a done event. pub done: bool, + /// γⁿ — effective discount for the n-step bootstrap term. + /// Bellman target: R_n + n_step_gamma × Q(s_{t+n}). + /// Set to 0.0 if any done occurred in the n-step window. + pub n_step_gamma: f32, + /// log π_old(action | state) recorded at sample time. PPO uses this /// as the denominator in the importance ratio /// `r_t = exp(log π_new - log π_old)`. diff --git a/crates/ml-alpha/src/rl/isv_slots.rs b/crates/ml-alpha/src/rl/isv_slots.rs index 91becdfb9..c0e15d6f8 100644 --- a/crates/ml-alpha/src/rl/isv_slots.rs +++ b/crates/ml-alpha/src/rl/isv_slots.rs @@ -951,6 +951,10 @@ pub const RL_SESSION_LOSS_LIMIT_INDEX: usize = 540; /// Session risk: EMA alpha for the PnL tracker. Default 0.02. pub const RL_SESSION_EMA_ALPHA_INDEX: usize = 541; +/// N-step return horizon. Bellman target uses R_n + γⁿQ(s_{t+n}) +/// instead of r + γQ(s'). Default 10 (2.5 seconds at 4 events/sec). +pub const RL_N_STEP_INDEX: usize = 542; + /// Last RL-allocated slot index (exclusive). The integrated trainer /// extends `ISV_TOTAL_DIM` to at least this value at trainer init time. -pub const RL_SLOTS_END: usize = 542; +pub const RL_SLOTS_END: usize = 543; diff --git a/crates/ml-alpha/src/trainer/integrated.rs b/crates/ml-alpha/src/trainer/integrated.rs index dacc5feaa..3e5684372 100644 --- a/crates/ml-alpha/src/trainer/integrated.rs +++ b/crates/ml-alpha/src/trainer/integrated.rs @@ -321,6 +321,18 @@ pub struct IntegratedStepStats { pub lambdas: LossLambdas, } +/// Entry in the n-step return ring buffer. Holds one step's data +/// until n steps have accumulated, at which point the oldest entry +/// is popped and pushed to PER with the n-step discounted return. +struct NStepEntry { + h_t: CudaSlice, + action: u32, + raw_reward: f32, + scaled_reward: f32, + done: bool, + log_pi_old: f32, +} + pub struct IntegratedTrainer { #[allow(dead_code)] cfg: IntegratedTrainerConfig, @@ -627,6 +639,7 @@ pub struct IntegratedTrainer { /// Per `feedback_always_per` PER is always-on. Wired in R7d after /// the existing dead struct sat in `src/rl/replay.rs` since Phase C. pub replay: crate::rl::replay::ReplayBuffer, + n_step_buffer: Vec>, /// Per-step gather buffer for PER-sampled `h_t`. Populated each step /// by a per-batch DtoD loop over `replay.transitions[sampled_idx].h_t`. @@ -1518,6 +1531,7 @@ impl IntegratedTrainer { returns_d, h_tp1_d, replay, + n_step_buffer: (0..b_size).map(|_| Vec::with_capacity(16)).collect(), sampled_h_t_d, sampled_h_tp1_d, sampled_actions_d, @@ -1600,7 +1614,7 @@ impl IntegratedTrainer { // (slot, value) pair — pure device write, no HtoD per // `feedback_no_htod_htoh_only_mapped_pinned`. { - let isv_constants: [(usize, f32); 77] = [ + let isv_constants: [(usize, f32); 78] = [ // Static seeds for the adaptive reward-clamp controller — // these are the initial values that // `rl_reward_clamp_controller` will replace once it @@ -1684,6 +1698,7 @@ impl IntegratedTrainer { (crate::rl::isv_slots::RL_ASYM_WIN_TRAIL_FACTOR_INDEX, 0.5), (crate::rl::isv_slots::RL_SESSION_LOSS_LIMIT_INDEX, -50.0), (crate::rl::isv_slots::RL_SESSION_EMA_ALPHA_INDEX, 0.02), + (crate::rl::isv_slots::RL_N_STEP_INDEX, 10.0), (crate::rl::isv_slots::RL_MULTIRES_HORIZON_1_INDEX, 1.0), (crate::rl::isv_slots::RL_MULTIRES_HORIZON_2_INDEX, 10.0), (crate::rl::isv_slots::RL_MULTIRES_HORIZON_3_INDEX, 600.0), @@ -4821,6 +4836,7 @@ impl IntegratedTrainer { raw_reward: raw_rewards_host[b], next_h_t: h_tp1_per_b, done: dones_host[b] > 0.5, + n_step_gamma: self.isv_host[crate::rl::isv_slots::RL_GAMMA_INDEX], log_pi_old: log_pi_old_host[b], q_value_old: [0.0; N_ACTIONS], });