wip(rl): n-step returns foundation — struct, ISV slot, buffer field

Adds RL_N_STEP_INDEX (slot 542, default 10), n_step_gamma field to
Transition, and NStepEntry ring buffer struct to IntegratedTrainer.

Remaining: implement n-step accumulation in push_to_replay and
modify bellman_target_projection.cu to use n_step_gamma.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

crates/ml-alpha/src/rl/common.rs

@@ -134,8 +134,14 @@ pub struct Transition {
     pub next_h_t: CudaSlice<f32>,
 
     /// True at trade close (position returned to 0) or end-of-stream.
+    /// For n-step: true if ANY step in [t, t+n) had a done event.
     pub done: bool,
 
+    /// γⁿ — effective discount for the n-step bootstrap term.
+    /// Bellman target: R_n + n_step_gamma × Q(s_{t+n}).
+    /// Set to 0.0 if any done occurred in the n-step window.
+    pub n_step_gamma: f32,
+
     /// log π_old(action | state) recorded at sample time. PPO uses this
     /// as the denominator in the importance ratio
     /// `r_t = exp(log π_new - log π_old)`.

crates/ml-alpha/src/rl/isv_slots.rs

@@ -951,6 +951,10 @@ pub const RL_SESSION_LOSS_LIMIT_INDEX: usize = 540;
 /// Session risk: EMA alpha for the PnL tracker. Default 0.02.
 pub const RL_SESSION_EMA_ALPHA_INDEX: usize = 541;
 
+/// N-step return horizon. Bellman target uses R_n + γⁿQ(s_{t+n})
+/// instead of r + γQ(s'). Default 10 (2.5 seconds at 4 events/sec).
+pub const RL_N_STEP_INDEX: usize = 542;
+
 /// Last RL-allocated slot index (exclusive). The integrated trainer
 /// extends `ISV_TOTAL_DIM` to at least this value at trainer init time.
-pub const RL_SLOTS_END: usize = 542;
+pub const RL_SLOTS_END: usize = 543;

crates/ml-alpha/src/trainer/integrated.rs

@@ -321,6 +321,18 @@ pub struct IntegratedStepStats {
     pub lambdas: LossLambdas,
 }
 
+/// Entry in the n-step return ring buffer. Holds one step's data
+/// until n steps have accumulated, at which point the oldest entry
+/// is popped and pushed to PER with the n-step discounted return.
+struct NStepEntry {
+    h_t: CudaSlice<f32>,
+    action: u32,
+    raw_reward: f32,
+    scaled_reward: f32,
+    done: bool,
+    log_pi_old: f32,
+}
+
 pub struct IntegratedTrainer {
     #[allow(dead_code)]
     cfg: IntegratedTrainerConfig,
@@ -627,6 +639,7 @@ pub struct IntegratedTrainer {
     /// Per `feedback_always_per` PER is always-on. Wired in R7d after
     /// the existing dead struct sat in `src/rl/replay.rs` since Phase C.
     pub replay: crate::rl::replay::ReplayBuffer,
+    n_step_buffer: Vec<Vec<NStepEntry>>,
 
     /// Per-step gather buffer for PER-sampled `h_t`. Populated each step
     /// by a per-batch DtoD loop over `replay.transitions[sampled_idx].h_t`.
@@ -1518,6 +1531,7 @@ impl IntegratedTrainer {
             returns_d,
             h_tp1_d,
             replay,
+            n_step_buffer: (0..b_size).map(|_| Vec::with_capacity(16)).collect(),
             sampled_h_t_d,
             sampled_h_tp1_d,
             sampled_actions_d,
@@ -1600,7 +1614,7 @@ impl IntegratedTrainer {
         // (slot, value) pair — pure device write, no HtoD per
         // `feedback_no_htod_htoh_only_mapped_pinned`.
         {
-            let isv_constants: [(usize, f32); 77] = [
+            let isv_constants: [(usize, f32); 78] = [
                 // Static seeds for the adaptive reward-clamp controller —
                 // these are the initial values that
                 // `rl_reward_clamp_controller` will replace once it
@@ -1684,6 +1698,7 @@ impl IntegratedTrainer {
                 (crate::rl::isv_slots::RL_ASYM_WIN_TRAIL_FACTOR_INDEX, 0.5),
                 (crate::rl::isv_slots::RL_SESSION_LOSS_LIMIT_INDEX, -50.0),
                 (crate::rl::isv_slots::RL_SESSION_EMA_ALPHA_INDEX,  0.02),
+                (crate::rl::isv_slots::RL_N_STEP_INDEX,            10.0),
                 (crate::rl::isv_slots::RL_MULTIRES_HORIZON_1_INDEX, 1.0),
                 (crate::rl::isv_slots::RL_MULTIRES_HORIZON_2_INDEX, 10.0),
                 (crate::rl::isv_slots::RL_MULTIRES_HORIZON_3_INDEX, 600.0),
@@ -4821,6 +4836,7 @@ impl IntegratedTrainer {
                 raw_reward: raw_rewards_host[b],
                 next_h_t: h_tp1_per_b,
                 done: dones_host[b] > 0.5,
+                n_step_gamma: self.isv_host[crate::rl::isv_slots::RL_GAMMA_INDEX],
                 log_pi_old: log_pi_old_host[b],
                 q_value_old: [0.0; N_ACTIONS],
             });