From 4fc7d631851061218c65090c530b9c4f9b770b06 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 26 May 2026 08:48:11 +0200 Subject: [PATCH] =?UTF-8?q?feat(rl):=20wire=20bidirectional=20HER=20?= =?UTF-8?q?=E2=80=94=20track=20+=20inject=20+=20forward=20in=20step=20pipe?= =?UTF-8?q?line?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backward HER: on trade close, injects synthetic with peak PnL if peak > 1.5× actual. Forward HER: evaluates closed trades after 50 steps, injects if holding would have been better. ISV bootstrap: threshold=1.5, priority_boost=3.0, lookahead=50. Co-Authored-By: Claude Opus 4.7 --- crates/ml-alpha/build.rs | 3 + crates/ml-alpha/src/trainer/integrated.rs | 165 +++++++++++++++++++++- 2 files changed, 167 insertions(+), 1 deletion(-) diff --git a/crates/ml-alpha/build.rs b/crates/ml-alpha/build.rs index aa9471c2c..e0ee321c6 100644 --- a/crates/ml-alpha/build.rs +++ b/crates/ml-alpha/build.rs @@ -106,6 +106,9 @@ const KERNELS: &[&str] = &[ "rl_per_sample", // GPU PER: stratified proportional sampling via sum-tree walk + gather "rl_per_update_priority", // GPU PER: write |TD|^α to tree leaves + block-wide max reduction "rl_per_tree_rebuild", // GPU PER: bottom-up parallel sum-tree rebuild (no atomics) + "rl_hindsight_track", // HER Phase 1: per-step mid-price ring + peak tracking for backward hindsight + "rl_hindsight_inject", // HER Phase 2: backward inject — synthetic replay push on done if peak >> actual + "rl_hindsight_forward", // HER Phase 3: forward continuation — evaluates closed trades after lookahead "rl_increment_step", // device-resident step counter bump (ISV[548] += 1.0); graph-safe prereq — removes scalar current_step from all downstream kernel args "rl_fused_controllers", // fused kernel: 10 RL ISV controllers in one launch (gamma, tau, ppo_clip, entropy_coef, rollout_steps, per_alpha, reward_scale, ppo_ratio_clamp, gate_threshold, q_distill_lambda) — saves 9 kernel launch overheads (~40-80μs/step) ]; diff --git a/crates/ml-alpha/src/trainer/integrated.rs b/crates/ml-alpha/src/trainer/integrated.rs index 49b1317b9..848565ee2 100644 --- a/crates/ml-alpha/src/trainer/integrated.rs +++ b/crates/ml-alpha/src/trainer/integrated.rs @@ -293,6 +293,16 @@ const RL_PER_UPDATE_PRIORITY_CUBIN: &[u8] = const RL_PER_TREE_REBUILD_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/rl_per_tree_rebuild.cubin")); +// Bidirectional HER kernels — track (mid-price ring + peak), backward +// inject (synthetic push on done if peak >> actual), forward (evaluate +// closed trades after lookahead window). +const RL_HINDSIGHT_TRACK_CUBIN: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/rl_hindsight_track.cubin")); +const RL_HINDSIGHT_INJECT_CUBIN: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/rl_hindsight_inject.cubin")); +const RL_HINDSIGHT_FORWARD_CUBIN: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/rl_hindsight_forward.cubin")); + /// Per-head LR controller Wiener-α target. The controller kernel floors /// this at 0.4 per `pearl_wiener_alpha_floor_for_nonstationary` so the /// effective blend rate stays ≥ 0.4 regardless of this seed value. Kept @@ -802,6 +812,15 @@ pub struct IntegratedTrainer { _rl_per_tree_rebuild_module: Arc, rl_per_tree_rebuild_fn: CudaFunction, + // ── Bidirectional HER kernel handles ─────────────────────────── + pub hindsight: crate::rl::gpu_hindsight::GpuHindsight, + _rl_hindsight_track_module: Arc, + rl_hindsight_track_fn: CudaFunction, + _rl_hindsight_inject_module: Arc, + rl_hindsight_inject_fn: CudaFunction, + _rl_hindsight_forward_module: Arc, + rl_hindsight_forward_fn: CudaFunction, + /// Per-step gather buffer for PER-sampled next-state argmax actions /// (Double-DQN online-Q argmax on `sampled_h_tp1`). Computed /// per-step inside `dqn_offpolicy_step` via `argmax_expected_q` — @@ -1449,6 +1468,28 @@ impl IntegratedTrainer { .load_function("rl_per_tree_rebuild") .context("load rl_per_tree_rebuild")?; + // Bidirectional HER kernels. + let rl_hindsight_track_module = ctx + .load_cubin(RL_HINDSIGHT_TRACK_CUBIN.to_vec()) + .context("load rl_hindsight_track cubin")?; + let rl_hindsight_track_fn = rl_hindsight_track_module + .load_function("rl_hindsight_track") + .context("load rl_hindsight_track")?; + let rl_hindsight_inject_module = ctx + .load_cubin(RL_HINDSIGHT_INJECT_CUBIN.to_vec()) + .context("load rl_hindsight_inject cubin")?; + let rl_hindsight_inject_fn = rl_hindsight_inject_module + .load_function("rl_hindsight_inject") + .context("load rl_hindsight_inject")?; + let rl_hindsight_forward_module = ctx + .load_cubin(RL_HINDSIGHT_FORWARD_CUBIN.to_vec()) + .context("load rl_hindsight_forward cubin")?; + let rl_hindsight_forward_fn = rl_hindsight_forward_module + .load_function("rl_hindsight_forward") + .context("load rl_hindsight_forward")?; + + let hindsight = crate::rl::gpu_hindsight::GpuHindsight::new(&stream, cfg.perception.n_batch)?; + // Per-batch PRNG state for the Thompson sampler. Seeded // deterministically from cfg.dqn_seed via ChaCha8 host RNG so // (cfg.dqn_seed, b_size) → identical Thompson draws across @@ -2112,6 +2153,13 @@ impl IntegratedTrainer { rl_per_update_priority_fn, _rl_per_tree_rebuild_module: rl_per_tree_rebuild_module, rl_per_tree_rebuild_fn, + hindsight, + _rl_hindsight_track_module: rl_hindsight_track_module, + rl_hindsight_track_fn, + _rl_hindsight_inject_module: rl_hindsight_inject_module, + rl_hindsight_inject_fn, + _rl_hindsight_forward_module: rl_hindsight_forward_module, + rl_hindsight_forward_fn, sampled_next_actions_d, td_per_sample_d, grad_h_t_combined_d, @@ -2241,7 +2289,7 @@ impl IntegratedTrainer { // (slot, value) pair — pure device write, no HtoD per // `feedback_no_htod_htoh_only_mapped_pinned`. { - let isv_constants: [(usize, f32); 83] = [ + let isv_constants: [(usize, f32); 86] = [ // Static seeds for the adaptive reward-clamp controller — // these are the initial values that // `rl_reward_clamp_controller` will replace once it @@ -2371,6 +2419,10 @@ impl IntegratedTrainer { // sentinel-zero) and available for runtime re-tuning via // diagnostic JSONL visibility. (crate::rl::isv_slots::RL_NOISY_SIGMA_INIT_INDEX, 0.5), + // Bidirectional HER ISV bootstrap — threshold, priority boost, lookahead. + (crate::rl::isv_slots::RL_HINDSIGHT_THRESHOLD_INDEX, 1.5), + (crate::rl::isv_slots::RL_HINDSIGHT_PRIORITY_BOOST_INDEX, 3.0), + (crate::rl::isv_slots::RL_HINDSIGHT_FORWARD_LOOKAHEAD_INDEX, 50.0), ]; let cfg_isv = LaunchConfig { grid_dim: (1, 1, 1), @@ -5642,6 +5694,39 @@ impl IntegratedTrainer { } } // end else (reward warmup / capture) + // HER: track mid price for backward hindsight — accumulate + // mid-price into ring buffer and update peak (max for long, + // min for short) while a position is open. Flat resets ring. + { + let pos_d_ref = lobsim.pos_d(); + let bid_px_d = lobsim.bid_px_d(); + let ask_px_d = lobsim.ask_px_d(); + let cfg = LaunchConfig { + grid_dim: (((b_size as u32) + 31) / 32, 1, 1), + block_dim: (32, 1, 1), + shared_mem_bytes: 0, + }; + let b_size_i = b_size as i32; + let pos_bytes_i = lobsim.pos_bytes() as i32; + let mut launch = self.stream.launch_builder(&self.rl_hindsight_track_fn); + launch + .arg(pos_d_ref) + .arg(bid_px_d) + .arg(ask_px_d) + .arg(&mut self.hindsight.mid_ring_d) + .arg(&mut self.hindsight.ring_write_idx_d) + .arg(&mut self.hindsight.peak_mid_d) + .arg(&mut self.hindsight.entry_mid_d) + .arg(&mut self.hindsight.position_dir_d) + .arg(&b_size_i) + .arg(&pos_bytes_i); + unsafe { + launch + .launch(cfg) + .context("step_with_lobsim: rl_hindsight_track")?; + } + } + // ── Step 7a (R7d): PER push + sample + gather. ──────────────── // Push CURRENT step's b_size transitions (one per batch) to // GPU PER buffer; sample b_size indices (priority^α from @@ -5749,6 +5834,84 @@ impl IntegratedTrainer { } } + // HER: backward inject — on trade close, if peak PnL >> actual + // PnL (threshold from ISV[549]), inject a synthetic "held to + // peak" transition at boosted priority (ISV[550]). + { + let b_size_i = b_size as i32; + let cap_i = self.gpu_replay.capacity as i32; + let cfg = LaunchConfig { + grid_dim: (1, 1, 1), + block_dim: (b_size as u32, 1, 1), + shared_mem_bytes: (2 * b_size * std::mem::size_of::()) as u32, + }; + // peak_mid_d appears twice: const read (arg 2) and mutable + // reset (arg 15). Same device pointer — extract raw to + // satisfy the borrow checker (same pattern as nstep_count_d + // in rl_per_push_flush). + let peak_mid_ptr = self.hindsight.peak_mid_d.raw_ptr(); + let mut launch = self.stream.launch_builder(&self.rl_hindsight_inject_fn); + launch + .arg(&self.dones_d) + .arg(&self.raw_rewards_d) + .arg(&peak_mid_ptr) + .arg(&self.hindsight.entry_mid_d) + .arg(&self.hindsight.position_dir_d) + .arg(self.perception.h_t_view()) + .arg(&self.isv_d) + .arg(&mut self.gpu_replay.h_t_d) + .arg(&mut self.gpu_replay.h_tp1_d) + .arg(&mut self.gpu_replay.scalars_d) + .arg(&mut self.gpu_replay.priority_tree_d) + .arg(&mut self.gpu_replay.write_head_d) + .arg(&mut self.gpu_replay.replay_len_d) + .arg(&mut self.gpu_replay.max_priority_d) + .arg(&mut self.hindsight.ring_write_idx_d) + .arg(&peak_mid_ptr) + .arg(&b_size_i) + .arg(&cap_i); + unsafe { + launch + .launch(cfg) + .context("step_with_lobsim: rl_hindsight_inject")?; + } + } + + // HER: forward — evaluate closed trades after lookahead window + // (ISV[552] steps). If holding would have been better, inject + // a synthetic "should-have-held-longer" transition. + { + let bid_px_d = lobsim.bid_px_d(); + let ask_px_d = lobsim.ask_px_d(); + let cfg = LaunchConfig { + grid_dim: (1, 1, 1), + block_dim: (256, 1, 1), // CLOSED_RING_SIZE + shared_mem_bytes: (2 * 256 * std::mem::size_of::()) as u32, + }; + let cap_i = self.gpu_replay.capacity as i32; + let mut launch = self.stream.launch_builder(&self.rl_hindsight_forward_fn); + launch + .arg(bid_px_d) + .arg(ask_px_d) + .arg(&self.isv_d) + .arg(&mut self.hindsight.closed_ring_d) + .arg(&self.hindsight.closed_h_t_d) + .arg(&self.hindsight.closed_step_d) + .arg(&mut self.gpu_replay.h_t_d) + .arg(&mut self.gpu_replay.h_tp1_d) + .arg(&mut self.gpu_replay.scalars_d) + .arg(&mut self.gpu_replay.priority_tree_d) + .arg(&mut self.gpu_replay.write_head_d) + .arg(&mut self.gpu_replay.replay_len_d) + .arg(&mut self.gpu_replay.max_priority_d) + .arg(&cap_i); + unsafe { + launch + .launch(cfg) + .context("step_with_lobsim: rl_hindsight_forward")?; + } + } + // ── Step 7b: K-loop training intensification driven by // `n_rollout_steps` (ISV[404]). Wires the previously-write-only // controller to actual training behavior.