gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(rl): wire bidirectional HER — track + inject + forward in step pipeline

Browse Source 
Backward HER: on trade close, injects synthetic with peak PnL if
peak > 1.5× actual. Forward HER: evaluates closed trades after 50
steps, injects if holding would have been better.

ISV bootstrap: threshold=1.5, priority_boost=3.0, lookahead=50.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-05-26 08:48:11 +02:00
parent 029f2956f1
commit 4fc7d63185
2 changed files with 167 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
crates/ml-alpha/build.rs
View File

@@ -106,6 +106,9 @@ const KERNELS: &[&str] = &[
"rl_per_sample", // GPU PER: stratified proportional sampling via sum-tree walk + gather
"rl_per_update_priority", // GPU PER: write |TD|^α to tree leaves + block-wide max reduction
"rl_per_tree_rebuild", // GPU PER: bottom-up parallel sum-tree rebuild (no atomics)
"rl_hindsight_track", // HER Phase 1: per-step mid-price ring + peak tracking for backward hindsight
"rl_hindsight_inject", // HER Phase 2: backward inject — synthetic replay push on done if peak >> actual
"rl_hindsight_forward", // HER Phase 3: forward continuation — evaluates closed trades after lookahead
"rl_increment_step", // device-resident step counter bump (ISV[548] += 1.0); graph-safe prereq — removes scalar current_step from all downstream kernel args
"rl_fused_controllers", // fused kernel: 10 RL ISV controllers in one launch (gamma, tau, ppo_clip, entropy_coef, rollout_steps, per_alpha, reward_scale, ppo_ratio_clamp, gate_threshold, q_distill_lambda) — saves 9 kernel launch overheads (~40-80μs/step)
];

165
crates/ml-alpha/src/trainer/integrated.rs
View File

@@ -293,6 +293,16 @@ const RL_PER_UPDATE_PRIORITY_CUBIN: &[u8] =
const RL_PER_TREE_REBUILD_CUBIN: &[u8] =
include_bytes!(concat!(env!("OUT_DIR"), "/rl_per_tree_rebuild.cubin"));
// Bidirectional HER kernels — track (mid-price ring + peak), backward
// inject (synthetic push on done if peak >> actual), forward (evaluate
// closed trades after lookahead window).
const RL_HINDSIGHT_TRACK_CUBIN: &[u8] =
include_bytes!(concat!(env!("OUT_DIR"), "/rl_hindsight_track.cubin"));
const RL_HINDSIGHT_INJECT_CUBIN: &[u8] =
include_bytes!(concat!(env!("OUT_DIR"), "/rl_hindsight_inject.cubin"));
const RL_HINDSIGHT_FORWARD_CUBIN: &[u8] =
include_bytes!(concat!(env!("OUT_DIR"), "/rl_hindsight_forward.cubin"));
/// Per-head LR controller Wiener-α target. The controller kernel floors
/// this at 0.4 per `pearl_wiener_alpha_floor_for_nonstationary` so the
/// effective blend rate stays ≥ 0.4 regardless of this seed value. Kept
@@ -802,6 +812,15 @@ pub struct IntegratedTrainer {
_rl_per_tree_rebuild_module: Arc<CudaModule>,
rl_per_tree_rebuild_fn: CudaFunction,
// ── Bidirectional HER kernel handles ───────────────────────────
pub hindsight: crate::rl::gpu_hindsight::GpuHindsight,
_rl_hindsight_track_module: Arc<CudaModule>,
rl_hindsight_track_fn: CudaFunction,
_rl_hindsight_inject_module: Arc<CudaModule>,
rl_hindsight_inject_fn: CudaFunction,
_rl_hindsight_forward_module: Arc<CudaModule>,
rl_hindsight_forward_fn: CudaFunction,
/// Per-step gather buffer for PER-sampled next-state argmax actions
/// (Double-DQN online-Q argmax on `sampled_h_tp1`). Computed
/// per-step inside `dqn_offpolicy_step` via `argmax_expected_q` —
@@ -1449,6 +1468,28 @@ impl IntegratedTrainer {
.load_function("rl_per_tree_rebuild")
.context("load rl_per_tree_rebuild")?;
// Bidirectional HER kernels.
let rl_hindsight_track_module = ctx
.load_cubin(RL_HINDSIGHT_TRACK_CUBIN.to_vec())
.context("load rl_hindsight_track cubin")?;
let rl_hindsight_track_fn = rl_hindsight_track_module
.load_function("rl_hindsight_track")
.context("load rl_hindsight_track")?;
let rl_hindsight_inject_module = ctx
.load_cubin(RL_HINDSIGHT_INJECT_CUBIN.to_vec())
.context("load rl_hindsight_inject cubin")?;
let rl_hindsight_inject_fn = rl_hindsight_inject_module
.load_function("rl_hindsight_inject")
.context("load rl_hindsight_inject")?;
let rl_hindsight_forward_module = ctx
.load_cubin(RL_HINDSIGHT_FORWARD_CUBIN.to_vec())
.context("load rl_hindsight_forward cubin")?;
let rl_hindsight_forward_fn = rl_hindsight_forward_module
.load_function("rl_hindsight_forward")
.context("load rl_hindsight_forward")?;
let hindsight = crate::rl::gpu_hindsight::GpuHindsight::new(&stream, cfg.perception.n_batch)?;
// Per-batch PRNG state for the Thompson sampler. Seeded
// deterministically from cfg.dqn_seed via ChaCha8 host RNG so
// (cfg.dqn_seed, b_size) → identical Thompson draws across
@@ -2112,6 +2153,13 @@ impl IntegratedTrainer {
rl_per_update_priority_fn,
_rl_per_tree_rebuild_module: rl_per_tree_rebuild_module,
rl_per_tree_rebuild_fn,
hindsight,
_rl_hindsight_track_module: rl_hindsight_track_module,
rl_hindsight_track_fn,
_rl_hindsight_inject_module: rl_hindsight_inject_module,
rl_hindsight_inject_fn,
_rl_hindsight_forward_module: rl_hindsight_forward_module,
rl_hindsight_forward_fn,
sampled_next_actions_d,
td_per_sample_d,
grad_h_t_combined_d,
@@ -2241,7 +2289,7 @@ impl IntegratedTrainer {
// (slot, value) pair — pure device write, no HtoD per
// `feedback_no_htod_htoh_only_mapped_pinned`.
{
let isv_constants: [(usize, f32); 83] = [
let isv_constants: [(usize, f32); 86] = [
// Static seeds for the adaptive reward-clamp controller —
// these are the initial values that
// `rl_reward_clamp_controller` will replace once it
@@ -2371,6 +2419,10 @@ impl IntegratedTrainer {
// sentinel-zero) and available for runtime re-tuning via
// diagnostic JSONL visibility.
(crate::rl::isv_slots::RL_NOISY_SIGMA_INIT_INDEX, 0.5),
// Bidirectional HER ISV bootstrap — threshold, priority boost, lookahead.
(crate::rl::isv_slots::RL_HINDSIGHT_THRESHOLD_INDEX, 1.5),
(crate::rl::isv_slots::RL_HINDSIGHT_PRIORITY_BOOST_INDEX, 3.0),
(crate::rl::isv_slots::RL_HINDSIGHT_FORWARD_LOOKAHEAD_INDEX, 50.0),
];
let cfg_isv = LaunchConfig {
grid_dim: (1, 1, 1),
@@ -5642,6 +5694,39 @@ impl IntegratedTrainer {
}
} // end else (reward warmup / capture)
// HER: track mid price for backward hindsight — accumulate
// mid-price into ring buffer and update peak (max for long,
// min for short) while a position is open. Flat resets ring.
{
let pos_d_ref = lobsim.pos_d();
let bid_px_d = lobsim.bid_px_d();
let ask_px_d = lobsim.ask_px_d();
let cfg = LaunchConfig {
grid_dim: (((b_size as u32) + 31) / 32, 1, 1),
block_dim: (32, 1, 1),
shared_mem_bytes: 0,
};
let b_size_i = b_size as i32;
let pos_bytes_i = lobsim.pos_bytes() as i32;
let mut launch = self.stream.launch_builder(&self.rl_hindsight_track_fn);
launch
.arg(pos_d_ref)
.arg(bid_px_d)
.arg(ask_px_d)
.arg(&mut self.hindsight.mid_ring_d)
.arg(&mut self.hindsight.ring_write_idx_d)
.arg(&mut self.hindsight.peak_mid_d)
.arg(&mut self.hindsight.entry_mid_d)
.arg(&mut self.hindsight.position_dir_d)
.arg(&b_size_i)
.arg(&pos_bytes_i);
unsafe {
launch
.launch(cfg)
.context("step_with_lobsim: rl_hindsight_track")?;
}
}
// ── Step 7a (R7d): PER push + sample + gather. ────────────────
// Push CURRENT step's b_size transitions (one per batch) to
// GPU PER buffer; sample b_size indices (priority^α from
@@ -5749,6 +5834,84 @@ impl IntegratedTrainer {
}
}
// HER: backward inject — on trade close, if peak PnL >> actual
// PnL (threshold from ISV[549]), inject a synthetic "held to
// peak" transition at boosted priority (ISV[550]).
{
let b_size_i = b_size as i32;
let cap_i = self.gpu_replay.capacity as i32;
let cfg = LaunchConfig {
grid_dim: (1, 1, 1),
block_dim: (b_size as u32, 1, 1),
shared_mem_bytes: (2 * b_size * std::mem::size_of::<i32>()) as u32,
};
// peak_mid_d appears twice: const read (arg 2) and mutable
// reset (arg 15). Same device pointer — extract raw to
// satisfy the borrow checker (same pattern as nstep_count_d
// in rl_per_push_flush).
let peak_mid_ptr = self.hindsight.peak_mid_d.raw_ptr();
let mut launch = self.stream.launch_builder(&self.rl_hindsight_inject_fn);
launch
.arg(&self.dones_d)
.arg(&self.raw_rewards_d)
.arg(&peak_mid_ptr)
.arg(&self.hindsight.entry_mid_d)
.arg(&self.hindsight.position_dir_d)
.arg(self.perception.h_t_view())
.arg(&self.isv_d)
.arg(&mut self.gpu_replay.h_t_d)
.arg(&mut self.gpu_replay.h_tp1_d)
.arg(&mut self.gpu_replay.scalars_d)
.arg(&mut self.gpu_replay.priority_tree_d)
.arg(&mut self.gpu_replay.write_head_d)
.arg(&mut self.gpu_replay.replay_len_d)
.arg(&mut self.gpu_replay.max_priority_d)
.arg(&mut self.hindsight.ring_write_idx_d)
.arg(&peak_mid_ptr)
.arg(&b_size_i)
.arg(&cap_i);
unsafe {
launch
.launch(cfg)
.context("step_with_lobsim: rl_hindsight_inject")?;
}
}
// HER: forward — evaluate closed trades after lookahead window
// (ISV[552] steps). If holding would have been better, inject
// a synthetic "should-have-held-longer" transition.
{
let bid_px_d = lobsim.bid_px_d();
let ask_px_d = lobsim.ask_px_d();
let cfg = LaunchConfig {
grid_dim: (1, 1, 1),
block_dim: (256, 1, 1), // CLOSED_RING_SIZE
shared_mem_bytes: (2 * 256 * std::mem::size_of::<i32>()) as u32,
};
let cap_i = self.gpu_replay.capacity as i32;
let mut launch = self.stream.launch_builder(&self.rl_hindsight_forward_fn);
launch
.arg(bid_px_d)
.arg(ask_px_d)
.arg(&self.isv_d)
.arg(&mut self.hindsight.closed_ring_d)
.arg(&self.hindsight.closed_h_t_d)
.arg(&self.hindsight.closed_step_d)
.arg(&mut self.gpu_replay.h_t_d)
.arg(&mut self.gpu_replay.h_tp1_d)
.arg(&mut self.gpu_replay.scalars_d)
.arg(&mut self.gpu_replay.priority_tree_d)
.arg(&mut self.gpu_replay.write_head_d)
.arg(&mut self.gpu_replay.replay_len_d)
.arg(&mut self.gpu_replay.max_priority_d)
.arg(&cap_i);
unsafe {
launch
.launch(cfg)
.context("step_with_lobsim: rl_hindsight_forward")?;
}
}
// ── Step 7b: K-loop training intensification driven by
// `n_rollout_steps` (ISV[404]). Wires the previously-write-only
// controller to actual training behavior.