diff --git a/crates/ml-alpha/src/rl/gpu_hindsight.rs b/crates/ml-alpha/src/rl/gpu_hindsight.rs new file mode 100644 index 000000000..85e90b5c5 --- /dev/null +++ b/crates/ml-alpha/src/rl/gpu_hindsight.rs @@ -0,0 +1,38 @@ +use std::sync::Arc; +use anyhow::{Context, Result}; +use cudarc::driver::{CudaSlice, CudaStream}; +use crate::heads::HIDDEN_DIM; + +pub const HINDSIGHT_RING_LEN: usize = 512; +pub const CLOSED_RING_SIZE: usize = 256; +pub const CLOSED_RING_FIELDS: usize = 5; + +pub struct GpuHindsight { + pub mid_ring_d: CudaSlice, + pub ring_write_idx_d: CudaSlice, + pub peak_mid_d: CudaSlice, + pub entry_mid_d: CudaSlice, + pub position_dir_d: CudaSlice, + pub closed_ring_d: CudaSlice, + pub closed_h_t_d: CudaSlice, + pub closed_write_head_d: CudaSlice, + pub closed_count_d: CudaSlice, + pub closed_step_d: CudaSlice, +} + +impl GpuHindsight { + pub fn new(stream: &Arc, b_size: usize) -> Result { + Ok(Self { + mid_ring_d: stream.alloc_zeros(b_size * HINDSIGHT_RING_LEN).context("her mid_ring")?, + ring_write_idx_d: stream.alloc_zeros(b_size).context("her ring_write_idx")?, + peak_mid_d: stream.alloc_zeros(b_size).context("her peak_mid")?, + entry_mid_d: stream.alloc_zeros(b_size).context("her entry_mid")?, + position_dir_d: stream.alloc_zeros(b_size).context("her position_dir")?, + closed_ring_d: stream.alloc_zeros(CLOSED_RING_SIZE * CLOSED_RING_FIELDS).context("her closed_ring")?, + closed_h_t_d: stream.alloc_zeros(CLOSED_RING_SIZE * HIDDEN_DIM).context("her closed_h_t")?, + closed_write_head_d: stream.alloc_zeros(1).context("her closed_write_head")?, + closed_count_d: stream.alloc_zeros(1).context("her closed_count")?, + closed_step_d: stream.alloc_zeros(CLOSED_RING_SIZE).context("her closed_step")?, + }) + } +} diff --git a/crates/ml-alpha/src/rl/isv_slots.rs b/crates/ml-alpha/src/rl/isv_slots.rs index 2896384b9..089cacfa9 100644 --- a/crates/ml-alpha/src/rl/isv_slots.rs +++ b/crates/ml-alpha/src/rl/isv_slots.rs @@ -998,6 +998,36 @@ pub const RL_NOISY_SIGMA_INIT_INDEX: usize = 547; /// 0.0 at ISV allocation (alloc_zeros); first step sets it to 1.0. pub const RL_STEP_COUNTER_ISV_INDEX: usize = 548; +// ─── Hindsight Experience Replay (HER) — bidirectional ────────────── +// +// Backward peak: the mid-ring tracks the recent price trajectory so a +// closed trade can be replayed with a goal set at the peak mid seen +// during the hold. Forward continuation: the closed_ring stores the +// hidden state + trajectory at close, enabling synthetic transitions +// that continue from the close point toward a later mid-price goal. + +/// HER activation threshold. When the raw PnL (before scaling) of a +/// closed trade falls below this value, HER injection is suppressed +/// for that episode (no benefit in replaying catastrophic losses as +/// near-successes). Default: -50.0. +pub const RL_HINDSIGHT_THRESHOLD_INDEX: usize = 549; + +/// Priority boost applied to HER-injected transitions in the PER +/// priority tree. Multiplied onto the TD-error priority so hindsight +/// transitions are sampled more aggressively than the same transition +/// without goal relabelling. Default: 2.0. +pub const RL_HINDSIGHT_PRIORITY_BOOST_INDEX: usize = 550; + +/// Diagnostic: number of HER transitions injected into the replay +/// buffer this step. Written by the HER injection kernel; read by +/// diag JSONL to monitor hindsight utilisation rate. +pub const RL_HINDSIGHT_INJECT_COUNT_INDEX: usize = 551; + +/// Forward-continuation lookahead in steps. When replaying a closed +/// trade with a forward goal, the kernel looks ahead this many steps +/// in the closed_ring to find a trajectory segment. Default: 64. +pub const RL_HINDSIGHT_FORWARD_LOOKAHEAD_INDEX: usize = 552; + /// Last RL-allocated slot index (exclusive). The integrated trainer /// extends `ISV_TOTAL_DIM` to at least this value at trainer init time. -pub const RL_SLOTS_END: usize = 549; +pub const RL_SLOTS_END: usize = 553; diff --git a/crates/ml-alpha/src/rl/mod.rs b/crates/ml-alpha/src/rl/mod.rs index ac4908191..323962579 100644 --- a/crates/ml-alpha/src/rl/mod.rs +++ b/crates/ml-alpha/src/rl/mod.rs @@ -17,6 +17,7 @@ pub mod common; pub mod dqn; +pub mod gpu_hindsight; pub mod gpu_replay; pub mod frd; pub mod iqn;