gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: experience collector — PORTFOLIO_STRIDE=12, composite reward kernel args

Browse Source 
Update GpuExperienceCollector for 8-component composite reward:
- PORTFOLIO_STRIDE=12 const (separate from PPO's PORTFOLIO_STATE_SIZE=8)
- Portfolio buffer allocation: 3 -> 12 floats per episode
- peak_equity and prev_equity init to initial_capital (not zero)
- Kernel launch: replace hold_reward with 7 reward weights + eta +
  features buffer pointer + market_dim
- ExperienceCollectorConfig: replace hold_reward with 7 reward fields,
  remove use_dsr (DSR always enabled via w_dsr weight)
- Training loop: map DQNHyperparameters reward fields to config

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-22 20:50:53 +01:00
parent af940671bc
commit a2fde27d09
2 changed files with 78 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
crates/ml/src/cuda_pipeline/gpu_experience_collector.rs
View File

@@ -49,6 +49,10 @@ const MAX_EPISODES_LIMIT: usize = 0x8000;
/// Absolute upper bound for validation — reject configs above this.
const MAX_TIMESTEPS_LIMIT: usize = 1000;
const PORTFOLIO_STATE_SIZE: usize = 8;
/// Portfolio stride for DQN experience kernels (12 floats per episode).
/// Matches PORTFOLIO_STRIDE in experience_kernels.cu.
/// Do NOT change PORTFOLIO_STATE_SIZE above — it's for the PPO/legacy path.
const PORTFOLIO_STRIDE: usize = 12;
// ---------------------------------------------------------------------------
// Pinned host memory for DtoH transfers
@@ -143,8 +147,20 @@ pub struct ExperienceCollectorConfig {
pub curiosity_scale: f32,
/// Risk penalty weight (e.g. 0.1)
pub risk_weight: f32,
/// Small positive reward for Flat action to break structural zero (e.g. 0.001)
pub hold_reward: f32,
/// DSR (Sharpe) weight for composite reward
pub w_dsr: f32,
/// Normalized PnL weight for composite reward
pub w_pnl: f32,
/// Drawdown penalty weight for composite reward
pub w_dd: f32,
/// Idle penalty weight for composite reward
pub w_idle: f32,
/// Drawdown tolerance before penalty (fraction, e.g. 0.02 = 2%)
pub dd_threshold: f32,
/// Asymmetric loss scaling factor (prospect theory, default 1.5)
pub loss_aversion: f32,
/// Position staleness rent per step
pub time_decay_rate: f32,
/// Transaction cost multiplier (hyperopt-tunable, scales base 0.01% rate)
pub tx_cost_multiplier: f32,
/// UCB count-bonus coefficient for GPU action selection (0.0 = disabled)
@@ -185,9 +201,7 @@ pub struct ExperienceCollectorConfig {
pub fill_spread_capture_frac: f32,
/// Whether fill simulation is enabled (requires median_spread > 0)
pub fill_simulation_enabled: bool,
/// Enable Differential Sharpe Ratio reward shaping (Moody & Saffell 2001)
pub use_dsr: bool,
/// DSR EMA decay rate (e.g. 0.01 = ~100-step window)
/// DSR EMA decay rate (e.g. 0.01 = ~100-step window). DSR is always enabled.
pub dsr_eta: f32,
/// N-step returns lookahead (1 = standard TD, 3-5 typical for Rainbow DQN)
pub n_steps: i32,
@@ -213,7 +227,13 @@ impl Default for ExperienceCollectorConfig {
diversity_scale: 1.0,
curiosity_scale: 1.0,
risk_weight: 0.1,
hold_reward: 0.001,
w_dsr: 1.0,
w_pnl: 0.3,
w_dd: 1.0,
w_idle: 0.01,
dd_threshold: 0.02,
loss_aversion: 1.5,
time_decay_rate: 0.0005,
tx_cost_multiplier: 1.0,
count_bonus_coefficient: 0.0,
q_clip_min: -500.0,
@@ -234,7 +254,6 @@ impl Default for ExperienceCollectorConfig {
fill_spread_cost_frac: 0.50,
fill_spread_capture_frac: 0.50,
fill_simulation_enabled: false,
use_dsr: true,
dsr_eta: 0.01,
n_steps: 1,
enable_action_masking: false,
@@ -363,7 +382,7 @@ pub struct GpuExperienceCollector {
target_branching: BranchingWeightSet,
// Per-episode state buffers [alloc_episodes, ...]
portfolio_states: CudaSlice<f32>, // [alloc_episodes * 3] for new kernels (position, cash, portfolio_value)
portfolio_states: CudaSlice<f32>, // [alloc_episodes * PORTFOLIO_STRIDE] (12 floats per episode)
rng_states: CudaSlice<u32>, // [alloc_episodes]
episode_starts_buf: CudaSlice<i32>,// [alloc_episodes]
@@ -570,18 +589,24 @@ impl GpuExperienceCollector {
}
// ── Step 6: Allocate per-episode buffers ────────────────────────
// Portfolio states for the new kernels: [N, 3] = (position, cash, portfolio_value)
// Portfolio states for experience kernels: [N, PORTFOLIO_STRIDE=12]
let mut portfolio_states = stream
.alloc_zeros::<f32>(alloc_episodes * 3)
.alloc_zeros::<f32>(alloc_episodes * PORTFOLIO_STRIDE)
.map_err(|e| MLError::ModelError(format!("alloc portfolio_states: {e}")))?;
// Initialize portfolio: position=0, cash=initial_capital, portfolio_value=initial_capital
let mut portfolio_init = vec![0.0_f32; alloc_episodes * 3];
// Initialize portfolio state (12 floats per episode)
let mut portfolio_init = vec![0.0_f32; alloc_episodes * PORTFOLIO_STRIDE];
for i in 0..alloc_episodes {
let off = i * 3;
portfolio_init[off] = 0.0; // position
portfolio_init[off + 1] = initial_capital; // cash
portfolio_init[off + 2] = initial_capital; // portfolio_value
let off = i * PORTFOLIO_STRIDE;
portfolio_init[off] = 0.0; // [0] position
portfolio_init[off + 1] = initial_capital; // [1] cash
portfolio_init[off + 2] = initial_capital; // [2] portfolio_value
// [3] dsr_A = 0.0, [4] dsr_B = 0.0 (already zero)
// [5] pnl_ema = 0.0, [6] pnl_var = 0.0 (already zero)
portfolio_init[off + 7] = initial_capital; // [7] peak_equity
// [8] flat_counter = 0.0 (already zero)
portfolio_init[off + 9] = initial_capital; // [9] prev_equity
// [10] hold_time = 0.0, [11] realized_pnl = 0.0 (already zero)
}
stream
.memcpy_htod(&portfolio_init, &mut portfolio_states)
@@ -996,10 +1021,17 @@ impl GpuExperienceCollector {
)))?;
}
// ── 5. Environment step ─────────────────────────────────────
// ── 5. Environment step (8-component composite reward) ──────
let max_pos = config.max_position;
let tx_cost = config.tx_cost_multiplier;
let hold_rw = config.hold_reward;
let rw_dsr = config.w_dsr;
let rw_pnl = config.w_pnl;
let rw_dd = config.w_dd;
let rw_idle = config.w_idle;
let rw_dd_thresh = config.dd_threshold;
let rw_loss_av = config.loss_aversion;
let rw_time_decay = config.time_decay_rate;
let rw_eta = config.dsr_eta;
let l_i32 = timesteps as i32;
let t_i32 = t as i32;
unsafe {
@@ -1017,7 +1049,16 @@ impl GpuExperienceCollector {
.arg(&self.batch_states)
.arg(&max_pos)
.arg(&tx_cost)
.arg(&hold_rw)
.arg(&rw_dsr)
.arg(&rw_pnl)
.arg(&rw_dd)
.arg(&rw_idle)
.arg(&rw_dd_thresh)
.arg(&rw_loss_av)
.arg(&rw_time_decay)
.arg(&rw_eta)
.arg(market_features_buf)
.arg(&md)
.arg(&l_i32)
.arg(&n_i32)
.arg(&total_bars)
@@ -1162,13 +1203,18 @@ impl GpuExperienceCollector {
_avg_spread: f32,
_cash_reserve_pct: f32,
) -> Result<(), MLError> {
// Reset portfolio states [N, 3]
let mut portfolio_init = vec![0.0_f32; self.alloc_episodes * 3];
// Reset portfolio states [N, PORTFOLIO_STRIDE=12]
let mut portfolio_init = vec![0.0_f32; self.alloc_episodes * PORTFOLIO_STRIDE];
for i in 0..self.alloc_episodes {
let off = i * 3;
portfolio_init[off] = 0.0; // position
portfolio_init[off + 1] = initial_capital; // cash
portfolio_init[off + 2] = initial_capital; // portfolio_value
let off = i * PORTFOLIO_STRIDE;
portfolio_init[off] = 0.0; // [0] position
portfolio_init[off + 1] = initial_capital; // [1] cash
portfolio_init[off + 2] = initial_capital; // [2] portfolio_value
// [3..6] dsr_A, dsr_B, pnl_ema, pnl_var = 0.0 (already zero)
portfolio_init[off + 7] = initial_capital; // [7] peak_equity
// [8] flat_counter = 0.0 (already zero)
portfolio_init[off + 9] = initial_capital; // [9] prev_equity
// [10] hold_time = 0.0, [11] realized_pnl = 0.0 (already zero)
}
self.stream
.memcpy_htod(&portfolio_init, &mut self.portfolio_states)

9
crates/ml/src/trainers/dqn/trainer/training_loop.rs
View File

@@ -742,7 +742,13 @@ impl DQNTrainer {
max_position: self.max_position as f32,
enable_action_masking: self.enable_action_masking,
curiosity_scale: if self.curiosity_module.is_some() { 1.0 } else { 0.0 },
hold_reward: self.hyperparams.hold_penalty.abs() as f32,
w_dsr: self.hyperparams.w_dsr as f32,
w_pnl: self.hyperparams.w_pnl as f32,
w_dd: self.hyperparams.w_dd as f32,
w_idle: self.hyperparams.w_idle as f32,
dd_threshold: self.hyperparams.dd_threshold as f32,
loss_aversion: self.hyperparams.loss_aversion as f32,
time_decay_rate: self.hyperparams.time_decay_rate as f32,
tx_cost_multiplier: self.hyperparams.transaction_cost_multiplier as f32,
count_bonus_coefficient: self.hyperparams.count_bonus_coefficient
.unwrap_or(0.0) as f32,
@@ -767,7 +773,6 @@ impl DQNTrainer {
fill_spread_cost_frac: 0.50,
fill_spread_capture_frac: 0.50,
fill_simulation_enabled: self.median_vol > 0.0,
use_dsr: self.hyperparams.use_dsr,
dsr_eta: self.hyperparams.dsr_eta as f32,
n_steps: self.hyperparams.n_steps as i32,
..Default::default()