diff --git a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs index 83fb4179c..57981bac3 100644 --- a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs +++ b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs @@ -49,6 +49,10 @@ const MAX_EPISODES_LIMIT: usize = 0x8000; /// Absolute upper bound for validation — reject configs above this. const MAX_TIMESTEPS_LIMIT: usize = 1000; const PORTFOLIO_STATE_SIZE: usize = 8; +/// Portfolio stride for DQN experience kernels (12 floats per episode). +/// Matches PORTFOLIO_STRIDE in experience_kernels.cu. +/// Do NOT change PORTFOLIO_STATE_SIZE above — it's for the PPO/legacy path. +const PORTFOLIO_STRIDE: usize = 12; // --------------------------------------------------------------------------- // Pinned host memory for DtoH transfers @@ -143,8 +147,20 @@ pub struct ExperienceCollectorConfig { pub curiosity_scale: f32, /// Risk penalty weight (e.g. 0.1) pub risk_weight: f32, - /// Small positive reward for Flat action to break structural zero (e.g. 0.001) - pub hold_reward: f32, + /// DSR (Sharpe) weight for composite reward + pub w_dsr: f32, + /// Normalized PnL weight for composite reward + pub w_pnl: f32, + /// Drawdown penalty weight for composite reward + pub w_dd: f32, + /// Idle penalty weight for composite reward + pub w_idle: f32, + /// Drawdown tolerance before penalty (fraction, e.g. 0.02 = 2%) + pub dd_threshold: f32, + /// Asymmetric loss scaling factor (prospect theory, default 1.5) + pub loss_aversion: f32, + /// Position staleness rent per step + pub time_decay_rate: f32, /// Transaction cost multiplier (hyperopt-tunable, scales base 0.01% rate) pub tx_cost_multiplier: f32, /// UCB count-bonus coefficient for GPU action selection (0.0 = disabled) @@ -185,9 +201,7 @@ pub struct ExperienceCollectorConfig { pub fill_spread_capture_frac: f32, /// Whether fill simulation is enabled (requires median_spread > 0) pub fill_simulation_enabled: bool, - /// Enable Differential Sharpe Ratio reward shaping (Moody & Saffell 2001) - pub use_dsr: bool, - /// DSR EMA decay rate (e.g. 0.01 = ~100-step window) + /// DSR EMA decay rate (e.g. 0.01 = ~100-step window). DSR is always enabled. pub dsr_eta: f32, /// N-step returns lookahead (1 = standard TD, 3-5 typical for Rainbow DQN) pub n_steps: i32, @@ -213,7 +227,13 @@ impl Default for ExperienceCollectorConfig { diversity_scale: 1.0, curiosity_scale: 1.0, risk_weight: 0.1, - hold_reward: 0.001, + w_dsr: 1.0, + w_pnl: 0.3, + w_dd: 1.0, + w_idle: 0.01, + dd_threshold: 0.02, + loss_aversion: 1.5, + time_decay_rate: 0.0005, tx_cost_multiplier: 1.0, count_bonus_coefficient: 0.0, q_clip_min: -500.0, @@ -234,7 +254,6 @@ impl Default for ExperienceCollectorConfig { fill_spread_cost_frac: 0.50, fill_spread_capture_frac: 0.50, fill_simulation_enabled: false, - use_dsr: true, dsr_eta: 0.01, n_steps: 1, enable_action_masking: false, @@ -363,7 +382,7 @@ pub struct GpuExperienceCollector { target_branching: BranchingWeightSet, // Per-episode state buffers [alloc_episodes, ...] - portfolio_states: CudaSlice, // [alloc_episodes * 3] for new kernels (position, cash, portfolio_value) + portfolio_states: CudaSlice, // [alloc_episodes * PORTFOLIO_STRIDE] (12 floats per episode) rng_states: CudaSlice, // [alloc_episodes] episode_starts_buf: CudaSlice,// [alloc_episodes] @@ -570,18 +589,24 @@ impl GpuExperienceCollector { } // ── Step 6: Allocate per-episode buffers ──────────────────────── - // Portfolio states for the new kernels: [N, 3] = (position, cash, portfolio_value) + // Portfolio states for experience kernels: [N, PORTFOLIO_STRIDE=12] let mut portfolio_states = stream - .alloc_zeros::(alloc_episodes * 3) + .alloc_zeros::(alloc_episodes * PORTFOLIO_STRIDE) .map_err(|e| MLError::ModelError(format!("alloc portfolio_states: {e}")))?; - // Initialize portfolio: position=0, cash=initial_capital, portfolio_value=initial_capital - let mut portfolio_init = vec![0.0_f32; alloc_episodes * 3]; + // Initialize portfolio state (12 floats per episode) + let mut portfolio_init = vec![0.0_f32; alloc_episodes * PORTFOLIO_STRIDE]; for i in 0..alloc_episodes { - let off = i * 3; - portfolio_init[off] = 0.0; // position - portfolio_init[off + 1] = initial_capital; // cash - portfolio_init[off + 2] = initial_capital; // portfolio_value + let off = i * PORTFOLIO_STRIDE; + portfolio_init[off] = 0.0; // [0] position + portfolio_init[off + 1] = initial_capital; // [1] cash + portfolio_init[off + 2] = initial_capital; // [2] portfolio_value + // [3] dsr_A = 0.0, [4] dsr_B = 0.0 (already zero) + // [5] pnl_ema = 0.0, [6] pnl_var = 0.0 (already zero) + portfolio_init[off + 7] = initial_capital; // [7] peak_equity + // [8] flat_counter = 0.0 (already zero) + portfolio_init[off + 9] = initial_capital; // [9] prev_equity + // [10] hold_time = 0.0, [11] realized_pnl = 0.0 (already zero) } stream .memcpy_htod(&portfolio_init, &mut portfolio_states) @@ -996,10 +1021,17 @@ impl GpuExperienceCollector { )))?; } - // ── 5. Environment step ───────────────────────────────────── + // ── 5. Environment step (8-component composite reward) ────── let max_pos = config.max_position; let tx_cost = config.tx_cost_multiplier; - let hold_rw = config.hold_reward; + let rw_dsr = config.w_dsr; + let rw_pnl = config.w_pnl; + let rw_dd = config.w_dd; + let rw_idle = config.w_idle; + let rw_dd_thresh = config.dd_threshold; + let rw_loss_av = config.loss_aversion; + let rw_time_decay = config.time_decay_rate; + let rw_eta = config.dsr_eta; let l_i32 = timesteps as i32; let t_i32 = t as i32; unsafe { @@ -1017,7 +1049,16 @@ impl GpuExperienceCollector { .arg(&self.batch_states) .arg(&max_pos) .arg(&tx_cost) - .arg(&hold_rw) + .arg(&rw_dsr) + .arg(&rw_pnl) + .arg(&rw_dd) + .arg(&rw_idle) + .arg(&rw_dd_thresh) + .arg(&rw_loss_av) + .arg(&rw_time_decay) + .arg(&rw_eta) + .arg(market_features_buf) + .arg(&md) .arg(&l_i32) .arg(&n_i32) .arg(&total_bars) @@ -1162,13 +1203,18 @@ impl GpuExperienceCollector { _avg_spread: f32, _cash_reserve_pct: f32, ) -> Result<(), MLError> { - // Reset portfolio states [N, 3] - let mut portfolio_init = vec![0.0_f32; self.alloc_episodes * 3]; + // Reset portfolio states [N, PORTFOLIO_STRIDE=12] + let mut portfolio_init = vec![0.0_f32; self.alloc_episodes * PORTFOLIO_STRIDE]; for i in 0..self.alloc_episodes { - let off = i * 3; - portfolio_init[off] = 0.0; // position - portfolio_init[off + 1] = initial_capital; // cash - portfolio_init[off + 2] = initial_capital; // portfolio_value + let off = i * PORTFOLIO_STRIDE; + portfolio_init[off] = 0.0; // [0] position + portfolio_init[off + 1] = initial_capital; // [1] cash + portfolio_init[off + 2] = initial_capital; // [2] portfolio_value + // [3..6] dsr_A, dsr_B, pnl_ema, pnl_var = 0.0 (already zero) + portfolio_init[off + 7] = initial_capital; // [7] peak_equity + // [8] flat_counter = 0.0 (already zero) + portfolio_init[off + 9] = initial_capital; // [9] prev_equity + // [10] hold_time = 0.0, [11] realized_pnl = 0.0 (already zero) } self.stream .memcpy_htod(&portfolio_init, &mut self.portfolio_states) diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 7a5cb067a..906649a9f 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -742,7 +742,13 @@ impl DQNTrainer { max_position: self.max_position as f32, enable_action_masking: self.enable_action_masking, curiosity_scale: if self.curiosity_module.is_some() { 1.0 } else { 0.0 }, - hold_reward: self.hyperparams.hold_penalty.abs() as f32, + w_dsr: self.hyperparams.w_dsr as f32, + w_pnl: self.hyperparams.w_pnl as f32, + w_dd: self.hyperparams.w_dd as f32, + w_idle: self.hyperparams.w_idle as f32, + dd_threshold: self.hyperparams.dd_threshold as f32, + loss_aversion: self.hyperparams.loss_aversion as f32, + time_decay_rate: self.hyperparams.time_decay_rate as f32, tx_cost_multiplier: self.hyperparams.transaction_cost_multiplier as f32, count_bonus_coefficient: self.hyperparams.count_bonus_coefficient .unwrap_or(0.0) as f32, @@ -767,7 +773,6 @@ impl DQNTrainer { fill_spread_cost_frac: 0.50, fill_spread_capture_frac: 0.50, fill_simulation_enabled: self.median_vol > 0.0, - use_dsr: self.hyperparams.use_dsr, dsr_eta: self.hyperparams.dsr_eta as f32, n_steps: self.hyperparams.n_steps as i32, ..Default::default()