From af940671bc488ddc2d90c4e5867a14165e797e86 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 22 Mar 2026 20:50:23 +0100 Subject: [PATCH] =?UTF-8?q?feat:=20reward=20config=20pipeline=20=E2=80=94?= =?UTF-8?q?=20DQNHyperparameters=20+=20TOML=20profiles?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add 7 composite reward fields to DQNHyperparameters: w_dsr, w_pnl, w_dd, w_idle, dd_threshold, loss_aversion, time_decay_rate. Add RewardSection to training_profile.rs with Option fields and apply_to() mapping. Add [reward] section to all 3 DQN TOML profiles (production, smoketest, hyperopt) with identical defaults. Remove hold_reward from ExperienceSection (replaced by w_idle). Add 7 reward search bounds to SearchSpaceSection and bound() match. Add 7 reward phase_fast defaults to PhaseFastSection. hold_penalty kept as deprecated field for hyperopt adapter compat (Task 4 will clean it up). Co-Authored-By: Claude Opus 4.6 (1M context) --- config/training/dqn-hyperopt.toml | 25 ++++++++++ config/training/dqn-production.toml | 10 +++- config/training/dqn-smoketest.toml | 9 ++++ crates/ml/src/trainers/dqn/config.rs | 27 +++++++++- crates/ml/src/training_profile.rs | 73 ++++++++++++++++++++++++++-- 5 files changed, 136 insertions(+), 8 deletions(-) diff --git a/config/training/dqn-hyperopt.toml b/config/training/dqn-hyperopt.toml index c8ee7f30f..e8ac9c399 100644 --- a/config/training/dqn-hyperopt.toml +++ b/config/training/dqn-hyperopt.toml @@ -61,6 +61,24 @@ gradient_accumulation_steps = [1, 1] # fixed at 1 for now # IQN dual-head iqn_lambda = [0.0, 2.0] +# Composite reward weights +w_dsr = [0.1, 2.0] +w_pnl = [0.0, 1.0] +w_dd = [0.0, 5.0] +w_idle = [0.0, 0.1] +dd_threshold = [0.01, 0.10] +loss_aversion = [1.0, 3.0] +time_decay_rate = [0.0001, 0.005] + +[reward] +w_dsr = 1.0 +w_pnl = 0.3 +w_dd = 1.0 +w_idle = 0.01 +dd_threshold = 0.02 +loss_aversion = 1.5 +time_decay_rate = 0.0005 + [fixed] use_branching = true # GPU pipeline requires branching use_double_dqn = true @@ -84,3 +102,10 @@ num_atoms = 11 branch_hidden_dim = 64 dueling_hidden_dim = 128 v_range = 20.0 +w_dsr = 1.0 +w_pnl = 0.3 +w_dd = 1.0 +w_idle = 0.01 +dd_threshold = 0.02 +loss_aversion = 1.5 +time_decay_rate = 0.0005 diff --git a/config/training/dqn-production.toml b/config/training/dqn-production.toml index 946dcac86..60f2c3ec6 100644 --- a/config/training/dqn-production.toml +++ b/config/training/dqn-production.toml @@ -63,4 +63,12 @@ min_loss_improvement_pct = 0.1 [experience] initial_capital = 100000.0 tx_cost_multiplier = 0.0001 -hold_reward = 0.001 + +[reward] +w_dsr = 1.0 +w_pnl = 0.3 +w_dd = 1.0 +w_idle = 0.01 +dd_threshold = 0.02 +loss_aversion = 1.5 +time_decay_rate = 0.0005 diff --git a/config/training/dqn-smoketest.toml b/config/training/dqn-smoketest.toml index e92cd58bd..946dd9530 100644 --- a/config/training/dqn-smoketest.toml +++ b/config/training/dqn-smoketest.toml @@ -27,3 +27,12 @@ min_epochs_before_stopping = 5 [experience] gpu_n_episodes = 16 gpu_timesteps_per_episode = 50 + +[reward] +w_dsr = 1.0 +w_pnl = 0.3 +w_dd = 1.0 +w_idle = 0.01 +dd_threshold = 0.02 +loss_aversion = 1.5 +time_decay_rate = 0.0005 diff --git a/crates/ml/src/trainers/dqn/config.rs b/crates/ml/src/trainers/dqn/config.rs index 4ae7d3caf..29bf05663 100644 --- a/crates/ml/src/trainers/dqn/config.rs +++ b/crates/ml/src/trainers/dqn/config.rs @@ -813,8 +813,24 @@ pub struct DQNHyperparameters { pub plateau_window: usize, /// Minimum epochs before early stopping can trigger (default: 50) pub min_epochs_before_stopping: usize, - /// Small negative penalty encourages action diversity (Bug #3 fix) + /// Deprecated: replaced by `w_idle` in composite reward. Kept for hyperopt adapter compat. pub hold_penalty: f64, + + // GPU composite reward weights (8-component composite in CUDA kernel) + /// DSR (Sharpe) weight for composite reward + pub w_dsr: f64, + /// Normalized PnL weight for composite reward + pub w_pnl: f64, + /// Drawdown penalty weight for composite reward + pub w_dd: f64, + /// Idle penalty weight for composite reward + pub w_idle: f64, + /// Drawdown tolerance before penalty kicks in (fraction, e.g. 0.02 = 2%) + pub dd_threshold: f64, + /// Asymmetric loss scaling factor (prospect theory, default 1.5) + pub loss_aversion: f64, + /// Position staleness rent per step held + pub time_decay_rate: f64, /// Use Huber loss instead of MSE (more robust to outliers) pub use_huber_loss: bool, /// Huber loss delta threshold (default: 1.0) @@ -1205,7 +1221,14 @@ impl DQNHyperparameters { min_loss_improvement_pct: 2.0, plateau_window: 30, min_epochs_before_stopping: 50, - hold_penalty: -0.001, + hold_penalty: -0.001, // deprecated: kept for hyperopt adapter compat + w_dsr: 1.0, + w_pnl: 0.3, + w_dd: 1.0, + w_idle: 0.01, + dd_threshold: 0.02, + loss_aversion: 1.5, + time_decay_rate: 0.0005, use_huber_loss: true, // Default: Huber loss enabled (more robust) huber_delta: 100.0, // BUG #12 FIX: Scale delta 100x for gradient explosion fix (was 1.0) use_double_dqn: true, // Default: Double DQN enabled (prevents overestimation bias) diff --git a/crates/ml/src/training_profile.rs b/crates/ml/src/training_profile.rs index 7ac726d17..eddd0cc80 100644 --- a/crates/ml/src/training_profile.rs +++ b/crates/ml/src/training_profile.rs @@ -138,6 +138,25 @@ pub struct EarlyStoppingSection { pub min_loss_improvement_pct: Option, } +/// GPU composite reward weights. +#[derive(Debug, Clone, Deserialize, Default)] +pub struct RewardSection { + /// DSR (Sharpe) weight + pub w_dsr: Option, + /// Normalized PnL weight + pub w_pnl: Option, + /// Drawdown penalty weight + pub w_dd: Option, + /// Idle penalty weight + pub w_idle: Option, + /// Drawdown tolerance before penalty (fraction) + pub dd_threshold: Option, + /// Asymmetric loss scaling factor (prospect theory) + pub loss_aversion: Option, + /// Position staleness rent per step + pub time_decay_rate: Option, +} + /// GPU experience collection parameters. #[derive(Debug, Clone, Deserialize, Default)] pub struct ExperienceSection { @@ -146,8 +165,6 @@ pub struct ExperienceSection { pub initial_capital: Option, /// Maps to `DQNHyperparameters::transaction_cost_multiplier`. pub tx_cost_multiplier: Option, - /// Maps to `DQNHyperparameters::hold_penalty`. - pub hold_reward: Option, } /// Walk-forward cross-validation window configuration. @@ -178,6 +195,7 @@ pub struct DqnTrainingProfile { pub risk: Option, pub early_stopping: Option, pub experience: Option, + pub reward: Option, pub walk_forward: Option, } @@ -219,6 +237,14 @@ pub struct SearchSpaceSection { pub branch_hidden_dim: Option<[f64; 2]>, pub gradient_accumulation_steps: Option<[f64; 2]>, pub iqn_lambda: Option<[f64; 2]>, + // Composite reward weights + pub w_dsr: Option<[f64; 2]>, + pub w_pnl: Option<[f64; 2]>, + pub w_dd: Option<[f64; 2]>, + pub w_idle: Option<[f64; 2]>, + pub dd_threshold: Option<[f64; 2]>, + pub loss_aversion: Option<[f64; 2]>, + pub time_decay_rate: Option<[f64; 2]>, } /// PSO optimizer configuration. @@ -241,6 +267,14 @@ pub struct PhaseFastSection { pub branch_hidden_dim: Option, pub dueling_hidden_dim: Option, pub v_range: Option, + // Composite reward weights (fixed in phase fast) + pub w_dsr: Option, + pub w_pnl: Option, + pub w_dd: Option, + pub w_idle: Option, + pub dd_threshold: Option, + pub loss_aversion: Option, + pub time_decay_rate: Option, } /// Two-phase hyperopt configuration. @@ -353,6 +387,13 @@ impl HyperoptProfile { "branch_hidden_dim" => ss.branch_hidden_dim, "gradient_accumulation_steps" => ss.gradient_accumulation_steps, "iqn_lambda" => ss.iqn_lambda, + "w_dsr" => ss.w_dsr, + "w_pnl" => ss.w_pnl, + "w_dd" => ss.w_dd, + "w_idle" => ss.w_idle, + "dd_threshold" => ss.dd_threshold, + "loss_aversion" => ss.loss_aversion, + "time_decay_rate" => ss.time_decay_rate, _ => None, }; match val { @@ -648,7 +689,6 @@ impl DqnTrainingProfile { // [experience] // TOML: tx_cost_multiplier → hp: transaction_cost_multiplier - // TOML: hold_reward → hp: hold_penalty // TOML: initial_capital → hp: initial_capital (f32 in hp, f64 in TOML) if let Some(ref ex) = self.experience { if let Some(v) = ex.gpu_n_episodes { @@ -663,8 +703,31 @@ impl DqnTrainingProfile { if let Some(v) = ex.tx_cost_multiplier { hp.transaction_cost_multiplier = v; } - if let Some(v) = ex.hold_reward { - hp.hold_penalty = v; + } + + // [reward] + // GPU composite reward weights — 7 fields map directly to DQNHyperparameters + if let Some(ref rw) = self.reward { + if let Some(v) = rw.w_dsr { + hp.w_dsr = v; + } + if let Some(v) = rw.w_pnl { + hp.w_pnl = v; + } + if let Some(v) = rw.w_dd { + hp.w_dd = v; + } + if let Some(v) = rw.w_idle { + hp.w_idle = v; + } + if let Some(v) = rw.dd_threshold { + hp.dd_threshold = v; + } + if let Some(v) = rw.loss_aversion { + hp.loss_aversion = v; + } + if let Some(v) = rw.time_decay_rate { + hp.time_decay_rate = v; } }