diff --git a/ml/src/trainers/ppo.rs b/ml/src/trainers/ppo.rs index de4a4789a..1b667bc05 100644 --- a/ml/src/trainers/ppo.rs +++ b/ml/src/trainers/ppo.rs @@ -73,9 +73,9 @@ impl From for PPOConfig { state_dim: 225, // Wave C (201) + Wave D (24) = 225 num_actions: 3, // Buy, Sell, Hold policy_hidden_dims: vec![128, 64], - value_hidden_dims: vec![128, 64], - policy_learning_rate: params.learning_rate, - value_learning_rate: params.learning_rate, + value_hidden_dims: vec![512, 384, 256, 128, 64], + policy_learning_rate: 3e-4, // Fixed optimal rate for policy stability + value_learning_rate: 1e-3, // Higher rate for faster value network convergence clip_epsilon: params.clip_epsilon, value_loss_coeff: params.vf_coef, entropy_coeff: params.ent_coef, @@ -854,8 +854,8 @@ impl PpoTrainer { fn compute_reward_pnl(&self, action_idx: usize, log_return: f32, current_position: i8) -> f32 { // Base PnL reward from position and market movement let pnl_reward = match current_position { - 1 => log_return, // Long: profit when price goes up - -1 => -log_return, // Short: profit when price goes down + 1 => log_return * 1000.0, // Long: profit when price goes up (scaled to ±0.1 range) + -1 => -log_return * 1000.0, // Short: profit when price goes down (scaled to ±0.1 range) _ => 0.0, // Neutral: no exposure }; @@ -880,7 +880,7 @@ impl PpoTrainer { let sharpe_bonus = if pnl_reward > 0.0 { pnl_reward * 0.1 // 10% bonus for positive returns } else if pnl_reward < 0.0 { - pnl_reward * 1.5 // 50% penalty for negative returns (risk aversion) + pnl_reward * 0.1 // 10% penalty for negative returns (symmetric scaling) } else { 0.0 }; @@ -1006,8 +1006,8 @@ mod tests { let params = PpoHyperparameters::default(); let config: PPOConfig = params.into(); - assert_eq!(config.policy_learning_rate, 1e-4); // Updated: matches new default - assert_eq!(config.value_learning_rate, 1e-4); // Updated: matches new default + assert_eq!(config.policy_learning_rate, 3e-4); // Fixed optimal rate for policy stability + assert_eq!(config.value_learning_rate, 1e-3); // Higher rate for faster value convergence assert_eq!(config.clip_epsilon, 0.2); assert_eq!(config.value_loss_coeff, 1.0); // Updated: increased for value learning assert_eq!(config.entropy_coeff, 0.05); // Updated