diff --git a/ml/src/hyperopt/adapters/continuous_ppo.rs b/ml/src/hyperopt/adapters/continuous_ppo.rs index ee2ae528f..9e8d4444f 100644 --- a/ml/src/hyperopt/adapters/continuous_ppo.rs +++ b/ml/src/hyperopt/adapters/continuous_ppo.rs @@ -43,7 +43,8 @@ use crate::ppo::continuous_ppo::{ ContinuousPPO, ContinuousPPOConfig, ContinuousTrajectory, ContinuousTrajectoryBatch, ContinuousTrajectoryStep, }; -use crate::ppo::continuous_policy::{ContinuousAction, ContinuousPolicyConfig}; +use crate::ppo::continuous_policy::ContinuousAction; +use crate::ppo::flow_policy::FlowPolicyConfig; use crate::ppo::gae::GAEConfig; use crate::MLError; @@ -121,26 +122,16 @@ impl ParameterSpace for ContinuousPPOParams { (0.001_f64.ln(), 0.1_f64.ln()), // entropy_coeff (log scale) (0.9, 0.99), // gae_lambda (linear) (0.95, 0.999), // gamma (linear) - ] - } - - fn integer_bounds() -> Vec<(i64, i64)> { - vec![ - (32, 256), // batch_size - (5, 15), // num_epochs - ] - } - - fn categorical_choices() -> Vec> { - vec![ - vec!["true".to_string(), "false".to_string()], // learnable_std + (32.0, 256.0), // batch_size (rounded to integer) + (5.0, 15.0), // num_epochs (rounded to integer) + (0.0, 1.0), // learnable_std (threshold at 0.5) ] } fn from_continuous(x: &[f64]) -> Result { - if x.len() != 9 { + if x.len() != 12 { return Err(MLError::ConfigError { - reason: format!("Expected 9 continuous parameters, got {}", x.len()), + reason: format!("Expected 12 continuous parameters, got {}", x.len()), }); } @@ -154,10 +145,9 @@ impl ParameterSpace for ContinuousPPOParams { entropy_coeff: x[6].exp(), gae_lambda: x[7].clamp(0.9, 0.99), gamma: x[8].clamp(0.95, 0.999), - // Default values for other fields (will be set by from_mixed) - learnable_std: true, - batch_size: 64, - num_epochs: 10, + batch_size: x[9].round().clamp(32.0, 256.0) as usize, + num_epochs: x[10].round().clamp(5.0, 15.0) as usize, + learnable_std: x[11] >= 0.5, }) } @@ -172,58 +162,12 @@ impl ParameterSpace for ContinuousPPOParams { self.entropy_coeff.ln(), self.gae_lambda, self.gamma, + self.batch_size as f64, + self.num_epochs as f64, + if self.learnable_std { 1.0 } else { 0.0 }, ] } - fn from_integer(x: &[i64]) -> Result { - if x.len() != 2 { - return Err(MLError::ConfigError { - reason: format!("Expected 2 integer parameters, got {}", x.len()), - }); - } - - Ok(Self { - batch_size: x[0].clamp(32, 256) as usize, - num_epochs: x[1].clamp(5, 15) as usize, - // Default values for other fields - ..Default::default() - }) - } - - fn to_integer(&self) -> Vec { - vec![self.batch_size as i64, self.num_epochs as i64] - } - - fn from_categorical(x: &[usize]) -> Result { - if x.is_empty() { - return Err(MLError::ConfigError { - reason: "Expected 1 categorical parameter, got 0".to_string(), - }); - } - - Ok(Self { - learnable_std: x[0] == 0, // 0 = "true", 1 = "false" - // Default values for other fields - ..Default::default() - }) - } - - fn to_categorical(&self) -> Vec { - vec![if self.learnable_std { 0 } else { 1 }] - } - - fn from_mixed(continuous: &[f64], integer: &[i64], categorical: &[usize]) -> Result { - let mut params = Self::from_continuous(continuous)?; - let int_params = Self::from_integer(integer)?; - let cat_params = Self::from_categorical(categorical)?; - - params.batch_size = int_params.batch_size; - params.num_epochs = int_params.num_epochs; - params.learnable_std = cat_params.learnable_std; - - Ok(params) - } - fn param_names() -> Vec<&'static str> { vec![ "policy_lr", @@ -443,14 +387,13 @@ impl HyperparameterOptimizable for ContinuousPPOTrainer { MLError::ModelError(format!("Failed to create training directories: {}", e)) })?; - // Create continuous PPO config - let policy_config = ContinuousPolicyConfig { + // Create continuous PPO config using FlowPolicyConfig + let policy_config = FlowPolicyConfig { state_dim: 54, - hidden_dims: vec![128, 64], - action_min: params.action_min as f32, - action_max: params.action_max as f32, - init_log_std: params.init_log_std as f32, - learnable_std: params.learnable_std, + action_dim: 1, + context_dim: 128, + num_layers: 4, + scale_clamp: 5.0, }; let ppo_config = ContinuousPPOConfig { @@ -465,6 +408,7 @@ impl HyperparameterOptimizable for ContinuousPPOTrainer { gae_config: GAEConfig { gamma: params.gamma as f32, lambda: params.gae_lambda as f32, + normalize_advantages: true, }, batch_size: params.batch_size, mini_batch_size: 64, @@ -652,9 +596,9 @@ impl ContinuousPPOTrainer { /// Compute maximum drawdown from episode returns fn compute_max_drawdown(&self, returns: &[f64]) -> f64 { - let mut cumulative = 0.0; - let mut peak = 0.0; - let mut max_drawdown = 0.0; + let mut cumulative: f64 = 0.0; + let mut peak: f64 = 0.0; + let mut max_drawdown: f64 = 0.0; for &ret in returns { cumulative += ret; @@ -689,14 +633,12 @@ mod tests { }; let continuous = params.to_continuous(); - let integer = params.to_integer(); - let categorical = params.to_categorical(); - - let recovered = ContinuousPPOParams::from_mixed(&continuous, &integer, &categorical).unwrap(); + let recovered = ContinuousPPOParams::from_continuous(&continuous).unwrap(); assert!((recovered.policy_lr - params.policy_lr).abs() < 1e-10); assert!((recovered.value_lr - params.value_lr).abs() < 1e-10); assert_eq!(recovered.batch_size, params.batch_size); + assert_eq!(recovered.num_epochs, params.num_epochs); assert_eq!(recovered.learnable_std, params.learnable_std); } diff --git a/ml/src/hyperopt/adapters/mod.rs b/ml/src/hyperopt/adapters/mod.rs index c39a13565..9eac2f407 100644 --- a/ml/src/hyperopt/adapters/mod.rs +++ b/ml/src/hyperopt/adapters/mod.rs @@ -49,7 +49,7 @@ // Active adapters (production-ready) pub mod async_data_loader; -// pub mod continuous_ppo; // TODO: Fix ParameterSpace trait implementation +pub mod continuous_ppo; pub mod dqn; pub mod mamba2; pub mod ppo; @@ -57,7 +57,7 @@ pub mod tft; // Re-export adapters for convenience pub use async_data_loader::AsyncDataLoader; -// pub use continuous_ppo::{ContinuousPPOMetrics, ContinuousPPOParams, ContinuousPPOTrainer}; +pub use continuous_ppo::{ContinuousPPOMetrics, ContinuousPPOParams, ContinuousPPOTrainer}; pub use dqn::{DQNMetrics, DQNParams, DQNTrainer}; pub use mamba2::{Mamba2Metrics, Mamba2Params, Mamba2Trainer}; pub use ppo::{PPOMetrics, PPOParams, PPOTrainer};