diff --git a/crates/ml/src/benchmark/ppo_benchmark.rs b/crates/ml/src/benchmark/ppo_benchmark.rs index 587828835..862041d19 100644 --- a/crates/ml/src/benchmark/ppo_benchmark.rs +++ b/crates/ml/src/benchmark/ppo_benchmark.rs @@ -10,7 +10,7 @@ use std::path::PathBuf; use std::sync::Arc; use tokio::sync::Mutex; -use crate::dqn::TradingAction; +use crate::common::action::FactoredAction; use crate::ppo::ppo::{PPOConfig, PPO}; use crate::ppo::trajectories::{Trajectory, TrajectoryBatch, TrajectoryStep}; use crate::MLError; @@ -246,8 +246,13 @@ impl PpoBenchmarkRunner { let state = self.create_synthetic_state(traj_idx, step_idx); // Create synthetic action and reward + let hold_action = FactoredAction::new( + crate::common::action::ExposureLevel::Flat, + crate::common::action::OrderType::Market, + crate::common::action::Urgency::Normal, + ); let action = - TradingAction::from_int((step_idx % 3) as u8).unwrap_or(TradingAction::Hold); + FactoredAction::from_index(step_idx % 45).unwrap_or(hold_action); let reward = ((step_idx as f32 * 0.01).sin() * 0.1).clamp(-1.0, 1.0); let log_prob = -1.0; // Simplified let value = reward * 0.5; // Simplified value estimate diff --git a/crates/ml/src/common/action.rs b/crates/ml/src/common/action.rs index 702bf9178..21957570e 100644 --- a/crates/ml/src/common/action.rs +++ b/crates/ml/src/common/action.rs @@ -222,6 +222,23 @@ impl FactoredAction { self.urgency.urgency_weight() } + /// Create a FactoredAction from a legacy TradingAction. + /// Maps Buy -> Long100/Market/Normal, Sell -> Short100/Market/Normal, Hold -> Flat/Market/Normal. + pub fn from_legacy(action: crate::dqn::agent::TradingAction) -> Self { + use crate::dqn::agent::TradingAction; + match action { + TradingAction::Buy => { + Self::new(ExposureLevel::Long100, OrderType::Market, Urgency::Normal) + } + TradingAction::Sell => { + Self::new(ExposureLevel::Short100, OrderType::Market, Urgency::Normal) + } + TradingAction::Hold => { + Self::new(ExposureLevel::Flat, OrderType::Market, Urgency::Normal) + } + } + } + /// Convert FactoredAction to legacy TradingAction for reward calculation /// /// Maps exposure levels to simple Buy/Sell/Hold actions: @@ -337,6 +354,37 @@ mod tests { assert_eq!(hold.to_legacy_action(), TradingAction::Hold); } + #[test] + fn test_factored_action_from_legacy_round_trip() { + use crate::dqn::agent::TradingAction; + // from_legacy -> to_legacy_action should round-trip + assert_eq!( + FactoredAction::from_legacy(TradingAction::Buy).to_legacy_action(), + TradingAction::Buy + ); + assert_eq!( + FactoredAction::from_legacy(TradingAction::Sell).to_legacy_action(), + TradingAction::Sell + ); + assert_eq!( + FactoredAction::from_legacy(TradingAction::Hold).to_legacy_action(), + TradingAction::Hold + ); + // Verify specific indices + assert_eq!( + FactoredAction::from_legacy(TradingAction::Buy).to_index(), + 37 // Long100=4 * 9 + Market=0 * 3 + Normal=1 = 37 + ); + assert_eq!( + FactoredAction::from_legacy(TradingAction::Sell).to_index(), + 1 // Short100=0 * 9 + Market=0 * 3 + Normal=1 = 1 + ); + assert_eq!( + FactoredAction::from_legacy(TradingAction::Hold).to_index(), + 19 // Flat=2 * 9 + Market=0 * 3 + Normal=1 = 19 + ); + } + #[test] fn test_exposure_level_values() { assert_eq!(ExposureLevel::Short100 as usize, 0); diff --git a/crates/ml/src/hyperopt/adapters/ppo.rs b/crates/ml/src/hyperopt/adapters/ppo.rs index bc12b958d..044b9c189 100644 --- a/crates/ml/src/hyperopt/adapters/ppo.rs +++ b/crates/ml/src/hyperopt/adapters/ppo.rs @@ -38,7 +38,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use tracing::{info, warn}; -use crate::dqn::TradingAction; +use crate::common::action::FactoredAction; use crate::hyperopt::paths::TrainingPaths; use crate::hyperopt::traits::{HardwareBudget, HyperparameterOptimizable, ParameterSpace}; use crate::ppo::gae::GAEConfig; @@ -645,14 +645,18 @@ fn gpu_ppo_batch_to_trajectory_batch( .map(|c| c.get(..model_state_dim).unwrap_or(c).to_vec()) .collect(); - let actions: Vec = batch + let hold_action = FactoredAction::new( + crate::common::action::ExposureLevel::Flat, + crate::common::action::OrderType::Market, + crate::common::action::Urgency::Normal, + ); + let actions: Vec = batch .actions .iter() .take(total) - .map(|&a| match a.clamp(0, 2) { - 0 => TradingAction::Buy, - 1 => TradingAction::Sell, - _ => TradingAction::Hold, + .map(|&a| { + let idx = (a.clamp(0, 44)) as usize; + FactoredAction::from_index(idx).unwrap_or(hold_action) }) .collect(); @@ -1449,13 +1453,16 @@ impl PPOTrainer { let (action, log_prob, value) = match agent.act_with_log_prob(&state) { Ok((a, lp, v)) => (a, lp, v), Err(_) => { - // Fallback: random action with uniform log_prob - let a = match rng.gen_range(0..3) { - 0 => TradingAction::Buy, - 1 => TradingAction::Sell, - _ => TradingAction::Hold, - }; - (a, -(3.0_f32).ln(), 0.0_f32) + // Fallback: random factored action with uniform log_prob + let random_idx = rng.gen_range(0..45_usize); + let a = FactoredAction::from_index(random_idx).unwrap_or_else(|_| { + FactoredAction::new( + crate::common::action::ExposureLevel::Flat, + crate::common::action::OrderType::Market, + crate::common::action::Urgency::Normal, + ) + }); + (a, -(45.0_f32).ln(), 0.0_f32) } }; @@ -1473,10 +1480,12 @@ impl PPOTrainer { }; let total_cost = (self.tx_cost_bps * 0.0001 + spread_cost) as f32; - match action { - TradingAction::Buy => price_change as f32 - total_cost, - TradingAction::Sell => -price_change as f32 - total_cost, - TradingAction::Hold => 0.0, + // Use target_exposure for PnL: +1.0 (long), -1.0 (short), 0.0 (flat) + let exposure = action.target_exposure() as f32; + if exposure.abs() > f32::EPSILON { + exposure * price_change as f32 - total_cost + } else { + 0.0 // Flat position — no PnL } } else { 0.0 diff --git a/crates/ml/src/ppo/gae.rs b/crates/ml/src/ppo/gae.rs index 572c4f5fa..7fbb474d6 100644 --- a/crates/ml/src/ppo/gae.rs +++ b/crates/ml/src/ppo/gae.rs @@ -291,9 +291,14 @@ pub fn compute_advantages( #[cfg(test)] mod tests { use super::*; + use crate::common::action::FactoredAction; use crate::dqn::TradingAction; use crate::ppo::trajectories::{Trajectory, TrajectoryStep}; + /// Helper: build a FactoredAction from a legacy TradingAction for test brevity. + fn fa(ta: TradingAction) -> FactoredAction { + FactoredAction::from_legacy(ta) + } fn create_test_trajectory() -> Trajectory { let mut trajectory = Trajectory::new(); @@ -301,7 +306,7 @@ mod tests { // Add some test steps trajectory.add_step(TrajectoryStep::new( vec![1.0, 2.0], - TradingAction::Buy, + fa(TradingAction::Buy), -0.5, 10.0, // value 1.0, // reward @@ -310,7 +315,7 @@ mod tests { trajectory.add_step(TrajectoryStep::new( vec![2.0, 3.0], - TradingAction::Sell, + fa(TradingAction::Sell), -0.3, 8.0, // value 2.0, // reward @@ -319,7 +324,7 @@ mod tests { trajectory.add_step(TrajectoryStep::new( vec![3.0, 4.0], - TradingAction::Hold, + fa(TradingAction::Hold), -0.7, 5.0, // value 0.0, // reward diff --git a/crates/ml/src/ppo/ppo.rs b/crates/ml/src/ppo/ppo.rs index cb3c24d4b..d843c15b5 100644 --- a/crates/ml/src/ppo/ppo.rs +++ b/crates/ml/src/ppo/ppo.rs @@ -31,7 +31,7 @@ use crate::dqn::circuit_breaker::{CircuitBreaker, CircuitBreakerConfig}; use crate::dqn::portfolio_tracker::PortfolioTracker; use crate::dqn::xavier_init::linear_xavier; use crate::dqn::reward::RewardNormalizer; -use crate::dqn::TradingAction; +use crate::common::action::FactoredAction; use crate::MLError; /// Actor network variants supporting both MLP and LSTM architectures @@ -97,7 +97,7 @@ impl ActorNetwork { } /// Sample action from policy (MLP-only) - pub fn sample_action(&self, input: &Tensor) -> Result<(TradingAction, f32), MLError> { + pub fn sample_action(&self, input: &Tensor) -> Result<(FactoredAction, f32), MLError> { match self { ActorNetwork::MLP(network) => network.sample_action(input), ActorNetwork::LSTM(_) => Err(MLError::ModelError( @@ -467,7 +467,7 @@ impl PolicyNetwork { } /// Sample action from policy - pub fn sample_action(&self, input: &Tensor) -> Result<(TradingAction, f32), MLError> { + pub fn sample_action(&self, input: &Tensor) -> Result<(FactoredAction, f32), MLError> { let probs = self.action_probabilities(input)?; let probs_vec = probs .flatten_all()? @@ -482,8 +482,7 @@ impl PolicyNetwork { for (i, &prob) in probs_vec.iter().enumerate() { cumulative += prob; if sample <= cumulative { - let action = TradingAction::from_int(i as u8) - .ok_or_else(|| MLError::InvalidInput(format!("Invalid action index: {}", i)))?; + let action = FactoredAction::from_index(i)?; const EPSILON: f32 = 1e-8; let log_prob = (prob + EPSILON).ln(); return Ok((action, log_prob)); @@ -491,11 +490,10 @@ impl PolicyNetwork { } // Fallback to last action if rounding errors occur - let last_idx = probs_vec.len() - 1; - let action = TradingAction::from_int(last_idx as u8) - .ok_or_else(|| MLError::InvalidInput(format!("Invalid action index: {}", last_idx)))?; + let last_idx = probs_vec.len().saturating_sub(1); + let action = FactoredAction::from_index(last_idx)?; const EPSILON: f32 = 1e-8; - let log_prob = (probs_vec[last_idx] + EPSILON).ln(); + let log_prob = (probs_vec.get(last_idx).copied().unwrap_or(EPSILON) + EPSILON).ln(); Ok((action, log_prob)) } @@ -904,7 +902,7 @@ impl PPO { } /// Select action and get value estimate - pub fn act(&self, state: &[f32]) -> Result<(TradingAction, f32), MLError> { + pub fn act(&self, state: &[f32]) -> Result<(FactoredAction, f32), MLError> { let (action, _log_prob, value) = self.act_with_log_prob(state)?; Ok((action, value)) } @@ -914,7 +912,7 @@ impl PPO { /// Returns the **real** policy log-probability needed for PPO importance /// sampling. Use this instead of [`act`] when collecting trajectories /// for training. - pub fn act_with_log_prob(&self, state: &[f32]) -> Result<(TradingAction, f32, f32), MLError> { + pub fn act_with_log_prob(&self, state: &[f32]) -> Result<(FactoredAction, f32, f32), MLError> { let state_tensor = Tensor::from_vec( state.to_vec(), (1, self.config.state_dim), @@ -1293,7 +1291,11 @@ impl PPO { let log_probs_dist = candle_nn::ops::log_softmax(&logits, candle_core::D::Minus1) .map_err(|e| MLError::ModelError(format!("Log softmax failed: {}", e)))?; - let action_idx = sequence.actions[t].to_int() as i64; + let action_idx = sequence.actions.get(t) + .map(|a| a.to_index() as i64) + .ok_or_else(|| MLError::ModelError( + format!("Sequence action index {} out of bounds", t) + ))?; let action_tensor = Tensor::from_vec( vec![action_idx], (1, 1), @@ -1664,7 +1666,7 @@ impl PPO { /// "training_steps": 1234, /// "config": { /// "state_dim": 64, - /// "num_actions": 3, + /// "num_actions": 45, /// ... /// } /// } @@ -2014,16 +2016,15 @@ impl PPO { /// /// Use this for validation/evaluation where stochastic sampling would /// introduce noise into the performance metric. - pub fn greedy_action(&self, state: &[f32]) -> Result { + pub fn greedy_action(&self, state: &[f32]) -> Result { let probs = self.predict(state)?; let best_idx = probs .iter() .enumerate() .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)) .map(|(i, _)| i) - .unwrap_or(2); // default HOLD - TradingAction::from_int(best_idx as u8) - .ok_or_else(|| MLError::InvalidInput(format!("Invalid action index: {}", best_idx))) + .unwrap_or(19); // default Flat/Market/Normal (index 19) + FactoredAction::from_index(best_idx) } } diff --git a/crates/ml/src/ppo/trainable_adapter.rs b/crates/ml/src/ppo/trainable_adapter.rs index 50f1bdb2b..684956445 100644 --- a/crates/ml/src/ppo/trainable_adapter.rs +++ b/crates/ml/src/ppo/trainable_adapter.rs @@ -12,7 +12,7 @@ use tracing::info; use super::gae::compute_gae_single_trajectory; use super::ppo::{PPOConfig, PPO}; use super::trajectories::{Trajectory, TrajectoryBatch, TrajectoryStep}; -use crate::dqn::TradingAction; +use crate::common::action::FactoredAction; use crate::training::unified_trainer::{CheckpointMetadata, TrainingMetrics, UnifiedTrainable}; use crate::MLError; @@ -101,9 +101,7 @@ impl UnifiedPPO { .to_scalar::() .map_err(|e| MLError::TrainingError(format!("Failed to extract action: {}", e)))?; - let action = TradingAction::from_int(action_idx as u8).ok_or_else(|| { - MLError::InvalidInput(format!("Invalid action index: {}", action_idx)) - })?; + let action = FactoredAction::from_index(action_idx as usize)?; // Get log prob and value from current policy let state_unsqueezed = state_tensor.unsqueeze(0)?; @@ -408,7 +406,7 @@ mod tests { fn test_unified_ppo_creation() -> Result<(), MLError> { let config = PPOConfig { state_dim: 16, - num_actions: 3, + num_actions: 45, policy_hidden_dims: vec![32], value_hidden_dims: vec![32], policy_learning_rate: 3e-4, @@ -430,7 +428,7 @@ mod tests { fn test_unified_ppo_forward() -> Result<(), MLError> { let config = PPOConfig { state_dim: 16, - num_actions: 3, + num_actions: 45, ..Default::default() }; @@ -443,8 +441,8 @@ mod tests { // Forward pass let output = ppo.forward(&input)?; - // Check output shape - assert_eq!(output.dims(), &[1, 3]); + // Check output shape (45 factored actions) + assert_eq!(output.dims(), &[1, 45]); Ok(()) } diff --git a/crates/ml/src/ppo/trajectories.rs b/crates/ml/src/ppo/trajectories.rs index 55a77dabf..794e04a61 100644 --- a/crates/ml/src/ppo/trajectories.rs +++ b/crates/ml/src/ppo/trajectories.rs @@ -6,7 +6,7 @@ use candle_core::Tensor; use serde::{Deserialize, Serialize}; -use crate::dqn::TradingAction; +use crate::common::action::FactoredAction; use crate::MLError; /// Single step trajectory data @@ -15,7 +15,7 @@ pub struct TrajectoryStep { /// State at this step pub state: Vec, /// Action taken - pub action: TradingAction, + pub action: FactoredAction, /// Action probability (log probability) pub log_prob: f32, /// Value estimate at this state @@ -30,7 +30,7 @@ impl TrajectoryStep { /// Create new trajectory step pub fn new( state: Vec, - action: TradingAction, + action: FactoredAction, log_prob: f32, value: f32, reward: f32, @@ -81,7 +81,7 @@ impl Trajectory { } /// Get trajectory actions - pub fn get_actions(&self) -> Vec { + pub fn get_actions(&self) -> Vec { self.steps.iter().map(|step| step.action).collect() } @@ -113,7 +113,7 @@ impl Trajectory { } /// Extend a buffer with all actions (avoids intermediate `Vec` allocation). - pub fn extend_actions(&self, buf: &mut Vec) { + pub fn extend_actions(&self, buf: &mut Vec) { buf.extend(self.steps.iter().map(|s| s.action)); } @@ -189,7 +189,7 @@ pub struct TrajectoryBatch { /// GPU batch constructors should set this to `vec![]` (the fallback path handles it). pub states_flat: Vec, /// Flattened actions from all trajectories - pub actions: Vec, + pub actions: Vec, /// Flattened log probabilities pub log_probs: Vec, /// Flattened values @@ -307,7 +307,7 @@ impl TrajectoryBatch { let action_indices: Vec = self .actions .iter() - .map(|action| action.to_int() as u32) + .map(|action| action.to_index() as u32) .collect(); let actions_tensor = Tensor::from_vec(action_indices, batch_size, device).map_err(|e| { MLError::TrainingError(format!("Failed to create actions tensor: {}", e)) @@ -478,7 +478,7 @@ pub struct TrajectoryTensors { #[derive(Debug, Clone)] pub struct MiniBatch { pub states: Vec>, - pub actions: Vec, + pub actions: Vec, pub log_probs: Vec, pub values: Vec, pub advantages: Vec, @@ -492,7 +492,7 @@ pub struct MiniBatch { #[derive(Debug, Clone)] pub struct TrajectorySequence { pub states: Vec>, - pub actions: Vec, + pub actions: Vec, pub log_probs: Vec, pub values: Vec, pub rewards: Vec, @@ -539,7 +539,7 @@ impl MiniBatch { let action_indices: Vec = self .actions .iter() - .map(|action| action.to_int() as u32) + .map(|action| action.to_index() as u32) .collect(); let actions_tensor = Tensor::from_vec(action_indices, batch_size, device).map_err(|e| { MLError::TrainingError(format!("Failed to create actions tensor: {}", e)) @@ -610,6 +610,13 @@ where #[cfg(test)] mod tests { use super::*; + use crate::common::action::{ExposureLevel, OrderType, Urgency}; + use crate::dqn::TradingAction; + + /// Helper: build a FactoredAction from a legacy TradingAction for test brevity. + fn fa(ta: TradingAction) -> FactoredAction { + FactoredAction::from_legacy(ta) + } #[test] fn test_trajectory_creation() { @@ -619,7 +626,7 @@ mod tests { let step = TrajectoryStep::new( vec![1.0, 2.0, 3.0], - TradingAction::Buy, + fa(TradingAction::Buy), -0.5, 10.0, 1.0, @@ -638,7 +645,7 @@ mod tests { // Add some steps trajectory.add_step(TrajectoryStep::new( vec![0.0], - TradingAction::Buy, + fa(TradingAction::Buy), 0.0, 0.0, 1.0, @@ -646,7 +653,7 @@ mod tests { )); trajectory.add_step(TrajectoryStep::new( vec![1.0], - TradingAction::Sell, + fa(TradingAction::Sell), 0.0, 0.0, 2.0, @@ -654,7 +661,7 @@ mod tests { )); trajectory.add_step(TrajectoryStep::new( vec![2.0], - TradingAction::Hold, + fa(TradingAction::Hold), 0.0, 0.0, 3.0, @@ -678,7 +685,7 @@ mod tests { let mut traj1 = Trajectory::new(); traj1.add_step(TrajectoryStep::new( vec![1.0], - TradingAction::Buy, + fa(TradingAction::Buy), 0.0, 0.0, 1.0, @@ -686,7 +693,7 @@ mod tests { )); traj1.add_step(TrajectoryStep::new( vec![2.0], - TradingAction::Sell, + fa(TradingAction::Sell), 0.0, 0.0, 2.0, @@ -696,7 +703,7 @@ mod tests { let mut traj2 = Trajectory::new(); traj2.add_step(TrajectoryStep::new( vec![3.0], - TradingAction::Hold, + fa(TradingAction::Hold), 0.0, 0.0, 3.0, @@ -743,7 +750,8 @@ mod tests { let advantages = vec![1.0, 2.0, 3.0, 4.0, 5.0]; let returns = vec![0.0; 5]; let states = vec![vec![1.0]; 5]; - let actions = vec![TradingAction::Buy; 5]; + let buy_action = FactoredAction::new(ExposureLevel::Long100, OrderType::Market, Urgency::Normal); + let actions = vec![buy_action; 5]; let log_probs = vec![0.0; 5]; let values = vec![0.0; 5]; let dones = vec![false; 5]; @@ -768,7 +776,7 @@ mod tests { let mut trajectory = Trajectory::new(); trajectory.add_step(TrajectoryStep::new( vec![1.0, 2.0], - TradingAction::Buy, + fa(TradingAction::Buy), 0.0, 0.0, 0.0, @@ -776,7 +784,7 @@ mod tests { )); trajectory.add_step(TrajectoryStep::new( vec![3.0, 4.0], - TradingAction::Sell, + fa(TradingAction::Sell), 0.0, 0.0, 0.0, @@ -818,7 +826,7 @@ mod tests { let mut traj = Trajectory::new(); traj.add_step(TrajectoryStep::new( vec![1.0, 2.0, 3.0], - TradingAction::Buy, + fa(TradingAction::Buy), 0.0, 0.0, 1.0, @@ -826,7 +834,7 @@ mod tests { )); traj.add_step(TrajectoryStep::new( vec![4.0, 5.0, 6.0], - TradingAction::Sell, + fa(TradingAction::Sell), 0.0, 0.0, 2.0, diff --git a/crates/ml/src/trainers/ppo.rs b/crates/ml/src/trainers/ppo.rs index bd5920aeb..7a6f41772 100644 --- a/crates/ml/src/trainers/ppo.rs +++ b/crates/ml/src/trainers/ppo.rs @@ -16,7 +16,7 @@ use candle_core::Device; use tokio::sync::Mutex; use tracing::{debug, info, warn}; -use crate::dqn::TradingAction; +use crate::common::action::FactoredAction; use crate::gpu::DeviceConfig; use crate::gpu::capabilities::cached_capabilities; use crate::gpu::memory_profile::{self, resolve_batch_size}; @@ -824,7 +824,13 @@ impl PpoTrainer { // sample_action returns (action_idx, probs_vec) — reuse probs_vec, no duplicate sync let (action_idx, probs_vec) = self.sample_action(&action_probs)?; - let action = TradingAction::from_int(action_idx as u8).unwrap_or(TradingAction::Hold); + // Default: Flat/Market/Normal (Hold equivalent) + let hold_action = FactoredAction::new( + crate::common::action::ExposureLevel::Flat, + crate::common::action::OrderType::Market, + crate::common::action::Urgency::Normal, + ); + let action = FactoredAction::from_index(action_idx).unwrap_or(hold_action); // Get log probability from the already-synced probs_vec (safe indexing) const EPSILON: f32 = 1e-8; @@ -942,8 +948,13 @@ impl PpoTrainer { // sample_action returns (action_idx, probs_vec) — reuse probs_vec, no duplicate sync let (action_idx, probs_vec) = self.sample_action(&action_probs)?; - let action = - TradingAction::from_int(action_idx as u8).unwrap_or(TradingAction::Hold); + // Default: Flat/Market/Normal (Hold equivalent) + let hold_action = FactoredAction::new( + crate::common::action::ExposureLevel::Flat, + crate::common::action::OrderType::Market, + crate::common::action::Urgency::Normal, + ); + let action = FactoredAction::from_index(action_idx).unwrap_or(hold_action); // Get log probability from the already-synced probs_vec (safe indexing) const EPSILON: f32 = 1e-8; @@ -1526,12 +1537,18 @@ fn gpu_batch_to_trajectory_batch( .map(|c| c.to_vec()) .collect(); - let actions: Vec = batch + let hold_action = FactoredAction::new( + crate::common::action::ExposureLevel::Flat, + crate::common::action::OrderType::Market, + crate::common::action::Urgency::Normal, + ); + let actions: Vec = batch .actions .iter() .take(total) .map(|&a| { - TradingAction::from_int(a.clamp(0, 44) as u8).unwrap_or(TradingAction::Hold) + let idx = (a.clamp(0, 44)) as usize; + FactoredAction::from_index(idx).unwrap_or(hold_action) }) .collect(); diff --git a/crates/ml/src/validation/ppo_adapter.rs b/crates/ml/src/validation/ppo_adapter.rs index 6fc32a7e8..e599503da 100644 --- a/crates/ml/src/validation/ppo_adapter.rs +++ b/crates/ml/src/validation/ppo_adapter.rs @@ -13,7 +13,7 @@ use std::cell::RefCell; use candle_core::{Device, Tensor}; use rand::Rng; -use crate::dqn::TradingAction; +use crate::common::action::FactoredAction; use crate::ppo::gae::compute_gae; #[cfg(test)] use crate::ppo::gae::GAEConfig; @@ -128,10 +128,8 @@ impl ValidatableStrategy for PpoStrategy { /// Evaluate the MLP PPO on out-of-sample data. /// /// For each bar, computes action probabilities via the actor, takes the - /// argmax action, and maps it to PnL: - /// - Buy (0): `+return` - /// - Sell (1): `-return` - /// - Hold (2): `0` + /// argmax action, and maps it to PnL using the factored action's target + /// exposure (e.g., Long100 -> +1.0, Flat -> 0.0, Short100 -> -1.0). fn evaluate(&self, data: &TimeSeriesData) -> Result, MLError> { let num_returns = data.returns.len(); let mut pnl = Vec::with_capacity(num_returns); @@ -160,10 +158,10 @@ impl ValidatableStrategy for PpoStrategy { let action_idx = argmax(&probs_vec); let bar_return = data.returns.get(i).copied().unwrap_or(0.0); - let bar_pnl = match action_idx { - 0 => bar_return, // Buy - 1 => -bar_return, // Sell - _ => 0.0, // Hold + let bar_pnl = if let Ok(action) = FactoredAction::from_index(action_idx) { + action.target_exposure() * bar_return + } else { + 0.0 }; pnl.push(bar_pnl); @@ -385,10 +383,10 @@ impl ValidatableStrategy for PpoLstmStrategy { let action_idx = argmax(&probs_vec); let bar_return = data.returns.get(i).copied().unwrap_or(0.0); - let bar_pnl = match action_idx { - 0 => bar_return, // Buy - 1 => -bar_return, // Sell - _ => 0.0, // Hold + let bar_pnl = if let Ok(action) = FactoredAction::from_index(action_idx) { + action.target_exposure() * bar_return + } else { + 0.0 }; pnl.push(bar_pnl); @@ -433,8 +431,8 @@ fn argmax(values: &[f32]) -> usize { /// Sample an action from logits via softmax + categorical sampling. /// -/// Returns `(TradingAction, log_prob)`. -fn sample_action_from_logits(logits: &Tensor, _device: &Device) -> Result<(TradingAction, f32), MLError> { +/// Returns `(FactoredAction, log_prob)`. +fn sample_action_from_logits(logits: &Tensor, _device: &Device) -> Result<(FactoredAction, f32), MLError> { let probs = candle_nn::ops::softmax(logits, candle_core::D::Minus1) .map_err(|e| MLError::ModelError(format!("Softmax failed: {}", e)))?; let probs_vec = probs @@ -451,8 +449,7 @@ fn sample_action_from_logits(logits: &Tensor, _device: &Device) -> Result<(Tradi for (i, &prob) in probs_vec.iter().enumerate() { cumulative += prob; if sample <= cumulative { - let action = TradingAction::from_int(i as u8) - .ok_or_else(|| MLError::InvalidInput(format!("Invalid action index: {}", i)))?; + let action = FactoredAction::from_index(i)?; const EPSILON: f32 = 1e-8; let log_prob = (prob + EPSILON).ln(); return Ok((action, log_prob)); @@ -461,8 +458,7 @@ fn sample_action_from_logits(logits: &Tensor, _device: &Device) -> Result<(Tradi // Fallback to last valid action let last_idx = probs_vec.len().saturating_sub(1); - let action = TradingAction::from_int(last_idx as u8) - .ok_or_else(|| MLError::InvalidInput(format!("Invalid action index: {}", last_idx)))?; + let action = FactoredAction::from_index(last_idx)?; const EPSILON: f32 = 1e-8; let log_prob = (probs_vec.get(last_idx).copied().unwrap_or(EPSILON) + EPSILON).ln(); Ok((action, log_prob)) @@ -513,7 +509,7 @@ mod tests { fn make_mlp_config() -> PPOConfig { PPOConfig { state_dim: 10, - num_actions: 3, + num_actions: 45, policy_hidden_dims: vec![16, 8], value_hidden_dims: vec![16, 8], batch_size: 4,