gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(ml): wire PPO to 45 factored actions via FactoredAction

Browse Source 
sample_action(), act(), act_with_log_prob(), greedy_action() now return
FactoredAction instead of TradingAction. Fixes the architectural disconnect
where num_actions=45 output neurons were sampled through a 3-action bottleneck.

TrajectoryStep.action and TrajectoryBatch.actions now use FactoredAction.
Added FactoredAction::from_legacy() for backward compatibility in tests.

Updated all PPO consumers: trainers/ppo.rs, hyperopt/adapters/ppo.rs,
validation/ppo_adapter.rs, benchmark/ppo_benchmark.rs.

2487 tests pass, 0 clippy warnings.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-02 14:46:50 +01:00
parent f12336f16f
commit 512535076f
9 changed files with 182 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
crates/ml/src/benchmark/ppo_benchmark.rs
View File

@@ -10,7 +10,7 @@ use std::path::PathBuf;
use std::sync::Arc;
use tokio::sync::Mutex;
use crate::dqn::TradingAction;
use crate::common::action::FactoredAction;
use crate::ppo::ppo::{PPOConfig, PPO};
use crate::ppo::trajectories::{Trajectory, TrajectoryBatch, TrajectoryStep};
use crate::MLError;
@@ -246,8 +246,13 @@ impl PpoBenchmarkRunner {
let state = self.create_synthetic_state(traj_idx, step_idx);
// Create synthetic action and reward
let hold_action = FactoredAction::new(
crate::common::action::ExposureLevel::Flat,
crate::common::action::OrderType::Market,
crate::common::action::Urgency::Normal,
);
let action =
TradingAction::from_int((step_idx % 3) as u8).unwrap_or(TradingAction::Hold);
FactoredAction::from_index(step_idx % 45).unwrap_or(hold_action);
let reward = ((step_idx as f32 * 0.01).sin() * 0.1).clamp(-1.0, 1.0);
let log_prob = -1.0; // Simplified
let value = reward * 0.5; // Simplified value estimate

48
crates/ml/src/common/action.rs
View File

@@ -222,6 +222,23 @@ impl FactoredAction {
self.urgency.urgency_weight()
}
/// Create a FactoredAction from a legacy TradingAction.
/// Maps Buy -> Long100/Market/Normal, Sell -> Short100/Market/Normal, Hold -> Flat/Market/Normal.
pub fn from_legacy(action: crate::dqn::agent::TradingAction) -> Self {
use crate::dqn::agent::TradingAction;
match action {
TradingAction::Buy => {
Self::new(ExposureLevel::Long100, OrderType::Market, Urgency::Normal)
}
TradingAction::Sell => {
Self::new(ExposureLevel::Short100, OrderType::Market, Urgency::Normal)
}
TradingAction::Hold => {
Self::new(ExposureLevel::Flat, OrderType::Market, Urgency::Normal)
}
}
}
/// Convert FactoredAction to legacy TradingAction for reward calculation
///
/// Maps exposure levels to simple Buy/Sell/Hold actions:
@@ -337,6 +354,37 @@ mod tests {
assert_eq!(hold.to_legacy_action(), TradingAction::Hold);
}
#[test]
fn test_factored_action_from_legacy_round_trip() {
use crate::dqn::agent::TradingAction;
// from_legacy -> to_legacy_action should round-trip
assert_eq!(
FactoredAction::from_legacy(TradingAction::Buy).to_legacy_action(),
TradingAction::Buy
);
assert_eq!(
FactoredAction::from_legacy(TradingAction::Sell).to_legacy_action(),
TradingAction::Sell
);
assert_eq!(
FactoredAction::from_legacy(TradingAction::Hold).to_legacy_action(),
TradingAction::Hold
);
// Verify specific indices
assert_eq!(
FactoredAction::from_legacy(TradingAction::Buy).to_index(),
37 // Long100=4 * 9 + Market=0 * 3 + Normal=1 = 37
);
assert_eq!(
FactoredAction::from_legacy(TradingAction::Sell).to_index(),
1 // Short100=0 * 9 + Market=0 * 3 + Normal=1 = 1
);
assert_eq!(
FactoredAction::from_legacy(TradingAction::Hold).to_index(),
19 // Flat=2 * 9 + Market=0 * 3 + Normal=1 = 19
);
}
#[test]
fn test_exposure_level_values() {
assert_eq!(ExposureLevel::Short100 as usize, 0);

43
crates/ml/src/hyperopt/adapters/ppo.rs
View File

@@ -38,7 +38,7 @@ use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use tracing::{info, warn};
use crate::dqn::TradingAction;
use crate::common::action::FactoredAction;
use crate::hyperopt::paths::TrainingPaths;
use crate::hyperopt::traits::{HardwareBudget, HyperparameterOptimizable, ParameterSpace};
use crate::ppo::gae::GAEConfig;
@@ -645,14 +645,18 @@ fn gpu_ppo_batch_to_trajectory_batch(
.map(|c| c.get(..model_state_dim).unwrap_or(c).to_vec())
.collect();
let actions: Vec<TradingAction> = batch
let hold_action = FactoredAction::new(
crate::common::action::ExposureLevel::Flat,
crate::common::action::OrderType::Market,
crate::common::action::Urgency::Normal,
);
let actions: Vec<FactoredAction> = batch
.actions
.iter()
.take(total)
.map(|&a| match a.clamp(0, 2) {
0 => TradingAction::Buy,
1 => TradingAction::Sell,
_ => TradingAction::Hold,
.map(|&a| {
let idx = (a.clamp(0, 44)) as usize;
FactoredAction::from_index(idx).unwrap_or(hold_action)
})
.collect();
@@ -1449,13 +1453,16 @@ impl PPOTrainer {
let (action, log_prob, value) = match agent.act_with_log_prob(&state) {
Ok((a, lp, v)) => (a, lp, v),
Err(_) => {
// Fallback: random action with uniform log_prob
let a = match rng.gen_range(0..3) {
0 => TradingAction::Buy,
1 => TradingAction::Sell,
_ => TradingAction::Hold,
};
(a, -(3.0_f32).ln(), 0.0_f32)
// Fallback: random factored action with uniform log_prob
let random_idx = rng.gen_range(0..45_usize);
let a = FactoredAction::from_index(random_idx).unwrap_or_else(|_| {
FactoredAction::new(
crate::common::action::ExposureLevel::Flat,
crate::common::action::OrderType::Market,
crate::common::action::Urgency::Normal,
)
});
(a, -(45.0_f32).ln(), 0.0_f32)
}
};
@@ -1473,10 +1480,12 @@ impl PPOTrainer {
};
let total_cost = (self.tx_cost_bps * 0.0001 + spread_cost) as f32;
match action {
TradingAction::Buy => price_change as f32 - total_cost,
TradingAction::Sell => -price_change as f32 - total_cost,
TradingAction::Hold => 0.0,
// Use target_exposure for PnL: +1.0 (long), -1.0 (short), 0.0 (flat)
let exposure = action.target_exposure() as f32;
if exposure.abs() > f32::EPSILON {
exposure * price_change as f32 - total_cost
} else {
0.0 // Flat position — no PnL
}
} else {
0.0

11
crates/ml/src/ppo/gae.rs
View File

@@ -291,9 +291,14 @@ pub fn compute_advantages(
#[cfg(test)]
mod tests {
use super::*;
use crate::common::action::FactoredAction;
use crate::dqn::TradingAction;
use crate::ppo::trajectories::{Trajectory, TrajectoryStep};
/// Helper: build a FactoredAction from a legacy TradingAction for test brevity.
fn fa(ta: TradingAction) -> FactoredAction {
FactoredAction::from_legacy(ta)
}
fn create_test_trajectory() -> Trajectory {
let mut trajectory = Trajectory::new();
@@ -301,7 +306,7 @@ mod tests {
// Add some test steps
trajectory.add_step(TrajectoryStep::new(
vec![1.0, 2.0],
TradingAction::Buy,
fa(TradingAction::Buy),
-0.5,
10.0, // value
1.0, // reward
@@ -310,7 +315,7 @@ mod tests {
trajectory.add_step(TrajectoryStep::new(
vec![2.0, 3.0],
TradingAction::Sell,
fa(TradingAction::Sell),
-0.3,
8.0, // value
2.0, // reward
@@ -319,7 +324,7 @@ mod tests {
trajectory.add_step(TrajectoryStep::new(
vec![3.0, 4.0],
TradingAction::Hold,
fa(TradingAction::Hold),
-0.7,
5.0, // value
0.0, // reward

35
crates/ml/src/ppo/ppo.rs
View File

@@ -31,7 +31,7 @@ use crate::dqn::circuit_breaker::{CircuitBreaker, CircuitBreakerConfig};
use crate::dqn::portfolio_tracker::PortfolioTracker;
use crate::dqn::xavier_init::linear_xavier;
use crate::dqn::reward::RewardNormalizer;
use crate::dqn::TradingAction;
use crate::common::action::FactoredAction;
use crate::MLError;
/// Actor network variants supporting both MLP and LSTM architectures
@@ -97,7 +97,7 @@ impl ActorNetwork {
}
/// Sample action from policy (MLP-only)
pub fn sample_action(&self, input: &Tensor) -> Result<(TradingAction, f32), MLError> {
pub fn sample_action(&self, input: &Tensor) -> Result<(FactoredAction, f32), MLError> {
match self {
ActorNetwork::MLP(network) => network.sample_action(input),
ActorNetwork::LSTM(_) => Err(MLError::ModelError(
@@ -467,7 +467,7 @@ impl PolicyNetwork {
}
/// Sample action from policy
pub fn sample_action(&self, input: &Tensor) -> Result<(TradingAction, f32), MLError> {
pub fn sample_action(&self, input: &Tensor) -> Result<(FactoredAction, f32), MLError> {
let probs = self.action_probabilities(input)?;
let probs_vec = probs
.flatten_all()?
@@ -482,8 +482,7 @@ impl PolicyNetwork {
for (i, &prob) in probs_vec.iter().enumerate() {
cumulative += prob;
if sample <= cumulative {
let action = TradingAction::from_int(i as u8)
.ok_or_else(|| MLError::InvalidInput(format!("Invalid action index: {}", i)))?;
let action = FactoredAction::from_index(i)?;
const EPSILON: f32 = 1e-8;
let log_prob = (prob + EPSILON).ln();
return Ok((action, log_prob));
@@ -491,11 +490,10 @@ impl PolicyNetwork {
}
// Fallback to last action if rounding errors occur
let last_idx = probs_vec.len() - 1;
let action = TradingAction::from_int(last_idx as u8)
.ok_or_else(|| MLError::InvalidInput(format!("Invalid action index: {}", last_idx)))?;
let last_idx = probs_vec.len().saturating_sub(1);
let action = FactoredAction::from_index(last_idx)?;
const EPSILON: f32 = 1e-8;
let log_prob = (probs_vec[last_idx] + EPSILON).ln();
let log_prob = (probs_vec.get(last_idx).copied().unwrap_or(EPSILON) + EPSILON).ln();
Ok((action, log_prob))
}
@@ -904,7 +902,7 @@ impl PPO {
}
/// Select action and get value estimate
pub fn act(&self, state: &[f32]) -> Result<(TradingAction, f32), MLError> {
pub fn act(&self, state: &[f32]) -> Result<(FactoredAction, f32), MLError> {
let (action, _log_prob, value) = self.act_with_log_prob(state)?;
Ok((action, value))
}
@@ -914,7 +912,7 @@ impl PPO {
/// Returns the **real** policy log-probability needed for PPO importance
/// sampling. Use this instead of [`act`] when collecting trajectories
/// for training.
pub fn act_with_log_prob(&self, state: &[f32]) -> Result<(TradingAction, f32, f32), MLError> {
pub fn act_with_log_prob(&self, state: &[f32]) -> Result<(FactoredAction, f32, f32), MLError> {
let state_tensor = Tensor::from_vec(
state.to_vec(),
(1, self.config.state_dim),
@@ -1293,7 +1291,11 @@ impl PPO {
let log_probs_dist = candle_nn::ops::log_softmax(&logits, candle_core::D::Minus1)
.map_err(|e| MLError::ModelError(format!("Log softmax failed: {}", e)))?;
let action_idx = sequence.actions[t].to_int() as i64;
let action_idx = sequence.actions.get(t)
.map(|a| a.to_index() as i64)
.ok_or_else(|| MLError::ModelError(
format!("Sequence action index {} out of bounds", t)
))?;
let action_tensor = Tensor::from_vec(
vec![action_idx],
(1, 1),
@@ -1664,7 +1666,7 @@ impl PPO {
/// "training_steps": 1234,
/// "config": {
/// "state_dim": 64,
/// "num_actions": 3,
/// "num_actions": 45,
/// ...
/// }
/// }
@@ -2014,16 +2016,15 @@ impl PPO {
///
/// Use this for validation/evaluation where stochastic sampling would
/// introduce noise into the performance metric.
pub fn greedy_action(&self, state: &[f32]) -> Result<TradingAction, MLError> {
pub fn greedy_action(&self, state: &[f32]) -> Result<FactoredAction, MLError> {
let probs = self.predict(state)?;
let best_idx = probs
.iter()
.enumerate()
.max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
.map(|(i, _)| i)
.unwrap_or(2); // default HOLD
TradingAction::from_int(best_idx as u8)
.ok_or_else(|| MLError::InvalidInput(format!("Invalid action index: {}", best_idx)))
.unwrap_or(19); // default Flat/Market/Normal (index 19)
FactoredAction::from_index(best_idx)
}
}

14
crates/ml/src/ppo/trainable_adapter.rs
View File

@@ -12,7 +12,7 @@ use tracing::info;
use super::gae::compute_gae_single_trajectory;
use super::ppo::{PPOConfig, PPO};
use super::trajectories::{Trajectory, TrajectoryBatch, TrajectoryStep};
use crate::dqn::TradingAction;
use crate::common::action::FactoredAction;
use crate::training::unified_trainer::{CheckpointMetadata, TrainingMetrics, UnifiedTrainable};
use crate::MLError;
@@ -101,9 +101,7 @@ impl UnifiedPPO {
.to_scalar::<u32>()
.map_err(|e| MLError::TrainingError(format!("Failed to extract action: {}", e)))?;
let action = TradingAction::from_int(action_idx as u8).ok_or_else(|| {
MLError::InvalidInput(format!("Invalid action index: {}", action_idx))
})?;
let action = FactoredAction::from_index(action_idx as usize)?;
// Get log prob and value from current policy
let state_unsqueezed = state_tensor.unsqueeze(0)?;
@@ -408,7 +406,7 @@ mod tests {
fn test_unified_ppo_creation() -> Result<(), MLError> {
let config = PPOConfig {
state_dim: 16,
num_actions: 3,
num_actions: 45,
policy_hidden_dims: vec![32],
value_hidden_dims: vec![32],
policy_learning_rate: 3e-4,
@@ -430,7 +428,7 @@ mod tests {
fn test_unified_ppo_forward() -> Result<(), MLError> {
let config = PPOConfig {
state_dim: 16,
num_actions: 3,
num_actions: 45,
..Default::default()
};
@@ -443,8 +441,8 @@ mod tests {
// Forward pass
let output = ppo.forward(&input)?;
// Check output shape
assert_eq!(output.dims(), &[1, 3]);
// Check output shape (45 factored actions)
assert_eq!(output.dims(), &[1, 45]);
Ok(())
}

52
crates/ml/src/ppo/trajectories.rs
View File

@@ -6,7 +6,7 @@
use candle_core::Tensor;
use serde::{Deserialize, Serialize};
use crate::dqn::TradingAction;
use crate::common::action::FactoredAction;
use crate::MLError;
/// Single step trajectory data
@@ -15,7 +15,7 @@ pub struct TrajectoryStep {
/// State at this step
pub state: Vec<f32>,
/// Action taken
pub action: TradingAction,
pub action: FactoredAction,
/// Action probability (log probability)
pub log_prob: f32,
/// Value estimate at this state
@@ -30,7 +30,7 @@ impl TrajectoryStep {
/// Create new trajectory step
pub fn new(
state: Vec<f32>,
action: TradingAction,
action: FactoredAction,
log_prob: f32,
value: f32,
reward: f32,
@@ -81,7 +81,7 @@ impl Trajectory {
}
/// Get trajectory actions
pub fn get_actions(&self) -> Vec<TradingAction> {
pub fn get_actions(&self) -> Vec<FactoredAction> {
self.steps.iter().map(|step| step.action).collect()
}
@@ -113,7 +113,7 @@ impl Trajectory {
}
/// Extend a buffer with all actions (avoids intermediate `Vec` allocation).
pub fn extend_actions(&self, buf: &mut Vec<TradingAction>) {
pub fn extend_actions(&self, buf: &mut Vec<FactoredAction>) {
buf.extend(self.steps.iter().map(|s| s.action));
}
@@ -189,7 +189,7 @@ pub struct TrajectoryBatch {
/// GPU batch constructors should set this to `vec![]` (the fallback path handles it).
pub states_flat: Vec<f32>,
/// Flattened actions from all trajectories
pub actions: Vec<TradingAction>,
pub actions: Vec<FactoredAction>,
/// Flattened log probabilities
pub log_probs: Vec<f32>,
/// Flattened values
@@ -307,7 +307,7 @@ impl TrajectoryBatch {
let action_indices: Vec<u32> = self
.actions
.iter()
.map(|action| action.to_int() as u32)
.map(|action| action.to_index() as u32)
.collect();
let actions_tensor = Tensor::from_vec(action_indices, batch_size, device).map_err(|e| {
MLError::TrainingError(format!("Failed to create actions tensor: {}", e))
@@ -478,7 +478,7 @@ pub struct TrajectoryTensors {
#[derive(Debug, Clone)]
pub struct MiniBatch {
pub states: Vec<Vec<f32>>,
pub actions: Vec<TradingAction>,
pub actions: Vec<FactoredAction>,
pub log_probs: Vec<f32>,
pub values: Vec<f32>,
pub advantages: Vec<f32>,
@@ -492,7 +492,7 @@ pub struct MiniBatch {
#[derive(Debug, Clone)]
pub struct TrajectorySequence {
pub states: Vec<Vec<f32>>,
pub actions: Vec<TradingAction>,
pub actions: Vec<FactoredAction>,
pub log_probs: Vec<f32>,
pub values: Vec<f32>,
pub rewards: Vec<f32>,
@@ -539,7 +539,7 @@ impl MiniBatch {
let action_indices: Vec<u32> = self
.actions
.iter()
.map(|action| action.to_int() as u32)
.map(|action| action.to_index() as u32)
.collect();
let actions_tensor = Tensor::from_vec(action_indices, batch_size, device).map_err(|e| {
MLError::TrainingError(format!("Failed to create actions tensor: {}", e))
@@ -610,6 +610,13 @@ where
#[cfg(test)]
mod tests {
use super::*;
use crate::common::action::{ExposureLevel, OrderType, Urgency};
use crate::dqn::TradingAction;
/// Helper: build a FactoredAction from a legacy TradingAction for test brevity.
fn fa(ta: TradingAction) -> FactoredAction {
FactoredAction::from_legacy(ta)
}
#[test]
fn test_trajectory_creation() {
@@ -619,7 +626,7 @@ mod tests {
let step = TrajectoryStep::new(
vec![1.0, 2.0, 3.0],
TradingAction::Buy,
fa(TradingAction::Buy),
-0.5,
10.0,
1.0,
@@ -638,7 +645,7 @@ mod tests {
// Add some steps
trajectory.add_step(TrajectoryStep::new(
vec![0.0],
TradingAction::Buy,
fa(TradingAction::Buy),
0.0,
0.0,
1.0,
@@ -646,7 +653,7 @@ mod tests {
));
trajectory.add_step(TrajectoryStep::new(
vec![1.0],
TradingAction::Sell,
fa(TradingAction::Sell),
0.0,
0.0,
2.0,
@@ -654,7 +661,7 @@ mod tests {
));
trajectory.add_step(TrajectoryStep::new(
vec![2.0],
TradingAction::Hold,
fa(TradingAction::Hold),
0.0,
0.0,
3.0,
@@ -678,7 +685,7 @@ mod tests {
let mut traj1 = Trajectory::new();
traj1.add_step(TrajectoryStep::new(
vec![1.0],
TradingAction::Buy,
fa(TradingAction::Buy),
0.0,
0.0,
1.0,
@@ -686,7 +693,7 @@ mod tests {
));
traj1.add_step(TrajectoryStep::new(
vec![2.0],
TradingAction::Sell,
fa(TradingAction::Sell),
0.0,
0.0,
2.0,
@@ -696,7 +703,7 @@ mod tests {
let mut traj2 = Trajectory::new();
traj2.add_step(TrajectoryStep::new(
vec![3.0],
TradingAction::Hold,
fa(TradingAction::Hold),
0.0,
0.0,
3.0,
@@ -743,7 +750,8 @@ mod tests {
let advantages = vec![1.0, 2.0, 3.0, 4.0, 5.0];
let returns = vec![0.0; 5];
let states = vec![vec![1.0]; 5];
let actions = vec![TradingAction::Buy; 5];
let buy_action = FactoredAction::new(ExposureLevel::Long100, OrderType::Market, Urgency::Normal);
let actions = vec![buy_action; 5];
let log_probs = vec![0.0; 5];
let values = vec![0.0; 5];
let dones = vec![false; 5];
@@ -768,7 +776,7 @@ mod tests {
let mut trajectory = Trajectory::new();
trajectory.add_step(TrajectoryStep::new(
vec![1.0, 2.0],
TradingAction::Buy,
fa(TradingAction::Buy),
0.0,
0.0,
0.0,
@@ -776,7 +784,7 @@ mod tests {
));
trajectory.add_step(TrajectoryStep::new(
vec![3.0, 4.0],
TradingAction::Sell,
fa(TradingAction::Sell),
0.0,
0.0,
0.0,
@@ -818,7 +826,7 @@ mod tests {
let mut traj = Trajectory::new();
traj.add_step(TrajectoryStep::new(
vec![1.0, 2.0, 3.0],
TradingAction::Buy,
fa(TradingAction::Buy),
0.0,
0.0,
1.0,
@@ -826,7 +834,7 @@ mod tests {
));
traj.add_step(TrajectoryStep::new(
vec![4.0, 5.0, 6.0],
TradingAction::Sell,
fa(TradingAction::Sell),
0.0,
0.0,
2.0,

29
crates/ml/src/trainers/ppo.rs
View File

@@ -16,7 +16,7 @@ use candle_core::Device;
use tokio::sync::Mutex;
use tracing::{debug, info, warn};
use crate::dqn::TradingAction;
use crate::common::action::FactoredAction;
use crate::gpu::DeviceConfig;
use crate::gpu::capabilities::cached_capabilities;
use crate::gpu::memory_profile::{self, resolve_batch_size};
@@ -824,7 +824,13 @@ impl PpoTrainer {
// sample_action returns (action_idx, probs_vec) — reuse probs_vec, no duplicate sync
let (action_idx, probs_vec) = self.sample_action(&action_probs)?;
let action = TradingAction::from_int(action_idx as u8).unwrap_or(TradingAction::Hold);
// Default: Flat/Market/Normal (Hold equivalent)
let hold_action = FactoredAction::new(
crate::common::action::ExposureLevel::Flat,
crate::common::action::OrderType::Market,
crate::common::action::Urgency::Normal,
);
let action = FactoredAction::from_index(action_idx).unwrap_or(hold_action);
// Get log probability from the already-synced probs_vec (safe indexing)
const EPSILON: f32 = 1e-8;
@@ -942,8 +948,13 @@ impl PpoTrainer {
// sample_action returns (action_idx, probs_vec) — reuse probs_vec, no duplicate sync
let (action_idx, probs_vec) = self.sample_action(&action_probs)?;
let action =
TradingAction::from_int(action_idx as u8).unwrap_or(TradingAction::Hold);
// Default: Flat/Market/Normal (Hold equivalent)
let hold_action = FactoredAction::new(
crate::common::action::ExposureLevel::Flat,
crate::common::action::OrderType::Market,
crate::common::action::Urgency::Normal,
);
let action = FactoredAction::from_index(action_idx).unwrap_or(hold_action);
// Get log probability from the already-synced probs_vec (safe indexing)
const EPSILON: f32 = 1e-8;
@@ -1526,12 +1537,18 @@ fn gpu_batch_to_trajectory_batch(
.map(|c| c.to_vec())
.collect();
let actions: Vec<TradingAction> = batch
let hold_action = FactoredAction::new(
crate::common::action::ExposureLevel::Flat,
crate::common::action::OrderType::Market,
crate::common::action::Urgency::Normal,
);
let actions: Vec<FactoredAction> = batch
.actions
.iter()
.take(total)
.map(|&a| {
TradingAction::from_int(a.clamp(0, 44) as u8).unwrap_or(TradingAction::Hold)
let idx = (a.clamp(0, 44)) as usize;
FactoredAction::from_index(idx).unwrap_or(hold_action)
})
.collect();

36
crates/ml/src/validation/ppo_adapter.rs
View File

@@ -13,7 +13,7 @@ use std::cell::RefCell;
use candle_core::{Device, Tensor};
use rand::Rng;
use crate::dqn::TradingAction;
use crate::common::action::FactoredAction;
use crate::ppo::gae::compute_gae;
#[cfg(test)]
use crate::ppo::gae::GAEConfig;
@@ -128,10 +128,8 @@ impl ValidatableStrategy for PpoStrategy {
/// Evaluate the MLP PPO on out-of-sample data.
///
/// For each bar, computes action probabilities via the actor, takes the
/// argmax action, and maps it to PnL:
/// - Buy (0): `+return`
/// - Sell (1): `-return`
/// - Hold (2): `0`
/// argmax action, and maps it to PnL using the factored action's target
/// exposure (e.g., Long100 -> +1.0, Flat -> 0.0, Short100 -> -1.0).
fn evaluate(&self, data: &TimeSeriesData) -> Result<Vec<f64>, MLError> {
let num_returns = data.returns.len();
let mut pnl = Vec::with_capacity(num_returns);
@@ -160,10 +158,10 @@ impl ValidatableStrategy for PpoStrategy {
let action_idx = argmax(&probs_vec);
let bar_return = data.returns.get(i).copied().unwrap_or(0.0);
let bar_pnl = match action_idx {
0 => bar_return, // Buy
1 => -bar_return, // Sell
_ => 0.0, // Hold
let bar_pnl = if let Ok(action) = FactoredAction::from_index(action_idx) {
action.target_exposure() * bar_return
} else {
0.0
};
pnl.push(bar_pnl);
@@ -385,10 +383,10 @@ impl ValidatableStrategy for PpoLstmStrategy {
let action_idx = argmax(&probs_vec);
let bar_return = data.returns.get(i).copied().unwrap_or(0.0);
let bar_pnl = match action_idx {
0 => bar_return, // Buy
1 => -bar_return, // Sell
_ => 0.0, // Hold
let bar_pnl = if let Ok(action) = FactoredAction::from_index(action_idx) {
action.target_exposure() * bar_return
} else {
0.0
};
pnl.push(bar_pnl);
@@ -433,8 +431,8 @@ fn argmax(values: &[f32]) -> usize {
/// Sample an action from logits via softmax + categorical sampling.
///
/// Returns `(TradingAction, log_prob)`.
fn sample_action_from_logits(logits: &Tensor, _device: &Device) -> Result<(TradingAction, f32), MLError> {
/// Returns `(FactoredAction, log_prob)`.
fn sample_action_from_logits(logits: &Tensor, _device: &Device) -> Result<(FactoredAction, f32), MLError> {
let probs = candle_nn::ops::softmax(logits, candle_core::D::Minus1)
.map_err(|e| MLError::ModelError(format!("Softmax failed: {}", e)))?;
let probs_vec = probs
@@ -451,8 +449,7 @@ fn sample_action_from_logits(logits: &Tensor, _device: &Device) -> Result<(Tradi
for (i, &prob) in probs_vec.iter().enumerate() {
cumulative += prob;
if sample <= cumulative {
let action = TradingAction::from_int(i as u8)
.ok_or_else(|| MLError::InvalidInput(format!("Invalid action index: {}", i)))?;
let action = FactoredAction::from_index(i)?;
const EPSILON: f32 = 1e-8;
let log_prob = (prob + EPSILON).ln();
return Ok((action, log_prob));
@@ -461,8 +458,7 @@ fn sample_action_from_logits(logits: &Tensor, _device: &Device) -> Result<(Tradi
// Fallback to last valid action
let last_idx = probs_vec.len().saturating_sub(1);
let action = TradingAction::from_int(last_idx as u8)
.ok_or_else(|| MLError::InvalidInput(format!("Invalid action index: {}", last_idx)))?;
let action = FactoredAction::from_index(last_idx)?;
const EPSILON: f32 = 1e-8;
let log_prob = (probs_vec.get(last_idx).copied().unwrap_or(EPSILON) + EPSILON).ln();
Ok((action, log_prob))
@@ -513,7 +509,7 @@ mod tests {
fn make_mlp_config() -> PPOConfig {
PPOConfig {
state_dim: 10,
num_actions: 3,
num_actions: 45,
policy_hidden_dims: vec![16, 8],
value_hidden_dims: vec![16, 8],
batch_size: 4,