From 448111ca30bc35bc6f7ee292b0a8e212d255c282 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Wed, 11 Mar 2026 15:46:01 +0100 Subject: [PATCH] feat(hyperopt): add GPU backtest fitness to PPO adapter Wire GpuBacktestEvaluator into PPO hyperopt trials so the optimizer ranks candidates by walk-forward Sharpe ratio instead of raw episode reward. The PPO actor's 45-action softmax probabilities are collapsed to 5 exposure scores via ppo_to_exposure_scores before the backtest loop. When CUDA is unavailable or the backtest fails, the adapter falls back to the original -avg_episode_reward objective. Co-Authored-By: Claude Opus 4.6 --- crates/ml/src/hyperopt/adapters/ppo.rs | 179 ++++++++++++++++++++++++- 1 file changed, 178 insertions(+), 1 deletion(-) diff --git a/crates/ml/src/hyperopt/adapters/ppo.rs b/crates/ml/src/hyperopt/adapters/ppo.rs index 35cc3200e..736231767 100644 --- a/crates/ml/src/hyperopt/adapters/ppo.rs +++ b/crates/ml/src/hyperopt/adapters/ppo.rs @@ -39,6 +39,7 @@ use std::sync::Arc; use tracing::{info, warn}; use crate::common::action::FactoredAction; +use crate::cuda_pipeline::signal_adapter::backtest_fitness; use crate::dqn::curiosity::CuriosityModule; use crate::hyperopt::paths::TrainingPaths; use crate::hyperopt::traits::{HardwareBudget, HyperparameterOptimizable, ParameterSpace}; @@ -50,6 +51,11 @@ use crate::ppo::trajectories::TrajectoryBatch; use crate::ppo::trajectory_replay::TrajectoryReplayBuffer; use crate::MLError; +#[cfg(feature = "cuda")] +use crate::cuda_pipeline::gpu_backtest_evaluator::{GpuBacktestConfig, GpuBacktestEvaluator}; +#[cfg(feature = "cuda")] +use crate::cuda_pipeline::signal_adapter::ppo_to_exposure_scores; + /// Pure model VRAM in MB (actor + critic + optimizers + gradients). const MODEL_OVERHEAD_MB: f64 = 300.0; @@ -269,6 +275,10 @@ pub struct PPOMetrics { pub avg_episode_reward: f64, /// Number of episodes completed pub episodes_completed: usize, + /// GPU walk-forward backtest Sharpe ratio (None when CUDA unavailable or backtest fails) + pub backtest_sharpe: Option, + /// GPU walk-forward backtest total trade count (None when CUDA unavailable or backtest fails) + pub backtest_trades: Option, } /// PPO trainer for hyperparameter optimization @@ -1149,6 +1159,25 @@ impl HyperparameterOptimizable for PPOTrainer { let avg_value_loss = total_value_loss / num_batches as f64; let avg_reward = total_reward / num_batches as f64; + // GPU walk-forward backtest (CUDA only, graceful fallback to None) + #[cfg(feature = "cuda")] + let (bt_sharpe, bt_trades) = if trial_device.is_cuda() { + match self.run_gpu_backtest(&ppo_agent, &trial_device, ¶ms) { + Ok((s, t)) => { + info!("PPO GPU backtest: Sharpe={:.4} trades={}", s, t); + (Some(s), Some(t)) + } + Err(e) => { + warn!("PPO GPU backtest failed: {e}"); + (None, None) + } + } + } else { + (None, None) + }; + #[cfg(not(feature = "cuda"))] + let (bt_sharpe, bt_trades): (Option, Option) = (None, None); + let metrics = PPOMetrics { policy_loss: avg_policy_loss, value_loss: avg_value_loss, @@ -1157,6 +1186,8 @@ impl HyperparameterOptimizable for PPOTrainer { combined_loss: avg_policy_loss + params.value_loss_coeff * avg_value_loss, avg_episode_reward: avg_reward, episodes_completed: self.episodes, + backtest_sharpe: bt_sharpe, + backtest_trades: bt_trades, }; info!("Training completed:"); @@ -1208,7 +1239,13 @@ impl HyperparameterOptimizable for PPOTrainer { } fn extract_objective(metrics: &Self::Metrics) -> f64 { - // CRITICAL: Maximize episode rewards (negative because optimizer MINIMIZES) + // Prefer GPU backtest fitness when available (walk-forward Sharpe is more + // reliable than episode reward for ranking hyperparameter trials). + if let (Some(sharpe), Some(trades)) = (metrics.backtest_sharpe, metrics.backtest_trades) { + return backtest_fitness(sharpe, trades, 30, 0.0); + } + + // Fallback: maximize episode rewards (negative because optimizer MINIMIZES) // // We optimize for avg_episode_reward, NOT validation loss, because: // 1. Loss minimization rewards frozen policies (policy_loss=0, KL_div=0) @@ -1291,6 +1328,84 @@ fn collect_dbn_files_recursive(dir: &std::path::Path) -> Vec } impl PPOTrainer { + /// Run GPU walk-forward backtest on validation data using the trained PPO actor. + /// + /// Uses the last 20% of preloaded bars as validation data. Builds a single + /// window, creates a `GpuBacktestEvaluator`, and calls `evaluate()` with + /// a forward function that converts PPO 45-action softmax probabilities into + /// 5-action exposure scores via `ppo_to_exposure_scores`. + /// + /// Returns `(sharpe, total_trades)` from the first (and only) window. + #[cfg(feature = "cuda")] + fn run_gpu_backtest( + &self, + ppo: &PPO, + device: &Device, + params: &PPOParams, + ) -> Result<(f32, u32), MLError> { + let data = self.preloaded_data.as_ref().ok_or_else(|| { + MLError::ConfigError("run_gpu_backtest: no preloaded data".to_owned()) + })?; + + // Use last 20% of data for validation + let val_start = (data.len() as f64 * 0.8) as usize; + let val_data = data.get(val_start..).ok_or_else(|| { + MLError::ConfigError("run_gpu_backtest: val slice out of bounds".to_owned()) + })?; + + if val_data.len() < 50 { + return Err(MLError::ConfigError(format!( + "run_gpu_backtest: insufficient validation data ({} < 50)", + val_data.len() + ))); + } + + // Build price and feature vectors for a single window + let mut prices = Vec::with_capacity(val_data.len()); + let mut features = Vec::with_capacity(val_data.len()); + for (fv, close_price) in val_data { + let close = *close_price as f32; + prices.push([close, close, close, close]); // OHLC approximation (same as DQN) + features.push(fv.to_vec()); + } + + let window_prices = vec![prices]; + let window_features = vec![features]; + + // Market features only -- portfolio features (3) added by evaluator's gather_states. + let feature_dim = 42; + let config = GpuBacktestConfig { + max_position: params.max_position_absolute as f32, + tx_cost_bps: self.tx_cost_bps as f32, + spread_cost: (self.tick_size * self.spread_ticks) as f32, + initial_capital: 35_000.0, + }; + + let mut evaluator = GpuBacktestEvaluator::new( + &window_prices, + &window_features, + feature_dim, + config, + device, + )?; + + // Forward function: PPO actor → softmax → ppo_to_exposure_scores → [B, 5] + let metrics = evaluator.evaluate( + &|states: &candle_core::Tensor| -> Result { + let probs = ppo.actor.action_probabilities(states)?; + ppo_to_exposure_scores(&probs) + }, + 3, // portfolio_dim + device, + )?; + + let first = metrics.first().ok_or_else(|| { + MLError::ModelError("run_gpu_backtest: evaluator returned no windows".to_owned()) + })?; + + Ok((first.sharpe, first.total_trades as u32)) + } + /// Load training data from Parquet/DBN files /// /// Returns feature vectors (42 dims) with target close prices for trajectory generation @@ -1856,6 +1971,8 @@ mod tests { combined_loss: 0.8, avg_episode_reward: 100.0, // Good performance episodes_completed: 1000, + backtest_sharpe: None, + backtest_trades: None, }; let objective_positive = PPOTrainer::extract_objective(&metrics_positive); assert_eq!( @@ -1872,6 +1989,8 @@ mod tests { combined_loss: 0.8, avg_episode_reward: -50.0, // Poor performance episodes_completed: 1000, + backtest_sharpe: None, + backtest_trades: None, }; let objective_negative = PPOTrainer::extract_objective(&metrics_negative); assert_eq!( @@ -1888,6 +2007,8 @@ mod tests { combined_loss: 0.0, avg_episode_reward: 0.0, // Neutral performance episodes_completed: 1000, + backtest_sharpe: None, + backtest_trades: None, }; let objective_zero = PPOTrainer::extract_objective(&metrics_zero); assert_eq!( @@ -1919,6 +2040,8 @@ mod tests { combined_loss: 40.0, // High combined loss avg_episode_reward: 200.0, // But high reward (good trading) episodes_completed: 1000, + backtest_sharpe: None, + backtest_trades: None, }; let metrics_low_reward_low_loss = PPOMetrics { @@ -1929,6 +2052,8 @@ mod tests { combined_loss: 0.0, // Zero combined loss avg_episode_reward: 10.0, // But low reward (poor trading) episodes_completed: 1000, + backtest_sharpe: None, + backtest_trades: None, }; let obj_high_reward = PPOTrainer::extract_objective(&metrics_high_reward_high_loss); @@ -1940,6 +2065,58 @@ mod tests { obj_high_reward, obj_low_reward); } + #[test] + fn test_objective_uses_backtest_fitness_when_available() { + // When GPU backtest results are available, extract_objective should use + // backtest_fitness instead of -avg_episode_reward. + let metrics_with_backtest = PPOMetrics { + policy_loss: 0.5, + value_loss: 0.3, + val_policy_loss: 0.4, + val_value_loss: 0.2, + combined_loss: 0.8, + avg_episode_reward: 100.0, + episodes_completed: 1000, + backtest_sharpe: Some(1.5), + backtest_trades: Some(50), + }; + + let obj = PPOTrainer::extract_objective(&metrics_with_backtest); + // backtest_fitness(1.5, 50, 30, 0.0) = -1.5 (trades >= min_trades) + assert!((obj - (-1.5)).abs() < 1e-9, + "Expected backtest_fitness(-1.5), got {obj}"); + + // Verify fallback: without backtest, should be -avg_episode_reward + let metrics_no_backtest = PPOMetrics { + backtest_sharpe: None, + backtest_trades: None, + ..metrics_with_backtest.clone() + }; + let obj_fallback = PPOTrainer::extract_objective(&metrics_no_backtest); + assert!((obj_fallback - (-100.0)).abs() < 1e-9, + "Expected -avg_episode_reward (-100.0), got {obj_fallback}"); + } + + #[test] + fn test_objective_backtest_few_trades_penalty() { + // When trades < min_trades, backtest_fitness applies a linear penalty + let metrics = PPOMetrics { + policy_loss: 0.0, + value_loss: 0.0, + val_policy_loss: 0.0, + val_value_loss: 0.0, + combined_loss: 0.0, + avg_episode_reward: 0.0, + episodes_completed: 100, + backtest_sharpe: Some(2.0), + backtest_trades: Some(15), // half of min_trades=30 + }; + let obj = PPOTrainer::extract_objective(&metrics); + // backtest_fitness(2.0, 15, 30, 0.0) = -2.0 * (15/30) = -1.0 + assert!((obj - (-1.0)).abs() < 1e-9, + "Expected penalized fitness (-1.0), got {obj}"); + } + #[test] fn test_curiosity_weight_in_params() -> Result<(), MLError> { let params = PPOParams {