gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(hyperopt): add GPU backtest fitness to PPO adapter

Browse Source 
Wire GpuBacktestEvaluator into PPO hyperopt trials so the optimizer
ranks candidates by walk-forward Sharpe ratio instead of raw episode
reward.  The PPO actor's 45-action softmax probabilities are collapsed
to 5 exposure scores via ppo_to_exposure_scores before the backtest
loop.  When CUDA is unavailable or the backtest fails, the adapter
falls back to the original -avg_episode_reward objective.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-11 15:46:01 +01:00
parent c0dd5ef99c
commit 448111ca30
1 changed files with 178 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

179
crates/ml/src/hyperopt/adapters/ppo.rs
View File

@@ -39,6 +39,7 @@ use std::sync::Arc;
use tracing::{info, warn};
use crate::common::action::FactoredAction;
use crate::cuda_pipeline::signal_adapter::backtest_fitness;
use crate::dqn::curiosity::CuriosityModule;
use crate::hyperopt::paths::TrainingPaths;
use crate::hyperopt::traits::{HardwareBudget, HyperparameterOptimizable, ParameterSpace};
@@ -50,6 +51,11 @@ use crate::ppo::trajectories::TrajectoryBatch;
use crate::ppo::trajectory_replay::TrajectoryReplayBuffer;
use crate::MLError;
#[cfg(feature = "cuda")]
use crate::cuda_pipeline::gpu_backtest_evaluator::{GpuBacktestConfig, GpuBacktestEvaluator};
#[cfg(feature = "cuda")]
use crate::cuda_pipeline::signal_adapter::ppo_to_exposure_scores;
/// Pure model VRAM in MB (actor + critic + optimizers + gradients).
const MODEL_OVERHEAD_MB: f64 = 300.0;
@@ -269,6 +275,10 @@ pub struct PPOMetrics {
pub avg_episode_reward: f64,
/// Number of episodes completed
pub episodes_completed: usize,
/// GPU walk-forward backtest Sharpe ratio (None when CUDA unavailable or backtest fails)
pub backtest_sharpe: Option<f32>,
/// GPU walk-forward backtest total trade count (None when CUDA unavailable or backtest fails)
pub backtest_trades: Option<u32>,
}
/// PPO trainer for hyperparameter optimization
@@ -1149,6 +1159,25 @@ impl HyperparameterOptimizable for PPOTrainer {
let avg_value_loss = total_value_loss / num_batches as f64;
let avg_reward = total_reward / num_batches as f64;
// GPU walk-forward backtest (CUDA only, graceful fallback to None)
#[cfg(feature = "cuda")]
let (bt_sharpe, bt_trades) = if trial_device.is_cuda() {
match self.run_gpu_backtest(&ppo_agent, &trial_device, &params) {
Ok((s, t)) => {
info!("PPO GPU backtest: Sharpe={:.4} trades={}", s, t);
(Some(s), Some(t))
}
Err(e) => {
warn!("PPO GPU backtest failed: {e}");
(None, None)
}
}
} else {
(None, None)
};
#[cfg(not(feature = "cuda"))]
let (bt_sharpe, bt_trades): (Option<f32>, Option<u32>) = (None, None);
let metrics = PPOMetrics {
policy_loss: avg_policy_loss,
value_loss: avg_value_loss,
@@ -1157,6 +1186,8 @@ impl HyperparameterOptimizable for PPOTrainer {
combined_loss: avg_policy_loss + params.value_loss_coeff * avg_value_loss,
avg_episode_reward: avg_reward,
episodes_completed: self.episodes,
backtest_sharpe: bt_sharpe,
backtest_trades: bt_trades,
};
info!("Training completed:");
@@ -1208,7 +1239,13 @@ impl HyperparameterOptimizable for PPOTrainer {
}
fn extract_objective(metrics: &Self::Metrics) -> f64 {
// CRITICAL: Maximize episode rewards (negative because optimizer MINIMIZES)
// Prefer GPU backtest fitness when available (walk-forward Sharpe is more
// reliable than episode reward for ranking hyperparameter trials).
if let (Some(sharpe), Some(trades)) = (metrics.backtest_sharpe, metrics.backtest_trades) {
return backtest_fitness(sharpe, trades, 30, 0.0);
}
// Fallback: maximize episode rewards (negative because optimizer MINIMIZES)
//
// We optimize for avg_episode_reward, NOT validation loss, because:
// 1. Loss minimization rewards frozen policies (policy_loss=0, KL_div=0)
@@ -1291,6 +1328,84 @@ fn collect_dbn_files_recursive(dir: &std::path::Path) -> Vec<std::path::PathBuf>
}
impl PPOTrainer {
/// Run GPU walk-forward backtest on validation data using the trained PPO actor.
///
/// Uses the last 20% of preloaded bars as validation data. Builds a single
/// window, creates a `GpuBacktestEvaluator`, and calls `evaluate()` with
/// a forward function that converts PPO 45-action softmax probabilities into
/// 5-action exposure scores via `ppo_to_exposure_scores`.
///
/// Returns `(sharpe, total_trades)` from the first (and only) window.
#[cfg(feature = "cuda")]
fn run_gpu_backtest(
&self,
ppo: &PPO,
device: &Device,
params: &PPOParams,
) -> Result<(f32, u32), MLError> {
let data = self.preloaded_data.as_ref().ok_or_else(|| {
MLError::ConfigError("run_gpu_backtest: no preloaded data".to_owned())
})?;
// Use last 20% of data for validation
let val_start = (data.len() as f64 * 0.8) as usize;
let val_data = data.get(val_start..).ok_or_else(|| {
MLError::ConfigError("run_gpu_backtest: val slice out of bounds".to_owned())
})?;
if val_data.len() < 50 {
return Err(MLError::ConfigError(format!(
"run_gpu_backtest: insufficient validation data ({} < 50)",
val_data.len()
)));
}
// Build price and feature vectors for a single window
let mut prices = Vec::with_capacity(val_data.len());
let mut features = Vec::with_capacity(val_data.len());
for (fv, close_price) in val_data {
let close = *close_price as f32;
prices.push([close, close, close, close]); // OHLC approximation (same as DQN)
features.push(fv.to_vec());
}
let window_prices = vec![prices];
let window_features = vec![features];
// Market features only -- portfolio features (3) added by evaluator's gather_states.
let feature_dim = 42;
let config = GpuBacktestConfig {
max_position: params.max_position_absolute as f32,
tx_cost_bps: self.tx_cost_bps as f32,
spread_cost: (self.tick_size * self.spread_ticks) as f32,
initial_capital: 35_000.0,
};
let mut evaluator = GpuBacktestEvaluator::new(
&window_prices,
&window_features,
feature_dim,
config,
device,
)?;
// Forward function: PPO actor → softmax → ppo_to_exposure_scores → [B, 5]
let metrics = evaluator.evaluate(
&|states: &candle_core::Tensor| -> Result<candle_core::Tensor, MLError> {
let probs = ppo.actor.action_probabilities(states)?;
ppo_to_exposure_scores(&probs)
},
3, // portfolio_dim
device,
)?;
let first = metrics.first().ok_or_else(|| {
MLError::ModelError("run_gpu_backtest: evaluator returned no windows".to_owned())
})?;
Ok((first.sharpe, first.total_trades as u32))
}
/// Load training data from Parquet/DBN files
///
/// Returns feature vectors (42 dims) with target close prices for trajectory generation
@@ -1856,6 +1971,8 @@ mod tests {
combined_loss: 0.8,
avg_episode_reward: 100.0, // Good performance
episodes_completed: 1000,
backtest_sharpe: None,
backtest_trades: None,
};
let objective_positive = PPOTrainer::extract_objective(&metrics_positive);
assert_eq!(
@@ -1872,6 +1989,8 @@ mod tests {
combined_loss: 0.8,
avg_episode_reward: -50.0, // Poor performance
episodes_completed: 1000,
backtest_sharpe: None,
backtest_trades: None,
};
let objective_negative = PPOTrainer::extract_objective(&metrics_negative);
assert_eq!(
@@ -1888,6 +2007,8 @@ mod tests {
combined_loss: 0.0,
avg_episode_reward: 0.0, // Neutral performance
episodes_completed: 1000,
backtest_sharpe: None,
backtest_trades: None,
};
let objective_zero = PPOTrainer::extract_objective(&metrics_zero);
assert_eq!(
@@ -1919,6 +2040,8 @@ mod tests {
combined_loss: 40.0, // High combined loss
avg_episode_reward: 200.0, // But high reward (good trading)
episodes_completed: 1000,
backtest_sharpe: None,
backtest_trades: None,
};
let metrics_low_reward_low_loss = PPOMetrics {
@@ -1929,6 +2052,8 @@ mod tests {
combined_loss: 0.0, // Zero combined loss
avg_episode_reward: 10.0, // But low reward (poor trading)
episodes_completed: 1000,
backtest_sharpe: None,
backtest_trades: None,
};
let obj_high_reward = PPOTrainer::extract_objective(&metrics_high_reward_high_loss);
@@ -1940,6 +2065,58 @@ mod tests {
obj_high_reward, obj_low_reward);
}
#[test]
fn test_objective_uses_backtest_fitness_when_available() {
// When GPU backtest results are available, extract_objective should use
// backtest_fitness instead of -avg_episode_reward.
let metrics_with_backtest = PPOMetrics {
policy_loss: 0.5,
value_loss: 0.3,
val_policy_loss: 0.4,
val_value_loss: 0.2,
combined_loss: 0.8,
avg_episode_reward: 100.0,
episodes_completed: 1000,
backtest_sharpe: Some(1.5),
backtest_trades: Some(50),
};
let obj = PPOTrainer::extract_objective(&metrics_with_backtest);
// backtest_fitness(1.5, 50, 30, 0.0) = -1.5 (trades >= min_trades)
assert!((obj - (-1.5)).abs() < 1e-9,
"Expected backtest_fitness(-1.5), got {obj}");
// Verify fallback: without backtest, should be -avg_episode_reward
let metrics_no_backtest = PPOMetrics {
backtest_sharpe: None,
backtest_trades: None,
..metrics_with_backtest.clone()
};
let obj_fallback = PPOTrainer::extract_objective(&metrics_no_backtest);
assert!((obj_fallback - (-100.0)).abs() < 1e-9,
"Expected -avg_episode_reward (-100.0), got {obj_fallback}");
}
#[test]
fn test_objective_backtest_few_trades_penalty() {
// When trades < min_trades, backtest_fitness applies a linear penalty
let metrics = PPOMetrics {
policy_loss: 0.0,
value_loss: 0.0,
val_policy_loss: 0.0,
val_value_loss: 0.0,
combined_loss: 0.0,
avg_episode_reward: 0.0,
episodes_completed: 100,
backtest_sharpe: Some(2.0),
backtest_trades: Some(15), // half of min_trades=30
};
let obj = PPOTrainer::extract_objective(&metrics);
// backtest_fitness(2.0, 15, 30, 0.0) = -2.0 * (15/30) = -1.0
assert!((obj - (-1.0)).abs() < 1e-9,
"Expected penalized fitness (-1.0), got {obj}");
}
#[test]
fn test_curiosity_weight_in_params() -> Result<(), MLError> {
let params = PPOParams {