5546a45bc3
gpu_n_episodes was manually overridden in GPU profiles, training configs, test files, and hyperopt — all set to 0 or small fixed values that bypassed the auto-scaling logic, causing a div-by-zero crash in train_baseline_rl. Now: single auto-scaling path via optimal_n_episodes() from VRAM/SM count. No manual override field. Cap at 16384 (consistent with AutoBatchSizer's 8192 cap pattern). Floor at 32 for small GPUs. Removed gpu_n_episodes from: - DQNHyperparameters, PpoHyperparameters structs - All 4 GPU profiles (rtx3050, h100, a100, default) - Training profiles (smoketest, localdev) - ExperienceProfile struct + serde - Hyperopt adapter - All test overrides Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
462 lines
16 KiB
Rust
462 lines
16 KiB
Rust
#![allow(
|
|
clippy::assertions_on_constants,
|
|
clippy::assertions_on_result_states,
|
|
clippy::clone_on_copy,
|
|
clippy::decimal_literal_representation,
|
|
clippy::doc_markdown,
|
|
clippy::empty_line_after_doc_comments,
|
|
clippy::field_reassign_with_default,
|
|
clippy::get_unwrap,
|
|
clippy::identity_op,
|
|
clippy::inconsistent_digit_grouping,
|
|
clippy::indexing_slicing,
|
|
clippy::integer_division,
|
|
clippy::len_zero,
|
|
clippy::let_underscore_must_use,
|
|
clippy::manual_div_ceil,
|
|
clippy::manual_let_else,
|
|
clippy::manual_range_contains,
|
|
clippy::modulo_arithmetic,
|
|
clippy::needless_range_loop,
|
|
clippy::non_ascii_literal,
|
|
clippy::redundant_clone,
|
|
clippy::shadow_reuse,
|
|
clippy::shadow_same,
|
|
clippy::shadow_unrelated,
|
|
clippy::single_match_else,
|
|
clippy::str_to_string,
|
|
clippy::string_slice,
|
|
clippy::tests_outside_test_module,
|
|
clippy::too_many_lines,
|
|
clippy::unnecessary_wraps,
|
|
clippy::unseparated_literal_suffix,
|
|
clippy::use_debug,
|
|
clippy::useless_vec,
|
|
clippy::wildcard_enum_match_arm,
|
|
clippy::else_if_without_else,
|
|
clippy::expect_used,
|
|
clippy::missing_const_for_fn,
|
|
clippy::similar_names,
|
|
clippy::type_complexity,
|
|
clippy::collapsible_else_if,
|
|
clippy::doc_lazy_continuation,
|
|
clippy::items_after_test_module,
|
|
clippy::map_clone,
|
|
clippy::multiple_unsafe_ops_per_block,
|
|
clippy::unwrap_or_default,
|
|
clippy::assign_op_pattern,
|
|
clippy::needless_borrow,
|
|
clippy::println_empty_string,
|
|
clippy::unnecessary_cast,
|
|
clippy::used_underscore_binding,
|
|
clippy::create_dir,
|
|
clippy::implicit_saturating_sub,
|
|
clippy::exit,
|
|
clippy::expect_fun_call,
|
|
clippy::too_many_arguments,
|
|
clippy::unnecessary_map_or,
|
|
clippy::unwrap_used,
|
|
dead_code,
|
|
unused_imports,
|
|
unused_variables,
|
|
clippy::cloned_ref_to_slice_refs,
|
|
clippy::neg_multiply,
|
|
clippy::while_let_loop,
|
|
clippy::bool_assert_comparison,
|
|
clippy::excessive_precision,
|
|
clippy::trivially_copy_pass_by_ref,
|
|
clippy::op_ref,
|
|
clippy::redundant_closure,
|
|
clippy::unnecessary_lazy_evaluations,
|
|
clippy::if_then_some_else_none,
|
|
clippy::unnecessary_to_owned,
|
|
clippy::single_component_path_imports,
|
|
)]
|
|
//! **AGENT 4.4B: Recurrent PPO Performance Tests**
|
|
//!
|
|
//! TDD implementation of 2 performance tests for Recurrent PPO:
|
|
//!
|
|
//! 1. **`test_recurrent_ppo_training_speed()`**
|
|
//! - Measures training time slowdown (recurrent vs feedforward)
|
|
//! - Expects: 1.5-2x slowdown (acceptable range: 1.2-3.0x)
|
|
//! - Fails if: >3x slowdown
|
|
//!
|
|
//! 2. **`test_recurrent_ppo_memory_usage()`**
|
|
//! - Measures GPU memory increase (recurrent vs feedforward)
|
|
//! - Expects: 20-50% more memory
|
|
//! - Fails if: >2x memory increase
|
|
//!
|
|
//! **Test Strategy**:
|
|
//! - Use small configs for fast iteration (5 epochs, batch_size=32)
|
|
//! - Train feedforward PPO first (baseline)
|
|
//! - Train recurrent PPO second (comparison)
|
|
//! - Measure time using std::time::Instant
|
|
//! - Measure memory using CUDA memory stats (if available)
|
|
//!
|
|
//! **Dependencies**:
|
|
//! - LSTM infrastructure (10/10 tests passing)
|
|
//! - PpoTrainer with LSTM support
|
|
//! - Gradient clipping: max_grad_norm_lstm=0.5
|
|
|
|
#![allow(unused_crate_dependencies)]
|
|
|
|
use ml_core::native_types::NativeDevice;
|
|
use chrono::{TimeZone, Utc};
|
|
use ml::features::extraction::{extract_ml_features, OHLCVBar};
|
|
use ml::ppo::PPOConfig;
|
|
use ml::trainers::ppo::PpoHyperparameters;
|
|
use std::time::Instant;
|
|
use tracing::info;
|
|
|
|
// ============================================================================
|
|
// Helper Functions
|
|
// ============================================================================
|
|
|
|
/// Generate synthetic OHLCV data for testing
|
|
fn generate_synthetic_bars(num_bars: usize) -> Vec<OHLCVBar> {
|
|
let mut bars = Vec::new();
|
|
let mut price = 100.0;
|
|
|
|
for i in 0..num_bars {
|
|
let change = (i as f64 * 0.1).sin() * 2.0; // Predictable oscillation
|
|
price += change;
|
|
|
|
let timestamp_secs = 1_700_000_000 + (i as i64 * 60); // Start from 2023-11-14, add 1 min per bar
|
|
let bar = OHLCVBar {
|
|
timestamp: Utc.timestamp_opt(timestamp_secs, 0).unwrap(),
|
|
open: price - 0.5,
|
|
high: price + 1.0,
|
|
low: price - 1.0,
|
|
close: price,
|
|
volume: 1000.0 + (i as f64 * 10.0),
|
|
};
|
|
bars.push(bar);
|
|
}
|
|
|
|
bars
|
|
}
|
|
|
|
/// Extract features from OHLCV bars
|
|
fn extract_features(bars: &[OHLCVBar]) -> Vec<Vec<f64>> {
|
|
extract_ml_features(bars)
|
|
.expect("Failed to extract features")
|
|
.into_iter()
|
|
.map(|f| f.to_vec())
|
|
.collect()
|
|
}
|
|
|
|
/// Create test hyperparameters for performance tests
|
|
fn create_test_hyperparams(_use_lstm: bool, sequence_length: usize) -> PpoHyperparameters {
|
|
PpoHyperparameters {
|
|
learning_rate: 1e-4,
|
|
actor_learning_rate: Some(1e-6),
|
|
critic_learning_rate: Some(0.001),
|
|
batch_size: 32, // Small batch for fast testing
|
|
gamma: 0.99,
|
|
clip_epsilon: 0.2,
|
|
vf_coef: 1.0,
|
|
ent_coef: 0.05,
|
|
gae_lambda: 0.95,
|
|
rollout_steps: 64, // Small rollout for fast testing
|
|
minibatch_size: 16,
|
|
epochs: 5, // Just 5 epochs for performance comparison
|
|
early_stopping_enabled: false,
|
|
min_value_loss_improvement_pct: 2.0,
|
|
min_explained_variance: 0.4,
|
|
plateau_window: 30,
|
|
min_epochs_before_stopping: 50,
|
|
max_position_absolute: 2.0,
|
|
transaction_cost_bps: 0.10,
|
|
cash_reserve_pct: 20.0,
|
|
circuit_breaker_threshold: 5,
|
|
sequence_length,
|
|
max_grad_norm_lstm: 0.5,
|
|
gpu_timesteps_per_episode: 50,
|
|
initial_capital: 1_000_000.0,
|
|
avg_spread: 0.0001,
|
|
accumulation_steps: 1,
|
|
hidden_dim_base: None,
|
|
}
|
|
}
|
|
|
|
// ============================================================================
|
|
// TEST 1: Recurrent PPO Training Speed
|
|
// ============================================================================
|
|
|
|
#[test]
|
|
fn test_recurrent_ppo_training_speed() {
|
|
info!("TEST 1: Recurrent PPO Training Speed Comparison");
|
|
|
|
// GPU device managed internally by PPO/LSTM constructors
|
|
|
|
// Generate test data
|
|
info!("Step 1: Generating synthetic data (200 bars)...");
|
|
let bars = generate_synthetic_bars(200);
|
|
let features = extract_features(&bars);
|
|
let state_dim = features[0].len();
|
|
let num_actions = 3; // BUY, HOLD, SELL
|
|
|
|
info!(bars_len = bars.len(), state_dim, "Generated synthetic data");
|
|
|
|
// Test 1a: Feedforward PPO (baseline)
|
|
info!("Step 2: Training feedforward PPO (baseline)...");
|
|
let ff_hyperparams = create_test_hyperparams(false, 1);
|
|
let ff_config = PPOConfig {
|
|
state_dim,
|
|
num_actions,
|
|
policy_hidden_dims: vec![64, 32],
|
|
value_hidden_dims: vec![64, 32],
|
|
policy_learning_rate: ff_hyperparams.actor_learning_rate.unwrap(),
|
|
value_learning_rate: ff_hyperparams.critic_learning_rate.unwrap(),
|
|
batch_size: ff_hyperparams.batch_size,
|
|
mini_batch_size: ff_hyperparams.minibatch_size,
|
|
num_epochs: 2, // Reduced for faster testing
|
|
use_lstm: false,
|
|
..PPOConfig::default()
|
|
};
|
|
|
|
let start = Instant::now();
|
|
let ff_ppo = ml::ppo::ppo::PPO::new(ff_config.clone());
|
|
assert!(ff_ppo.is_ok(), "Failed to create feedforward PPO");
|
|
|
|
// Simulate training loop (just network operations, no full trainer)
|
|
let dummy_state = vec![0.0f32; state_dim]; // flat state vector for PPO API
|
|
|
|
for _epoch in 0..ff_hyperparams.epochs {
|
|
let _ = ff_ppo
|
|
.as_ref()
|
|
.unwrap()
|
|
.actor
|
|
.forward_host(&dummy_state, 1);
|
|
let _ = ff_ppo.as_ref().unwrap().critic.forward_host(&dummy_state, 1);
|
|
}
|
|
|
|
let ff_duration = start.elapsed();
|
|
info!(
|
|
epochs = ff_hyperparams.epochs,
|
|
duration_s = ff_duration.as_secs_f64(),
|
|
ms_per_epoch = ff_duration.as_millis() as f64 / ff_hyperparams.epochs as f64,
|
|
"Feedforward PPO training complete"
|
|
);
|
|
|
|
// Test 1b: Recurrent PPO (with LSTM)
|
|
info!("Step 3: Training recurrent PPO (with LSTM)...");
|
|
let lstm_hyperparams = create_test_hyperparams(true, 16);
|
|
|
|
let start = Instant::now();
|
|
|
|
// For LSTM, we need to use LSTM networks
|
|
let lstm_policy =
|
|
ml::ppo::lstm_networks::LSTMPolicyNetwork::new(state_dim, 128, 1, num_actions);
|
|
assert!(
|
|
lstm_policy.is_ok(),
|
|
"Failed to create LSTM policy network"
|
|
);
|
|
|
|
let lstm_value =
|
|
ml::ppo::lstm_networks::LSTMValueNetwork::new(state_dim, 128, 1);
|
|
assert!(lstm_value.is_ok(), "Failed to create LSTM value network");
|
|
|
|
// Simulate training loop with hidden state propagation
|
|
let batch_size = lstm_hyperparams.batch_size;
|
|
let hidden_dim = 128;
|
|
let num_layers = 1;
|
|
|
|
let h0 = vec![0.0f32; num_layers * batch_size * hidden_dim];
|
|
let c0 = vec![0.0f32; num_layers * batch_size * hidden_dim];
|
|
|
|
for _epoch in 0..lstm_hyperparams.epochs {
|
|
let mut h_t = h0.clone();
|
|
let mut c_t = c0.clone();
|
|
|
|
// Simulate sequence processing
|
|
for _t in 0..lstm_hyperparams.sequence_length {
|
|
let (_, new_h, new_c) = lstm_policy
|
|
.as_ref()
|
|
.unwrap()
|
|
.forward(&dummy_state, &h_t, &c_t, 1)
|
|
.expect("LSTM policy forward failed");
|
|
let (_, _new_h_v, _new_c_v) = lstm_value
|
|
.as_ref()
|
|
.unwrap()
|
|
.forward(&dummy_state, &h_t, &c_t, 1)
|
|
.expect("LSTM value forward failed");
|
|
|
|
h_t = new_h;
|
|
c_t = new_c;
|
|
}
|
|
}
|
|
|
|
let lstm_duration = start.elapsed();
|
|
info!(
|
|
epochs = lstm_hyperparams.epochs,
|
|
duration_s = lstm_duration.as_secs_f64(),
|
|
ms_per_epoch = lstm_duration.as_millis() as f64 / lstm_hyperparams.epochs as f64,
|
|
"Recurrent PPO training complete"
|
|
);
|
|
|
|
// Step 4: Calculate slowdown
|
|
info!("Step 4: Calculating slowdown ratio...");
|
|
let slowdown = lstm_duration.as_secs_f64() / ff_duration.as_secs_f64();
|
|
info!(
|
|
ff_duration_s = ff_duration.as_secs_f64(),
|
|
lstm_duration_s = lstm_duration.as_secs_f64(),
|
|
slowdown,
|
|
"Slowdown ratio calculated"
|
|
);
|
|
|
|
// Step 5: Validation
|
|
info!("Step 5: Validating performance expectations...");
|
|
|
|
// Expected range: With seq_len=16, we expect 10-20x slowdown (processing 16 timesteps per epoch)
|
|
// Theoretical minimum: 16x (seq_len factor alone)
|
|
// Actual includes LSTM overhead (gates, state management)
|
|
assert!(
|
|
slowdown >= 1.0,
|
|
"Recurrent should be slower than feedforward (got {:.2}x)",
|
|
slowdown
|
|
);
|
|
assert!(
|
|
slowdown >= 5.0,
|
|
"Recurrent should be at least 5x slower with seq_len=16 (got {:.2}x)",
|
|
slowdown
|
|
);
|
|
assert!(
|
|
slowdown <= 30.0,
|
|
"Recurrent should be <30x slower (got {:.2}x - check for bugs!)",
|
|
slowdown
|
|
);
|
|
|
|
if slowdown >= 10.0 && slowdown <= 20.0 {
|
|
info!("Slowdown within expected range (10-20x for seq_len=16)");
|
|
} else if slowdown >= 5.0 && slowdown < 10.0 {
|
|
info!("Slowdown better than expected (5-10x, efficient implementation!)");
|
|
} else {
|
|
info!("Slowdown higher than expected (20-30x, still acceptable)");
|
|
}
|
|
|
|
info!(
|
|
ff_duration_s = ff_duration.as_secs_f64(),
|
|
ff_ms_per_epoch = ff_duration.as_millis() as f64 / ff_hyperparams.epochs as f64,
|
|
lstm_duration_s = lstm_duration.as_secs_f64(),
|
|
lstm_ms_per_epoch = lstm_duration.as_millis() as f64 / lstm_hyperparams.epochs as f64,
|
|
slowdown,
|
|
"TEST 1 PASSED: Training Speed Validated"
|
|
);
|
|
}
|
|
|
|
// ============================================================================
|
|
// TEST 2: Recurrent PPO Memory Usage
|
|
// ============================================================================
|
|
|
|
#[test]
|
|
fn test_recurrent_ppo_memory_usage() {
|
|
info!("TEST 2: Recurrent PPO Memory Usage Comparison");
|
|
|
|
// GPU device managed internally by LSTM constructors
|
|
|
|
// Generate test data
|
|
info!("Step 1: Generating synthetic data (200 bars)...");
|
|
let bars = generate_synthetic_bars(200);
|
|
let features = extract_features(&bars);
|
|
let state_dim = features[0].len();
|
|
let num_actions = 3;
|
|
|
|
info!(bars_len = bars.len(), state_dim, "Generated synthetic data");
|
|
|
|
// Test 2a: Feedforward PPO memory footprint
|
|
info!("Step 2: Measuring feedforward PPO memory...");
|
|
let ff_config = PPOConfig {
|
|
state_dim,
|
|
num_actions,
|
|
policy_hidden_dims: vec![64, 32],
|
|
value_hidden_dims: vec![64, 32],
|
|
policy_learning_rate: 1e-6,
|
|
value_learning_rate: 0.001,
|
|
batch_size: 32,
|
|
mini_batch_size: 16,
|
|
num_epochs: 2,
|
|
use_lstm: false,
|
|
..PPOConfig::default()
|
|
};
|
|
|
|
let _ff_ppo = ml::ppo::ppo::PPO::new(ff_config.clone())
|
|
.expect("Failed to create feedforward PPO");
|
|
|
|
// Estimate parameter count
|
|
let ff_policy_params = (state_dim * 64 + 64) + (64 * 32 + 32) + (32 * num_actions + num_actions);
|
|
let ff_value_params = (state_dim * 64 + 64) + (64 * 32 + 32) + (32 * 1 + 1);
|
|
let ff_total_params = ff_policy_params + ff_value_params;
|
|
let ff_memory_bytes = ff_total_params * std::mem::size_of::<f32>();
|
|
let ff_memory_mb = ff_memory_bytes as f64 / (1024.0 * 1024.0);
|
|
|
|
info!(ff_policy_params, ff_value_params, ff_total_params, ff_memory_mb, "Feedforward PPO parameters");
|
|
|
|
// Test 2b: Recurrent PPO memory footprint
|
|
info!("Step 3: Measuring recurrent PPO memory...");
|
|
|
|
let _lstm_policy =
|
|
ml::ppo::lstm_networks::LSTMPolicyNetwork::new(state_dim, 128, 1, num_actions)
|
|
.expect("Failed to create LSTM policy");
|
|
let _lstm_value = ml::ppo::lstm_networks::LSTMValueNetwork::new(state_dim, 128, 1)
|
|
.expect("Failed to create LSTM value");
|
|
|
|
// Estimate LSTM parameters
|
|
// LSTM has 4 gates: input, forget, cell, output
|
|
// Each gate has: W_ih (input_dim x hidden_dim) + W_hh (hidden_dim x hidden_dim) + bias (hidden_dim)
|
|
let lstm_input_layer_params = state_dim * 128 + 128;
|
|
let lstm_layer_params = 4 * ((128 * 128) + (128 * 128) + 128); // 4 gates
|
|
let lstm_output_layer_params = 128 * num_actions + num_actions;
|
|
let lstm_policy_params = lstm_input_layer_params + lstm_layer_params + lstm_output_layer_params;
|
|
|
|
let lstm_value_input_params = state_dim * 128 + 128;
|
|
let lstm_value_layer_params = 4 * ((128 * 128) + (128 * 128) + 128);
|
|
let lstm_value_output_params = 128 * 1 + 1;
|
|
let lstm_value_params = lstm_value_input_params + lstm_value_layer_params + lstm_value_output_params;
|
|
|
|
let lstm_total_params = lstm_policy_params + lstm_value_params;
|
|
let lstm_memory_bytes = lstm_total_params * std::mem::size_of::<f32>();
|
|
let lstm_memory_mb = lstm_memory_bytes as f64 / (1024.0 * 1024.0);
|
|
|
|
info!(lstm_policy_params, lstm_value_params, lstm_total_params, lstm_memory_mb, "Recurrent PPO parameters");
|
|
|
|
// Step 4: Calculate memory increase
|
|
info!("Step 4: Calculating memory increase...");
|
|
let memory_increase = lstm_memory_mb / ff_memory_mb;
|
|
let memory_increase_pct = (memory_increase - 1.0) * 100.0;
|
|
|
|
info!(ff_memory_mb, lstm_memory_mb, memory_increase, memory_increase_pct, "Memory comparison");
|
|
|
|
// Step 5: Validation
|
|
info!("Step 5: Validating memory expectations...");
|
|
|
|
// Expected range: LSTM has 4 gates with large weight matrices
|
|
// For hidden_dim=128: ~130K params vs ~33K for feedforward (~4-10x increase expected)
|
|
assert!(
|
|
memory_increase >= 1.0,
|
|
"Recurrent should use more memory than feedforward (got {:.2}x)",
|
|
memory_increase
|
|
);
|
|
assert!(
|
|
memory_increase <= 15.0,
|
|
"Recurrent should use <15x memory (got {:.2}x - check for memory leak!)",
|
|
memory_increase
|
|
);
|
|
|
|
if memory_increase >= 4.0 && memory_increase <= 10.0 {
|
|
info!("Memory increase within expected range (4-10x for LSTM with hidden_dim=128)");
|
|
} else if memory_increase < 4.0 {
|
|
info!("Memory increase lower than expected (<4x, very efficient!)");
|
|
} else {
|
|
info!("Memory increase higher than expected (10-15x, still acceptable)");
|
|
}
|
|
|
|
info!(
|
|
ff_memory_mb,
|
|
lstm_memory_mb,
|
|
memory_increase,
|
|
memory_increase_pct,
|
|
"TEST 2 PASSED: Memory Usage Validated"
|
|
);
|
|
}
|