From 4aceea16998fc7646266a3bd5e022fe39c9cba3a Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sat, 4 Apr 2026 15:25:02 +0200 Subject: [PATCH] fix: profile is single source of truth, q_clip wired as guard, fewer walk-forward folds Co-Authored-By: Claude Opus 4.6 (1M context) --- config/training/dqn-smoketest.toml | 3 + .../src/trainers/dqn/trainer/training_loop.rs | 10 ++++ crates/ml/tests/dqn_training_pipeline_test.rs | 55 +++---------------- 3 files changed, 21 insertions(+), 47 deletions(-) diff --git a/config/training/dqn-smoketest.toml b/config/training/dqn-smoketest.toml index 02af792c7..ec7924ab0 100644 --- a/config/training/dqn-smoketest.toml +++ b/config/training/dqn-smoketest.toml @@ -119,3 +119,6 @@ loss_aversion = 1.5 q_gap_threshold = 0.1 w_dd = 1.0 dd_threshold = 0.02 + +[walk_forward] +step_fraction = 0.25 diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 058f1ec42..760ded631 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -1318,6 +1318,16 @@ impl DQNTrainer { } } + // Q-value explosion guard: halt if Q-values exceed configured clip range + let q_clip_min = self.hyperparams.q_clip_min; + let q_clip_max = self.hyperparams.q_clip_max; + if avg_q < q_clip_min || avg_q > q_clip_max { + return Err(anyhow::anyhow!( + "Q-value explosion: avg_q={:.4} outside clip range [{}, {}]", + avg_q, q_clip_min, q_clip_max + )); + } + // Loss history (epoch-level average) self.safety_loss_history.push_back(avg_loss); if self.safety_loss_history.len() > 30 { diff --git a/crates/ml/tests/dqn_training_pipeline_test.rs b/crates/ml/tests/dqn_training_pipeline_test.rs index 1d38da0f8..b59a21faf 100644 --- a/crates/ml/tests/dqn_training_pipeline_test.rs +++ b/crates/ml/tests/dqn_training_pipeline_test.rs @@ -92,7 +92,6 @@ #![allow(unused_crate_dependencies)] use anyhow::{Context, Result}; -use ml::gpu::profile::GpuProfile; use ml::trainers::dqn::{DQNHyperparameters, DQNTrainer}; use std::path::PathBuf; use std::time::Instant; @@ -109,21 +108,6 @@ fn init_test_tracing() { .try_init(); } -/// Apply GPU-profile-aware scaling to hyperparameters. -/// Uses TOML profile system instead of hardcoded VRAM if/else chains. -/// Caps each value to the profile maximum to prevent OOM while preserving test semantics. -fn scale_for_gpu(hp: &mut DQNHyperparameters) { - let profile = GpuProfile::load(); - hp.buffer_size = hp.buffer_size.min(profile.training.buffer_size); - hp.batch_size = hp.batch_size.min(profile.training.batch_size); - hp.gpu_timesteps_per_episode = hp.gpu_timesteps_per_episode.min(profile.experience.gpu_timesteps_per_episode); - // num_atoms is algorithmic (C51 distribution resolution), not a VRAM knob — don't cap it - info!( - "GPU profile: buffer={}, batch={}, timesteps={}", - hp.buffer_size, hp.batch_size, hp.gpu_timesteps_per_episode, - ); -} - /// Helper: Get path to ES.FUT test data (DBN format) fn get_es_fut_data_dir() -> Result { // CI: TEST_DATA_DIR points to test-data-pvc on H100 @@ -193,11 +177,6 @@ async fn test_dqn_trains_on_es_fut() -> Result<()> { hyperparams.early_stopping_enabled = false; hyperparams.checkpoint_frequency = 1; hyperparams.cql_alpha = 0.0; - hyperparams.batch_size = 32; - hyperparams.buffer_size = 1024; - hyperparams.min_replay_size = 32; - hyperparams.warmup_steps = 0; - hyperparams.replay_buffer_vram_fraction = 0.0; info!(data_dir = %data_dir, checkpoint_dir = %checkpoint_dir.display(), epochs = hyperparams.epochs, "Configuration ready"); @@ -206,7 +185,6 @@ async fn test_dqn_trains_on_es_fut() -> Result<()> { // ======================================================================== info!("ACT: Running DQN training..."); - scale_for_gpu(&mut hyperparams); let mut trainer = DQNTrainer::new(hyperparams.clone())?; let mut checkpoint_saved = false; @@ -316,15 +294,10 @@ async fn test_dqn_loss_decreases() -> Result<()> { // Train for 3 epochs — CI validates gradient flow, not full convergence let mut hyperparams = DQNHyperparameters::conservative(); ml::training_profile::DqnTrainingProfile::load("dqn-smoketest").apply_to(&mut hyperparams); - hyperparams.replay_buffer_vram_fraction = 0.0; - // GPU PER is mandatory for fused CUDA training — do not disable hyperparams.epochs = 3; - hyperparams.batch_size = 64; - hyperparams.learning_rate = 0.001; hyperparams.early_stopping_enabled = false; + hyperparams.cql_alpha = 0.0; - hyperparams.gpu_timesteps_per_episode = 50; - scale_for_gpu(&mut hyperparams); let mut trainer = DQNTrainer::new(hyperparams)?; // Track losses per epoch (would need to modify trainer to expose this) @@ -392,14 +365,11 @@ async fn test_dqn_checkpoint_save_load() -> Result<()> { // Train for 2 epochs and save checkpoint let mut hyperparams = DQNHyperparameters::conservative(); ml::training_profile::DqnTrainingProfile::load("dqn-smoketest").apply_to(&mut hyperparams); - hyperparams.replay_buffer_vram_fraction = 0.0; - // GPU PER is mandatory for fused CUDA training — do not disable hyperparams.epochs = 2; - hyperparams.batch_size = 64; + hyperparams.early_stopping_enabled = false; hyperparams.checkpoint_frequency = 2; + hyperparams.cql_alpha = 0.0; - hyperparams.gpu_timesteps_per_episode = 50; - scale_for_gpu(&mut hyperparams); let mut trainer = DQNTrainer::new(hyperparams)?; let mut saved_checkpoint_path = PathBuf::new(); @@ -463,13 +433,11 @@ async fn test_dqn_q_value_predictions() -> Result<()> { // Train minimal model let mut hyperparams = DQNHyperparameters::conservative(); ml::training_profile::DqnTrainingProfile::load("dqn-smoketest").apply_to(&mut hyperparams); - hyperparams.replay_buffer_vram_fraction = 0.0; - // GPU PER is mandatory for fused CUDA training — do not disable hyperparams.epochs = 2; - hyperparams.batch_size = 32; + hyperparams.early_stopping_enabled = false; + hyperparams.checkpoint_frequency = 1; + hyperparams.cql_alpha = 0.0; - hyperparams.gpu_timesteps_per_episode = 50; - scale_for_gpu(&mut hyperparams); let mut trainer = DQNTrainer::new(hyperparams)?; let metrics = trainer @@ -522,18 +490,13 @@ async fn test_dqn_epsilon_greedy() -> Result<()> { // Configure with high epsilon decay let mut hyperparams = DQNHyperparameters::conservative(); ml::training_profile::DqnTrainingProfile::load("dqn-smoketest").apply_to(&mut hyperparams); - hyperparams.replay_buffer_vram_fraction = 0.0; hyperparams.epochs = 2; + hyperparams.early_stopping_enabled = false; + hyperparams.cql_alpha = 0.0; hyperparams.epsilon_start = 1.0; hyperparams.epsilon_end = 0.01; hyperparams.epsilon_decay = 0.9; // Fast decay - hyperparams.batch_size = 64; - hyperparams.gpu_timesteps_per_episode = 50; - hyperparams.min_replay_size = 50; - hyperparams.warmup_steps = 0; - hyperparams.buffer_size = 5_000; - scale_for_gpu(&mut hyperparams); let mut trainer = DQNTrainer::new(hyperparams)?; let _metrics = trainer @@ -612,8 +575,6 @@ async fn test_dqn_full_production_training() -> Result<()> { hyperparams.checkpoint_frequency = 10; hyperparams.early_stopping_enabled = true; - hyperparams.gpu_timesteps_per_episode = 50; - scale_for_gpu(&mut hyperparams); let mut trainer = DQNTrainer::new(hyperparams.clone())?; let mut epoch_count = 0;