gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: profile is single source of truth, q_clip wired as guard, fewer walk-forward folds

Browse Source 
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-04-04 15:25:02 +02:00
parent 3b6c9b5e4d
commit 4aceea1699
3 changed files with 21 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
config/training/dqn-smoketest.toml
View File

@@ -119,3 +119,6 @@ loss_aversion = 1.5
q_gap_threshold = 0.1
w_dd = 1.0
dd_threshold = 0.02
[walk_forward]
step_fraction = 0.25

10
crates/ml/src/trainers/dqn/trainer/training_loop.rs
View File

@@ -1318,6 +1318,16 @@ impl DQNTrainer {
}
}
// Q-value explosion guard: halt if Q-values exceed configured clip range
let q_clip_min = self.hyperparams.q_clip_min;
let q_clip_max = self.hyperparams.q_clip_max;
if avg_q < q_clip_min || avg_q > q_clip_max {
return Err(anyhow::anyhow!(
"Q-value explosion: avg_q={:.4} outside clip range [{}, {}]",
avg_q, q_clip_min, q_clip_max
));
}
// Loss history (epoch-level average)
self.safety_loss_history.push_back(avg_loss);
if self.safety_loss_history.len() > 30 {

55
crates/ml/tests/dqn_training_pipeline_test.rs
View File

@@ -92,7 +92,6 @@
#![allow(unused_crate_dependencies)]
use anyhow::{Context, Result};
use ml::gpu::profile::GpuProfile;
use ml::trainers::dqn::{DQNHyperparameters, DQNTrainer};
use std::path::PathBuf;
use std::time::Instant;
@@ -109,21 +108,6 @@ fn init_test_tracing() {
.try_init();
}
/// Apply GPU-profile-aware scaling to hyperparameters.
/// Uses TOML profile system instead of hardcoded VRAM if/else chains.
/// Caps each value to the profile maximum to prevent OOM while preserving test semantics.
fn scale_for_gpu(hp: &mut DQNHyperparameters) {
let profile = GpuProfile::load();
hp.buffer_size = hp.buffer_size.min(profile.training.buffer_size);
hp.batch_size = hp.batch_size.min(profile.training.batch_size);
hp.gpu_timesteps_per_episode = hp.gpu_timesteps_per_episode.min(profile.experience.gpu_timesteps_per_episode);
// num_atoms is algorithmic (C51 distribution resolution), not a VRAM knob — don't cap it
info!(
"GPU profile: buffer={}, batch={}, timesteps={}",
hp.buffer_size, hp.batch_size, hp.gpu_timesteps_per_episode,
);
}
/// Helper: Get path to ES.FUT test data (DBN format)
fn get_es_fut_data_dir() -> Result<String> {
// CI: TEST_DATA_DIR points to test-data-pvc on H100
@@ -193,11 +177,6 @@ async fn test_dqn_trains_on_es_fut() -> Result<()> {
hyperparams.early_stopping_enabled = false;
hyperparams.checkpoint_frequency = 1;
hyperparams.cql_alpha = 0.0;
hyperparams.batch_size = 32;
hyperparams.buffer_size = 1024;
hyperparams.min_replay_size = 32;
hyperparams.warmup_steps = 0;
hyperparams.replay_buffer_vram_fraction = 0.0;
info!(data_dir = %data_dir, checkpoint_dir = %checkpoint_dir.display(), epochs = hyperparams.epochs, "Configuration ready");
@@ -206,7 +185,6 @@ async fn test_dqn_trains_on_es_fut() -> Result<()> {
// ========================================================================
info!("ACT: Running DQN training...");
scale_for_gpu(&mut hyperparams);
let mut trainer = DQNTrainer::new(hyperparams.clone())?;
let mut checkpoint_saved = false;
@@ -316,15 +294,10 @@ async fn test_dqn_loss_decreases() -> Result<()> {
// Train for 3 epochs — CI validates gradient flow, not full convergence
let mut hyperparams = DQNHyperparameters::conservative();
ml::training_profile::DqnTrainingProfile::load("dqn-smoketest").apply_to(&mut hyperparams);
hyperparams.replay_buffer_vram_fraction = 0.0;
// GPU PER is mandatory for fused CUDA training — do not disable
hyperparams.epochs = 3;
hyperparams.batch_size = 64;
hyperparams.learning_rate = 0.001;
hyperparams.early_stopping_enabled = false;
hyperparams.cql_alpha = 0.0;
hyperparams.gpu_timesteps_per_episode = 50;
scale_for_gpu(&mut hyperparams);
let mut trainer = DQNTrainer::new(hyperparams)?;
// Track losses per epoch (would need to modify trainer to expose this)
@@ -392,14 +365,11 @@ async fn test_dqn_checkpoint_save_load() -> Result<()> {
// Train for 2 epochs and save checkpoint
let mut hyperparams = DQNHyperparameters::conservative();
ml::training_profile::DqnTrainingProfile::load("dqn-smoketest").apply_to(&mut hyperparams);
hyperparams.replay_buffer_vram_fraction = 0.0;
// GPU PER is mandatory for fused CUDA training — do not disable
hyperparams.epochs = 2;
hyperparams.batch_size = 64;
hyperparams.early_stopping_enabled = false;
hyperparams.checkpoint_frequency = 2;
hyperparams.cql_alpha = 0.0;
hyperparams.gpu_timesteps_per_episode = 50;
scale_for_gpu(&mut hyperparams);
let mut trainer = DQNTrainer::new(hyperparams)?;
let mut saved_checkpoint_path = PathBuf::new();
@@ -463,13 +433,11 @@ async fn test_dqn_q_value_predictions() -> Result<()> {
// Train minimal model
let mut hyperparams = DQNHyperparameters::conservative();
ml::training_profile::DqnTrainingProfile::load("dqn-smoketest").apply_to(&mut hyperparams);
hyperparams.replay_buffer_vram_fraction = 0.0;
// GPU PER is mandatory for fused CUDA training — do not disable
hyperparams.epochs = 2;
hyperparams.batch_size = 32;
hyperparams.early_stopping_enabled = false;
hyperparams.checkpoint_frequency = 1;
hyperparams.cql_alpha = 0.0;
hyperparams.gpu_timesteps_per_episode = 50;
scale_for_gpu(&mut hyperparams);
let mut trainer = DQNTrainer::new(hyperparams)?;
let metrics = trainer
@@ -522,18 +490,13 @@ async fn test_dqn_epsilon_greedy() -> Result<()> {
// Configure with high epsilon decay
let mut hyperparams = DQNHyperparameters::conservative();
ml::training_profile::DqnTrainingProfile::load("dqn-smoketest").apply_to(&mut hyperparams);
hyperparams.replay_buffer_vram_fraction = 0.0;
hyperparams.epochs = 2;
hyperparams.early_stopping_enabled = false;
hyperparams.cql_alpha = 0.0;
hyperparams.epsilon_start = 1.0;
hyperparams.epsilon_end = 0.01;
hyperparams.epsilon_decay = 0.9; // Fast decay
hyperparams.batch_size = 64;
hyperparams.gpu_timesteps_per_episode = 50;
hyperparams.min_replay_size = 50;
hyperparams.warmup_steps = 0;
hyperparams.buffer_size = 5_000;
scale_for_gpu(&mut hyperparams);
let mut trainer = DQNTrainer::new(hyperparams)?;
let _metrics = trainer
@@ -612,8 +575,6 @@ async fn test_dqn_full_production_training() -> Result<()> {
hyperparams.checkpoint_frequency = 10;
hyperparams.early_stopping_enabled = true;
hyperparams.gpu_timesteps_per_episode = 50;
scale_for_gpu(&mut hyperparams);
let mut trainer = DQNTrainer::new(hyperparams.clone())?;
let mut epoch_count = 0;