fix(ci): disable GPU experience collector in integration tests

The fused NVRTC kernel (branching+C51+NoisyNets+DSR+fill-sim) takes
30+ min to compile at runtime on H100, causing CI tests to hit the
90-minute workflow deadline. Disable enable_gpu_experience_collector
in all integration tests that call DQNTrainer::train(). The GPU
experience collector is validated by lib tests (gpu_residency).
Training forward/backward/optimizer still runs on CUDA.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-13 14:59:40 +01:00
parent fefb9e3185
commit 12df39ad51
4 changed files with 26 additions and 0 deletions

View File

@@ -130,6 +130,8 @@ async fn test_early_stopping_terminates_with_error() {
hyperparams.gradient_collapse_patience = 3; // Trigger after 3 consecutive collapses
hyperparams.checkpoint_frequency = 100; // Disable periodic checkpoints (focus on early stop)
hyperparams.batch_size = 32; // Small batch for faster testing
// Disable GPU experience collector to avoid 30+ min NVRTC compilation in CI.
hyperparams.enable_gpu_experience_collector = false;
let mut trainer = DQNTrainer::new(hyperparams.clone())
.expect("Failed to create DQN trainer");
@@ -200,6 +202,8 @@ async fn test_gradient_collapse_propagates_error() {
hyperparams.gradient_collapse_multiplier = 100.0;
hyperparams.gradient_collapse_patience = 3; // 3 consecutive checks
hyperparams.batch_size = 32;
// Disable GPU experience collector to avoid 30+ min NVRTC compilation in CI.
hyperparams.enable_gpu_experience_collector = false;
let mut trainer = DQNTrainer::new(hyperparams.clone())
.expect("Failed to create DQN trainer");
@@ -250,6 +254,8 @@ async fn test_healthy_training_completes_successfully() {
hyperparams.gradient_collapse_patience = 5;
hyperparams.batch_size = 59;
hyperparams.gamma = 0.961042;
// Disable GPU experience collector to avoid 30+ min NVRTC compilation in CI.
hyperparams.enable_gpu_experience_collector = false;
let mut trainer = DQNTrainer::new(hyperparams.clone())
.expect("Failed to create DQN trainer");

View File

@@ -168,6 +168,8 @@ async fn test_dqn_trains_on_es_fut() -> Result<()> {
hyperparams.epsilon_end = 0.05;
hyperparams.checkpoint_frequency = 5;
hyperparams.early_stopping_enabled = false; // Test all 10 epochs
// Disable GPU experience collector to avoid 30+ min NVRTC compilation in CI.
hyperparams.enable_gpu_experience_collector = false;
println!(" ✅ Configuration ready");
println!(" 📂 Data directory: {}", data_dir);
@@ -299,6 +301,8 @@ async fn test_dqn_loss_decreases() -> Result<()> {
hyperparams.batch_size = 64;
hyperparams.learning_rate = 0.001;
hyperparams.early_stopping_enabled = false;
// Disable GPU experience collector to avoid 30+ min NVRTC compilation in CI.
hyperparams.enable_gpu_experience_collector = false;
let mut trainer = DQNTrainer::new(hyperparams)?;
@@ -371,6 +375,8 @@ async fn test_dqn_checkpoint_save_load() -> Result<()> {
hyperparams.epochs = 5;
hyperparams.batch_size = 64;
hyperparams.checkpoint_frequency = 5;
// Disable GPU experience collector to avoid 30+ min NVRTC compilation in CI.
hyperparams.enable_gpu_experience_collector = false;
let mut trainer = DQNTrainer::new(hyperparams)?;
@@ -436,6 +442,8 @@ async fn test_dqn_q_value_predictions() -> Result<()> {
let mut hyperparams = DQNHyperparameters::conservative();
hyperparams.epochs = 5;
hyperparams.batch_size = 32;
// Disable GPU experience collector to avoid 30+ min NVRTC compilation in CI.
hyperparams.enable_gpu_experience_collector = false;
let mut trainer = DQNTrainer::new(hyperparams)?;
@@ -492,6 +500,8 @@ async fn test_dqn_epsilon_greedy() -> Result<()> {
hyperparams.epsilon_start = 1.0;
hyperparams.epsilon_end = 0.01;
hyperparams.epsilon_decay = 0.9; // Fast decay
// Disable GPU experience collector to avoid 30+ min NVRTC compilation in CI.
hyperparams.enable_gpu_experience_collector = false;
let mut trainer = DQNTrainer::new(hyperparams)?;
@@ -567,6 +577,8 @@ async fn test_dqn_full_production_training() -> Result<()> {
hyperparams.epsilon_decay = 0.995;
hyperparams.checkpoint_frequency = 10;
hyperparams.early_stopping_enabled = true;
// Disable GPU experience collector to avoid 30+ min NVRTC compilation in CI.
hyperparams.enable_gpu_experience_collector = false;
let mut trainer = DQNTrainer::new(hyperparams.clone())?;

View File

@@ -136,6 +136,9 @@ async fn test_dqn_training_smoke() -> Result<()> {
hyperparams.early_stopping_enabled = true;
hyperparams.min_epochs_before_stopping = 25; // > 20 epochs = won't trigger
hyperparams.checkpoint_frequency = 10;
// Disable GPU experience collector to avoid 30+ min NVRTC compilation of the
// fused kernel. GPU training (forward/backward/optimizer) still runs on CUDA.
hyperparams.enable_gpu_experience_collector = false;
// === ACT: Train ===
let mut trainer = DQNTrainer::new(hyperparams)?;

View File

@@ -821,6 +821,11 @@ async fn smoke_e2e_dqn_training_loop() {
hyperparams.use_branching = true;
hyperparams.warmup_steps = 0; // No warmup — small dataset
hyperparams.min_replay_size = 50; // Low threshold for smoke test
// Disable GPU experience collector to avoid 30+ min NVRTC compilation of the
// fused branching+C51+NoisyNets kernel. The GPU experience collector is validated
// by lib tests (gpu_residency). This test uses the batched CPU experience collection
// path, which still runs forward/backward/optimizer on GPU.
hyperparams.enable_gpu_experience_collector = false;
let checkpoint_dir = tempfile::tempdir().expect("Failed to create temp dir");
let mut trainer = DQNTrainer::new(hyperparams).expect("Failed to create DQN trainer");