From 12df39ad519ee491b62c3d5716da45baba5cd378 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 13 Mar 2026 14:59:40 +0100 Subject: [PATCH] fix(ci): disable GPU experience collector in integration tests The fused NVRTC kernel (branching+C51+NoisyNets+DSR+fill-sim) takes 30+ min to compile at runtime on H100, causing CI tests to hit the 90-minute workflow deadline. Disable enable_gpu_experience_collector in all integration tests that call DQNTrainer::train(). The GPU experience collector is validated by lib tests (gpu_residency). Training forward/backward/optimizer still runs on CUDA. Co-Authored-By: Claude Opus 4.6 --- .../ml/tests/dqn_early_stopping_termination_test.rs | 6 ++++++ crates/ml/tests/dqn_training_pipeline_test.rs | 12 ++++++++++++ crates/ml/tests/dqn_training_smoke_test.rs | 3 +++ crates/ml/tests/smoke_test_real_data.rs | 5 +++++ 4 files changed, 26 insertions(+) diff --git a/crates/ml/tests/dqn_early_stopping_termination_test.rs b/crates/ml/tests/dqn_early_stopping_termination_test.rs index 94143ba42..71d5c989c 100644 --- a/crates/ml/tests/dqn_early_stopping_termination_test.rs +++ b/crates/ml/tests/dqn_early_stopping_termination_test.rs @@ -130,6 +130,8 @@ async fn test_early_stopping_terminates_with_error() { hyperparams.gradient_collapse_patience = 3; // Trigger after 3 consecutive collapses hyperparams.checkpoint_frequency = 100; // Disable periodic checkpoints (focus on early stop) hyperparams.batch_size = 32; // Small batch for faster testing + // Disable GPU experience collector to avoid 30+ min NVRTC compilation in CI. + hyperparams.enable_gpu_experience_collector = false; let mut trainer = DQNTrainer::new(hyperparams.clone()) .expect("Failed to create DQN trainer"); @@ -200,6 +202,8 @@ async fn test_gradient_collapse_propagates_error() { hyperparams.gradient_collapse_multiplier = 100.0; hyperparams.gradient_collapse_patience = 3; // 3 consecutive checks hyperparams.batch_size = 32; + // Disable GPU experience collector to avoid 30+ min NVRTC compilation in CI. + hyperparams.enable_gpu_experience_collector = false; let mut trainer = DQNTrainer::new(hyperparams.clone()) .expect("Failed to create DQN trainer"); @@ -250,6 +254,8 @@ async fn test_healthy_training_completes_successfully() { hyperparams.gradient_collapse_patience = 5; hyperparams.batch_size = 59; hyperparams.gamma = 0.961042; + // Disable GPU experience collector to avoid 30+ min NVRTC compilation in CI. + hyperparams.enable_gpu_experience_collector = false; let mut trainer = DQNTrainer::new(hyperparams.clone()) .expect("Failed to create DQN trainer"); diff --git a/crates/ml/tests/dqn_training_pipeline_test.rs b/crates/ml/tests/dqn_training_pipeline_test.rs index 2b0fd7a59..2ec1448fd 100644 --- a/crates/ml/tests/dqn_training_pipeline_test.rs +++ b/crates/ml/tests/dqn_training_pipeline_test.rs @@ -168,6 +168,8 @@ async fn test_dqn_trains_on_es_fut() -> Result<()> { hyperparams.epsilon_end = 0.05; hyperparams.checkpoint_frequency = 5; hyperparams.early_stopping_enabled = false; // Test all 10 epochs + // Disable GPU experience collector to avoid 30+ min NVRTC compilation in CI. + hyperparams.enable_gpu_experience_collector = false; println!(" ✅ Configuration ready"); println!(" 📂 Data directory: {}", data_dir); @@ -299,6 +301,8 @@ async fn test_dqn_loss_decreases() -> Result<()> { hyperparams.batch_size = 64; hyperparams.learning_rate = 0.001; hyperparams.early_stopping_enabled = false; + // Disable GPU experience collector to avoid 30+ min NVRTC compilation in CI. + hyperparams.enable_gpu_experience_collector = false; let mut trainer = DQNTrainer::new(hyperparams)?; @@ -371,6 +375,8 @@ async fn test_dqn_checkpoint_save_load() -> Result<()> { hyperparams.epochs = 5; hyperparams.batch_size = 64; hyperparams.checkpoint_frequency = 5; + // Disable GPU experience collector to avoid 30+ min NVRTC compilation in CI. + hyperparams.enable_gpu_experience_collector = false; let mut trainer = DQNTrainer::new(hyperparams)?; @@ -436,6 +442,8 @@ async fn test_dqn_q_value_predictions() -> Result<()> { let mut hyperparams = DQNHyperparameters::conservative(); hyperparams.epochs = 5; hyperparams.batch_size = 32; + // Disable GPU experience collector to avoid 30+ min NVRTC compilation in CI. + hyperparams.enable_gpu_experience_collector = false; let mut trainer = DQNTrainer::new(hyperparams)?; @@ -492,6 +500,8 @@ async fn test_dqn_epsilon_greedy() -> Result<()> { hyperparams.epsilon_start = 1.0; hyperparams.epsilon_end = 0.01; hyperparams.epsilon_decay = 0.9; // Fast decay + // Disable GPU experience collector to avoid 30+ min NVRTC compilation in CI. + hyperparams.enable_gpu_experience_collector = false; let mut trainer = DQNTrainer::new(hyperparams)?; @@ -567,6 +577,8 @@ async fn test_dqn_full_production_training() -> Result<()> { hyperparams.epsilon_decay = 0.995; hyperparams.checkpoint_frequency = 10; hyperparams.early_stopping_enabled = true; + // Disable GPU experience collector to avoid 30+ min NVRTC compilation in CI. + hyperparams.enable_gpu_experience_collector = false; let mut trainer = DQNTrainer::new(hyperparams.clone())?; diff --git a/crates/ml/tests/dqn_training_smoke_test.rs b/crates/ml/tests/dqn_training_smoke_test.rs index 95bf829a9..2b235ec19 100644 --- a/crates/ml/tests/dqn_training_smoke_test.rs +++ b/crates/ml/tests/dqn_training_smoke_test.rs @@ -136,6 +136,9 @@ async fn test_dqn_training_smoke() -> Result<()> { hyperparams.early_stopping_enabled = true; hyperparams.min_epochs_before_stopping = 25; // > 20 epochs = won't trigger hyperparams.checkpoint_frequency = 10; + // Disable GPU experience collector to avoid 30+ min NVRTC compilation of the + // fused kernel. GPU training (forward/backward/optimizer) still runs on CUDA. + hyperparams.enable_gpu_experience_collector = false; // === ACT: Train === let mut trainer = DQNTrainer::new(hyperparams)?; diff --git a/crates/ml/tests/smoke_test_real_data.rs b/crates/ml/tests/smoke_test_real_data.rs index 0eb886049..9f30db68f 100644 --- a/crates/ml/tests/smoke_test_real_data.rs +++ b/crates/ml/tests/smoke_test_real_data.rs @@ -821,6 +821,11 @@ async fn smoke_e2e_dqn_training_loop() { hyperparams.use_branching = true; hyperparams.warmup_steps = 0; // No warmup — small dataset hyperparams.min_replay_size = 50; // Low threshold for smoke test + // Disable GPU experience collector to avoid 30+ min NVRTC compilation of the + // fused branching+C51+NoisyNets kernel. The GPU experience collector is validated + // by lib tests (gpu_residency). This test uses the batched CPU experience collection + // path, which still runs forward/backward/optimizer on GPU. + hyperparams.enable_gpu_experience_collector = false; let checkpoint_dir = tempfile::tempdir().expect("Failed to create temp dir"); let mut trainer = DQNTrainer::new(hyperparams).expect("Failed to create DQN trainer");