fix(ci): disable GPU experience collector in integration tests
The fused NVRTC kernel (branching+C51+NoisyNets+DSR+fill-sim) takes 30+ min to compile at runtime on H100, causing CI tests to hit the 90-minute workflow deadline. Disable enable_gpu_experience_collector in all integration tests that call DQNTrainer::train(). The GPU experience collector is validated by lib tests (gpu_residency). Training forward/backward/optimizer still runs on CUDA. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -130,6 +130,8 @@ async fn test_early_stopping_terminates_with_error() {
|
||||
hyperparams.gradient_collapse_patience = 3; // Trigger after 3 consecutive collapses
|
||||
hyperparams.checkpoint_frequency = 100; // Disable periodic checkpoints (focus on early stop)
|
||||
hyperparams.batch_size = 32; // Small batch for faster testing
|
||||
// Disable GPU experience collector to avoid 30+ min NVRTC compilation in CI.
|
||||
hyperparams.enable_gpu_experience_collector = false;
|
||||
|
||||
let mut trainer = DQNTrainer::new(hyperparams.clone())
|
||||
.expect("Failed to create DQN trainer");
|
||||
@@ -200,6 +202,8 @@ async fn test_gradient_collapse_propagates_error() {
|
||||
hyperparams.gradient_collapse_multiplier = 100.0;
|
||||
hyperparams.gradient_collapse_patience = 3; // 3 consecutive checks
|
||||
hyperparams.batch_size = 32;
|
||||
// Disable GPU experience collector to avoid 30+ min NVRTC compilation in CI.
|
||||
hyperparams.enable_gpu_experience_collector = false;
|
||||
|
||||
let mut trainer = DQNTrainer::new(hyperparams.clone())
|
||||
.expect("Failed to create DQN trainer");
|
||||
@@ -250,6 +254,8 @@ async fn test_healthy_training_completes_successfully() {
|
||||
hyperparams.gradient_collapse_patience = 5;
|
||||
hyperparams.batch_size = 59;
|
||||
hyperparams.gamma = 0.961042;
|
||||
// Disable GPU experience collector to avoid 30+ min NVRTC compilation in CI.
|
||||
hyperparams.enable_gpu_experience_collector = false;
|
||||
|
||||
let mut trainer = DQNTrainer::new(hyperparams.clone())
|
||||
.expect("Failed to create DQN trainer");
|
||||
|
||||
@@ -168,6 +168,8 @@ async fn test_dqn_trains_on_es_fut() -> Result<()> {
|
||||
hyperparams.epsilon_end = 0.05;
|
||||
hyperparams.checkpoint_frequency = 5;
|
||||
hyperparams.early_stopping_enabled = false; // Test all 10 epochs
|
||||
// Disable GPU experience collector to avoid 30+ min NVRTC compilation in CI.
|
||||
hyperparams.enable_gpu_experience_collector = false;
|
||||
|
||||
println!(" ✅ Configuration ready");
|
||||
println!(" 📂 Data directory: {}", data_dir);
|
||||
@@ -299,6 +301,8 @@ async fn test_dqn_loss_decreases() -> Result<()> {
|
||||
hyperparams.batch_size = 64;
|
||||
hyperparams.learning_rate = 0.001;
|
||||
hyperparams.early_stopping_enabled = false;
|
||||
// Disable GPU experience collector to avoid 30+ min NVRTC compilation in CI.
|
||||
hyperparams.enable_gpu_experience_collector = false;
|
||||
|
||||
let mut trainer = DQNTrainer::new(hyperparams)?;
|
||||
|
||||
@@ -371,6 +375,8 @@ async fn test_dqn_checkpoint_save_load() -> Result<()> {
|
||||
hyperparams.epochs = 5;
|
||||
hyperparams.batch_size = 64;
|
||||
hyperparams.checkpoint_frequency = 5;
|
||||
// Disable GPU experience collector to avoid 30+ min NVRTC compilation in CI.
|
||||
hyperparams.enable_gpu_experience_collector = false;
|
||||
|
||||
let mut trainer = DQNTrainer::new(hyperparams)?;
|
||||
|
||||
@@ -436,6 +442,8 @@ async fn test_dqn_q_value_predictions() -> Result<()> {
|
||||
let mut hyperparams = DQNHyperparameters::conservative();
|
||||
hyperparams.epochs = 5;
|
||||
hyperparams.batch_size = 32;
|
||||
// Disable GPU experience collector to avoid 30+ min NVRTC compilation in CI.
|
||||
hyperparams.enable_gpu_experience_collector = false;
|
||||
|
||||
let mut trainer = DQNTrainer::new(hyperparams)?;
|
||||
|
||||
@@ -492,6 +500,8 @@ async fn test_dqn_epsilon_greedy() -> Result<()> {
|
||||
hyperparams.epsilon_start = 1.0;
|
||||
hyperparams.epsilon_end = 0.01;
|
||||
hyperparams.epsilon_decay = 0.9; // Fast decay
|
||||
// Disable GPU experience collector to avoid 30+ min NVRTC compilation in CI.
|
||||
hyperparams.enable_gpu_experience_collector = false;
|
||||
|
||||
let mut trainer = DQNTrainer::new(hyperparams)?;
|
||||
|
||||
@@ -567,6 +577,8 @@ async fn test_dqn_full_production_training() -> Result<()> {
|
||||
hyperparams.epsilon_decay = 0.995;
|
||||
hyperparams.checkpoint_frequency = 10;
|
||||
hyperparams.early_stopping_enabled = true;
|
||||
// Disable GPU experience collector to avoid 30+ min NVRTC compilation in CI.
|
||||
hyperparams.enable_gpu_experience_collector = false;
|
||||
|
||||
let mut trainer = DQNTrainer::new(hyperparams.clone())?;
|
||||
|
||||
|
||||
@@ -136,6 +136,9 @@ async fn test_dqn_training_smoke() -> Result<()> {
|
||||
hyperparams.early_stopping_enabled = true;
|
||||
hyperparams.min_epochs_before_stopping = 25; // > 20 epochs = won't trigger
|
||||
hyperparams.checkpoint_frequency = 10;
|
||||
// Disable GPU experience collector to avoid 30+ min NVRTC compilation of the
|
||||
// fused kernel. GPU training (forward/backward/optimizer) still runs on CUDA.
|
||||
hyperparams.enable_gpu_experience_collector = false;
|
||||
|
||||
// === ACT: Train ===
|
||||
let mut trainer = DQNTrainer::new(hyperparams)?;
|
||||
|
||||
@@ -821,6 +821,11 @@ async fn smoke_e2e_dqn_training_loop() {
|
||||
hyperparams.use_branching = true;
|
||||
hyperparams.warmup_steps = 0; // No warmup — small dataset
|
||||
hyperparams.min_replay_size = 50; // Low threshold for smoke test
|
||||
// Disable GPU experience collector to avoid 30+ min NVRTC compilation of the
|
||||
// fused branching+C51+NoisyNets kernel. The GPU experience collector is validated
|
||||
// by lib tests (gpu_residency). This test uses the batched CPU experience collection
|
||||
// path, which still runs forward/backward/optimizer on GPU.
|
||||
hyperparams.enable_gpu_experience_collector = false;
|
||||
|
||||
let checkpoint_dir = tempfile::tempdir().expect("Failed to create temp dir");
|
||||
let mut trainer = DQNTrainer::new(hyperparams).expect("Failed to create DQN trainer");
|
||||
|
||||
Reference in New Issue
Block a user