From ea92143eda90465d5d305260ada53d3262b83cbd Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Thu, 19 Mar 2026 16:57:34 +0100 Subject: [PATCH] fix: DQNTrainer hard-errors on CUDA init failure, no silent CPU fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cuda_if_available() silently fell back to CPU when CUDA init failed, causing DQNTrainer to crash later with "cuda_stream() called on CPU device". This was the root cause of the CI "hang" — the test failed immediately but the error was invisible due to buffered output. - DQNTrainer::new() now uses MlDevice::cuda(0)? with explicit error - cuda_if_available() now logs the CUDA error before falling back Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/ml-core/src/device.rs | 5 ++++- crates/ml/src/trainers/dqn/trainer/constructor.rs | 12 ++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/crates/ml-core/src/device.rs b/crates/ml-core/src/device.rs index c5436e6ab..66ae20cb6 100644 --- a/crates/ml-core/src/device.rs +++ b/crates/ml-core/src/device.rs @@ -64,7 +64,10 @@ impl MlDevice { pub fn cuda_if_available(ordinal: usize) -> Self { match Self::cuda(ordinal) { Ok(dev) => dev, - Err(_) => MlDevice::Cpu, + Err(e) => { + tracing::warn!("CUDA device {ordinal} unavailable ({e}), falling back to CPU"); + MlDevice::Cpu + } } } diff --git a/crates/ml/src/trainers/dqn/trainer/constructor.rs b/crates/ml/src/trainers/dqn/trainer/constructor.rs index 107b76f3f..e8fc349c7 100644 --- a/crates/ml/src/trainers/dqn/trainer/constructor.rs +++ b/crates/ml/src/trainers/dqn/trainer/constructor.rs @@ -118,11 +118,19 @@ impl DQNTrainer { } // Use override device if provided (hyperopt shares one CUDA context), - // otherwise auto-detect GPU + // otherwise require CUDA GPU — no CPU fallback in CUDA builds. let device = if let Some(dev) = override_device { dev } else { - MlDevice::cuda_if_available(0) + match MlDevice::cuda(0) { + Ok(dev) => dev, + Err(e) => { + tracing::error!("CUDA device init failed: {e}"); + return Err(anyhow::anyhow!( + "DQNTrainer requires CUDA GPU. CUDA init failed: {e}" + )); + } + } }; // Fork a dedicated CudaStream once. All GPU components (experience collector,