fix: DQNTrainer hard-errors on CUDA init failure, no silent CPU fallback

cuda_if_available() silently fell back to CPU when CUDA init failed,
causing DQNTrainer to crash later with "cuda_stream() called on CPU
device". This was the root cause of the CI "hang" — the test failed
immediately but the error was invisible due to buffered output.

- DQNTrainer::new() now uses MlDevice::cuda(0)? with explicit error
- cuda_if_available() now logs the CUDA error before falling back

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-19 16:57:34 +01:00
parent 04b2cb7849
commit ea92143eda
2 changed files with 14 additions and 3 deletions

View File

@@ -64,7 +64,10 @@ impl MlDevice {
pub fn cuda_if_available(ordinal: usize) -> Self {
match Self::cuda(ordinal) {
Ok(dev) => dev,
Err(_) => MlDevice::Cpu,
Err(e) => {
tracing::warn!("CUDA device {ordinal} unavailable ({e}), falling back to CPU");
MlDevice::Cpu
}
}
}

View File

@@ -118,11 +118,19 @@ impl DQNTrainer {
}
// Use override device if provided (hyperopt shares one CUDA context),
// otherwise auto-detect GPU
// otherwise require CUDA GPU — no CPU fallback in CUDA builds.
let device = if let Some(dev) = override_device {
dev
} else {
MlDevice::cuda_if_available(0)
match MlDevice::cuda(0) {
Ok(dev) => dev,
Err(e) => {
tracing::error!("CUDA device init failed: {e}");
return Err(anyhow::anyhow!(
"DQNTrainer requires CUDA GPU. CUDA init failed: {e}"
));
}
}
};
// Fork a dedicated CudaStream once. All GPU components (experience collector,