fix: DQNTrainer hard-errors on CUDA init failure, no silent CPU fallback
cuda_if_available() silently fell back to CPU when CUDA init failed, causing DQNTrainer to crash later with "cuda_stream() called on CPU device". This was the root cause of the CI "hang" — the test failed immediately but the error was invisible due to buffered output. - DQNTrainer::new() now uses MlDevice::cuda(0)? with explicit error - cuda_if_available() now logs the CUDA error before falling back Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -64,7 +64,10 @@ impl MlDevice {
|
||||
pub fn cuda_if_available(ordinal: usize) -> Self {
|
||||
match Self::cuda(ordinal) {
|
||||
Ok(dev) => dev,
|
||||
Err(_) => MlDevice::Cpu,
|
||||
Err(e) => {
|
||||
tracing::warn!("CUDA device {ordinal} unavailable ({e}), falling back to CPU");
|
||||
MlDevice::Cpu
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -118,11 +118,19 @@ impl DQNTrainer {
|
||||
}
|
||||
|
||||
// Use override device if provided (hyperopt shares one CUDA context),
|
||||
// otherwise auto-detect GPU
|
||||
// otherwise require CUDA GPU — no CPU fallback in CUDA builds.
|
||||
let device = if let Some(dev) = override_device {
|
||||
dev
|
||||
} else {
|
||||
MlDevice::cuda_if_available(0)
|
||||
match MlDevice::cuda(0) {
|
||||
Ok(dev) => dev,
|
||||
Err(e) => {
|
||||
tracing::error!("CUDA device init failed: {e}");
|
||||
return Err(anyhow::anyhow!(
|
||||
"DQNTrainer requires CUDA GPU. CUDA init failed: {e}"
|
||||
));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Fork a dedicated CudaStream once. All GPU components (experience collector,
|
||||
|
||||
Reference in New Issue
Block a user