From e0e1e2fff3485e9cc0c363f1b6527e960cdd7ce2 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 13 Mar 2026 00:23:19 +0100 Subject: [PATCH] fix(cuda): eliminate CPU fallbacks in all 10 supervised/RL model adapters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convert Device::Cpu fallback patterns to hard errors across all hyperopt adapters and the Mamba2 trainer. On H100, CUDA must be available — silent CPU fallback runs at 1/10th throughput. Models hardened: TFT, Mamba2, TGGN, TLOB, Liquid, KAN, xLSTM, Diffusion, ContinuousPPO (hyperopt adapters) + Mamba2 (trainer). Pattern: Device::new_cuda(0).unwrap_or_else(|e| { warn!(...); Cpu }) → Device::new_cuda(0).map_err(|e| MLError::ConfigError(...))? Co-Authored-By: Claude Opus 4.6 --- .../ml/src/hyperopt/adapters/continuous_ppo.rs | 9 +++------ crates/ml/src/hyperopt/adapters/diffusion.rs | 6 ++---- crates/ml/src/hyperopt/adapters/kan.rs | 6 ++---- crates/ml/src/hyperopt/adapters/liquid.rs | 6 ++---- crates/ml/src/hyperopt/adapters/mamba2.rs | 6 ++---- crates/ml/src/hyperopt/adapters/tft.rs | 6 ++---- crates/ml/src/hyperopt/adapters/tggn.rs | 6 ++---- crates/ml/src/hyperopt/adapters/tlob.rs | 6 ++---- crates/ml/src/hyperopt/adapters/xlstm.rs | 6 ++---- crates/ml/src/trainers/mamba2.rs | 16 ++++------------ 10 files changed, 23 insertions(+), 50 deletions(-) diff --git a/crates/ml/src/hyperopt/adapters/continuous_ppo.rs b/crates/ml/src/hyperopt/adapters/continuous_ppo.rs index 1401bf32e..b8ef94ed1 100644 --- a/crates/ml/src/hyperopt/adapters/continuous_ppo.rs +++ b/crates/ml/src/hyperopt/adapters/continuous_ppo.rs @@ -35,7 +35,7 @@ use candle_core::Device; use serde::{Deserialize, Serialize}; use std::fs::OpenOptions; use std::io::Write as IoWrite; -use tracing::{info, warn}; +use tracing::info; use crate::hyperopt::paths::TrainingPaths; use crate::hyperopt::traits::{HardwareBudget, HyperparameterOptimizable, ParameterSpace}; @@ -285,11 +285,8 @@ impl ContinuousPPOTrainer { .into()); } - // Initialize device (CUDA preferred, CPU fallback) - let device = Device::new_cuda(0).unwrap_or_else(|e| { - warn!("CUDA unavailable ({}), falling back to CPU", e); - Device::Cpu - }); + let device = Device::new_cuda(0) + .map_err(|e| MLError::ConfigError(format!("CUDA GPU required for ContinuousPPO hyperopt: {}", e)))?; info!("Continuous PPO Trainer initialized:"); info!(" Parquet file: {}", parquet_file.display()); diff --git a/crates/ml/src/hyperopt/adapters/diffusion.rs b/crates/ml/src/hyperopt/adapters/diffusion.rs index fc97b3f64..5f6f3df8b 100644 --- a/crates/ml/src/hyperopt/adapters/diffusion.rs +++ b/crates/ml/src/hyperopt/adapters/diffusion.rs @@ -207,10 +207,8 @@ impl DiffusionTrainer { return Err(MLError::ConfigError(format!("Data directory not found: {}", data_dir.display()))); } - let device = Device::new_cuda(0).unwrap_or_else(|e| { - warn!("CUDA unavailable ({}), falling back to CPU", e); - Device::Cpu - }); + let device = Device::new_cuda(0) + .map_err(|e| MLError::ConfigError(format!("CUDA GPU required for Diffusion hyperopt: {}", e)))?; info!( "Diffusion Trainer initialized: Device={:?}, Data={}, Epochs={}", diff --git a/crates/ml/src/hyperopt/adapters/kan.rs b/crates/ml/src/hyperopt/adapters/kan.rs index 8d5b5b373..682179140 100644 --- a/crates/ml/src/hyperopt/adapters/kan.rs +++ b/crates/ml/src/hyperopt/adapters/kan.rs @@ -196,10 +196,8 @@ impl KANTrainer { return Err(MLError::ConfigError(format!("Data directory not found: {}", data_dir.display()))); } - let device = Device::new_cuda(0).unwrap_or_else(|e| { - warn!("CUDA unavailable ({}), falling back to CPU", e); - Device::Cpu - }); + let device = Device::new_cuda(0) + .map_err(|e| MLError::ConfigError(format!("CUDA GPU required for KAN hyperopt: {}", e)))?; info!( "KAN Trainer initialized: Device={:?}, Data={}, Epochs={}", diff --git a/crates/ml/src/hyperopt/adapters/liquid.rs b/crates/ml/src/hyperopt/adapters/liquid.rs index 08acd68c3..28e86349e 100644 --- a/crates/ml/src/hyperopt/adapters/liquid.rs +++ b/crates/ml/src/hyperopt/adapters/liquid.rs @@ -258,10 +258,8 @@ impl LiquidTrainer { return Err(MLError::ConfigError(format!("Data directory not found: {}", data_dir.display()))); } - let device = Device::new_cuda(0).unwrap_or_else(|e| { - warn!("CUDA unavailable ({}), falling back to CPU", e); - Device::Cpu - }); + let device = Device::new_cuda(0) + .map_err(|e| MLError::ConfigError(format!("CUDA GPU required for Liquid hyperopt: {}", e)))?; info!( "Liquid Trainer initialized: Device={:?}, Data={}, Epochs={}", diff --git a/crates/ml/src/hyperopt/adapters/mamba2.rs b/crates/ml/src/hyperopt/adapters/mamba2.rs index 50c0752fe..ed1ca30b1 100644 --- a/crates/ml/src/hyperopt/adapters/mamba2.rs +++ b/crates/ml/src/hyperopt/adapters/mamba2.rs @@ -295,10 +295,8 @@ impl Mamba2Trainer { .into()); } - let device = Device::new_cuda(0).unwrap_or_else(|e| { - warn!("CUDA unavailable ({}), falling back to CPU", e); - Device::Cpu - }); + let device = Device::new_cuda(0) + .map_err(|e| MLError::ConfigError(format!("CUDA GPU required for Mamba2 hyperopt: {}", e)))?; let feature_config = FeatureConfig::wave_d(); let d_model = feature_config.feature_count(); diff --git a/crates/ml/src/hyperopt/adapters/tft.rs b/crates/ml/src/hyperopt/adapters/tft.rs index 613b03658..c95a4429f 100644 --- a/crates/ml/src/hyperopt/adapters/tft.rs +++ b/crates/ml/src/hyperopt/adapters/tft.rs @@ -272,10 +272,8 @@ impl TFTTrainer { .into()); } - let device = Device::new_cuda(0).unwrap_or_else(|e| { - warn!("CUDA unavailable ({}), falling back to CPU", e); - Device::Cpu - }); + let device = Device::new_cuda(0) + .map_err(|e| MLError::ConfigError(format!("CUDA GPU required for TFT hyperopt: {}", e)))?; info!( "TFT Trainer initialized: Device={:?}, Data={}, Epochs per trial={}", diff --git a/crates/ml/src/hyperopt/adapters/tggn.rs b/crates/ml/src/hyperopt/adapters/tggn.rs index 728fa1306..7a392ca4f 100644 --- a/crates/ml/src/hyperopt/adapters/tggn.rs +++ b/crates/ml/src/hyperopt/adapters/tggn.rs @@ -221,10 +221,8 @@ impl TGGNTrainer { return Err(MLError::ConfigError(format!("Data directory not found: {}", data_dir.display()))); } - let device = Device::new_cuda(0).unwrap_or_else(|e| { - warn!("CUDA unavailable ({}), falling back to CPU", e); - Device::Cpu - }); + let device = Device::new_cuda(0) + .map_err(|e| MLError::ConfigError(format!("CUDA GPU required for TGGN hyperopt: {}", e)))?; info!( "TGGN Trainer initialized: Device={:?}, Data={}, Epochs={}", diff --git a/crates/ml/src/hyperopt/adapters/tlob.rs b/crates/ml/src/hyperopt/adapters/tlob.rs index 15df0a73a..4f0feeda0 100644 --- a/crates/ml/src/hyperopt/adapters/tlob.rs +++ b/crates/ml/src/hyperopt/adapters/tlob.rs @@ -210,10 +210,8 @@ impl TLOBTrainer { return Err(MLError::ConfigError(format!("Data directory not found: {}", data_dir.display()))); } - let device = Device::new_cuda(0).unwrap_or_else(|e| { - warn!("CUDA unavailable ({}), falling back to CPU", e); - Device::Cpu - }); + let device = Device::new_cuda(0) + .map_err(|e| MLError::ConfigError(format!("CUDA GPU required for TLOB hyperopt: {}", e)))?; info!( "TLOB Trainer initialized: Device={:?}, Data={}, Epochs={}", diff --git a/crates/ml/src/hyperopt/adapters/xlstm.rs b/crates/ml/src/hyperopt/adapters/xlstm.rs index 59a6a0663..89009098a 100644 --- a/crates/ml/src/hyperopt/adapters/xlstm.rs +++ b/crates/ml/src/hyperopt/adapters/xlstm.rs @@ -184,10 +184,8 @@ impl XLSTMTrainer { return Err(MLError::ConfigError(format!("Data directory not found: {}", data_dir.display()))); } - let device = Device::new_cuda(0).unwrap_or_else(|e| { - warn!("CUDA unavailable ({}), falling back to CPU", e); - Device::Cpu - }); + let device = Device::new_cuda(0) + .map_err(|e| MLError::ConfigError(format!("CUDA GPU required for xLSTM hyperopt: {}", e)))?; info!( "xLSTM Trainer initialized: Device={:?}, Data={}, Epochs={}", diff --git a/crates/ml/src/trainers/mamba2.rs b/crates/ml/src/trainers/mamba2.rs index bba3dd735..0b989a63f 100644 --- a/crates/ml/src/trainers/mamba2.rs +++ b/crates/ml/src/trainers/mamba2.rs @@ -16,7 +16,7 @@ use std::time::{Instant, SystemTime}; use candle_core::{Device, Tensor}; use serde::{Deserialize, Serialize}; -use tracing::{info, warn}; +use tracing::info; use uuid::Uuid; use crate::mamba::{Mamba2Config, Mamba2SSM, OptimizerType, TrainingEpoch}; @@ -330,17 +330,9 @@ impl Mamba2Trainer { estimated_memory ); - // Try to use GPU, fall back to CPU if unavailable - let device = match Device::cuda_if_available(0) { - Ok(cuda_device) => { - info!("Using CUDA device for MAMBA-2 training"); - cuda_device - }, - Err(e) => { - warn!("CUDA not available ({}), using CPU", e); - Device::Cpu - }, - }; + let device = Device::cuda_if_available(0) + .map_err(|e| MLError::ConfigError(format!("CUDA GPU required for Mamba2 training: {}", e)))?; + info!("Using CUDA device for MAMBA-2 training"); // Create MAMBA-2 model let config = hyperparameters.to_mamba_config();