gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(cuda): eliminate CPU fallbacks in all 10 supervised/RL model adapters

Browse Source 
Convert Device::Cpu fallback patterns to hard errors across all
hyperopt adapters and the Mamba2 trainer. On H100, CUDA must be
available — silent CPU fallback runs at 1/10th throughput.

Models hardened: TFT, Mamba2, TGGN, TLOB, Liquid, KAN, xLSTM,
Diffusion, ContinuousPPO (hyperopt adapters) + Mamba2 (trainer).

Pattern: Device::new_cuda(0).unwrap_or_else(|e| { warn!(...); Cpu })
      →  Device::new_cuda(0).map_err(|e| MLError::ConfigError(...))?

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-13 00:23:19 +01:00
parent bde510bf8e
commit e0e1e2fff3
10 changed files with 23 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
crates/ml/src/hyperopt/adapters/continuous_ppo.rs
View File

@@ -35,7 +35,7 @@ use candle_core::Device;
use serde::{Deserialize, Serialize};
use std::fs::OpenOptions;
use std::io::Write as IoWrite;
use tracing::{info, warn};
use tracing::info;
use crate::hyperopt::paths::TrainingPaths;
use crate::hyperopt::traits::{HardwareBudget, HyperparameterOptimizable, ParameterSpace};
@@ -285,11 +285,8 @@ impl ContinuousPPOTrainer {
.into());
}
// Initialize device (CUDA preferred, CPU fallback)
let device = Device::new_cuda(0).unwrap_or_else(|e| {
warn!("CUDA unavailable ({}), falling back to CPU", e);
Device::Cpu
});
let device = Device::new_cuda(0)
.map_err(|e| MLError::ConfigError(format!("CUDA GPU required for ContinuousPPO hyperopt: {}", e)))?;
info!("Continuous PPO Trainer initialized:");
info!(" Parquet file: {}", parquet_file.display());

6
crates/ml/src/hyperopt/adapters/diffusion.rs
View File

@@ -207,10 +207,8 @@ impl DiffusionTrainer {
return Err(MLError::ConfigError(format!("Data directory not found: {}", data_dir.display())));
}
let device = Device::new_cuda(0).unwrap_or_else(|e| {
warn!("CUDA unavailable ({}), falling back to CPU", e);
Device::Cpu
});
let device = Device::new_cuda(0)
.map_err(|e| MLError::ConfigError(format!("CUDA GPU required for Diffusion hyperopt: {}", e)))?;
info!(
"Diffusion Trainer initialized: Device={:?}, Data={}, Epochs={}",

6
crates/ml/src/hyperopt/adapters/kan.rs
View File

@@ -196,10 +196,8 @@ impl KANTrainer {
return Err(MLError::ConfigError(format!("Data directory not found: {}", data_dir.display())));
}
let device = Device::new_cuda(0).unwrap_or_else(|e| {
warn!("CUDA unavailable ({}), falling back to CPU", e);
Device::Cpu
});
let device = Device::new_cuda(0)
.map_err(|e| MLError::ConfigError(format!("CUDA GPU required for KAN hyperopt: {}", e)))?;
info!(
"KAN Trainer initialized: Device={:?}, Data={}, Epochs={}",

6
crates/ml/src/hyperopt/adapters/liquid.rs
View File

@@ -258,10 +258,8 @@ impl LiquidTrainer {
return Err(MLError::ConfigError(format!("Data directory not found: {}", data_dir.display())));
}
let device = Device::new_cuda(0).unwrap_or_else(|e| {
warn!("CUDA unavailable ({}), falling back to CPU", e);
Device::Cpu
});
let device = Device::new_cuda(0)
.map_err(|e| MLError::ConfigError(format!("CUDA GPU required for Liquid hyperopt: {}", e)))?;
info!(
"Liquid Trainer initialized: Device={:?}, Data={}, Epochs={}",

6
crates/ml/src/hyperopt/adapters/mamba2.rs
View File

@@ -295,10 +295,8 @@ impl Mamba2Trainer {
.into());
}
let device = Device::new_cuda(0).unwrap_or_else(|e| {
warn!("CUDA unavailable ({}), falling back to CPU", e);
Device::Cpu
});
let device = Device::new_cuda(0)
.map_err(|e| MLError::ConfigError(format!("CUDA GPU required for Mamba2 hyperopt: {}", e)))?;
let feature_config = FeatureConfig::wave_d();
let d_model = feature_config.feature_count();

6
crates/ml/src/hyperopt/adapters/tft.rs
View File

@@ -272,10 +272,8 @@ impl TFTTrainer {
.into());
}
let device = Device::new_cuda(0).unwrap_or_else(|e| {
warn!("CUDA unavailable ({}), falling back to CPU", e);
Device::Cpu
});
let device = Device::new_cuda(0)
.map_err(|e| MLError::ConfigError(format!("CUDA GPU required for TFT hyperopt: {}", e)))?;
info!(
"TFT Trainer initialized: Device={:?}, Data={}, Epochs per trial={}",

6
crates/ml/src/hyperopt/adapters/tggn.rs
View File

@@ -221,10 +221,8 @@ impl TGGNTrainer {
return Err(MLError::ConfigError(format!("Data directory not found: {}", data_dir.display())));
}
let device = Device::new_cuda(0).unwrap_or_else(|e| {
warn!("CUDA unavailable ({}), falling back to CPU", e);
Device::Cpu
});
let device = Device::new_cuda(0)
.map_err(|e| MLError::ConfigError(format!("CUDA GPU required for TGGN hyperopt: {}", e)))?;
info!(
"TGGN Trainer initialized: Device={:?}, Data={}, Epochs={}",

6
crates/ml/src/hyperopt/adapters/tlob.rs
View File

@@ -210,10 +210,8 @@ impl TLOBTrainer {
return Err(MLError::ConfigError(format!("Data directory not found: {}", data_dir.display())));
}
let device = Device::new_cuda(0).unwrap_or_else(|e| {
warn!("CUDA unavailable ({}), falling back to CPU", e);
Device::Cpu
});
let device = Device::new_cuda(0)
.map_err(|e| MLError::ConfigError(format!("CUDA GPU required for TLOB hyperopt: {}", e)))?;
info!(
"TLOB Trainer initialized: Device={:?}, Data={}, Epochs={}",

6
crates/ml/src/hyperopt/adapters/xlstm.rs
View File

@@ -184,10 +184,8 @@ impl XLSTMTrainer {
return Err(MLError::ConfigError(format!("Data directory not found: {}", data_dir.display())));
}
let device = Device::new_cuda(0).unwrap_or_else(|e| {
warn!("CUDA unavailable ({}), falling back to CPU", e);
Device::Cpu
});
let device = Device::new_cuda(0)
.map_err(|e| MLError::ConfigError(format!("CUDA GPU required for xLSTM hyperopt: {}", e)))?;
info!(
"xLSTM Trainer initialized: Device={:?}, Data={}, Epochs={}",

16
crates/ml/src/trainers/mamba2.rs
View File

@@ -16,7 +16,7 @@ use std::time::{Instant, SystemTime};
use candle_core::{Device, Tensor};
use serde::{Deserialize, Serialize};
use tracing::{info, warn};
use tracing::info;
use uuid::Uuid;
use crate::mamba::{Mamba2Config, Mamba2SSM, OptimizerType, TrainingEpoch};
@@ -330,17 +330,9 @@ impl Mamba2Trainer {
estimated_memory
);
// Try to use GPU, fall back to CPU if unavailable
let device = match Device::cuda_if_available(0) {
Ok(cuda_device) => {
info!("Using CUDA device for MAMBA-2 training");
cuda_device
},
Err(e) => {
warn!("CUDA not available ({}), using CPU", e);
Device::Cpu
},
};
let device = Device::cuda_if_available(0)
.map_err(|e| MLError::ConfigError(format!("CUDA GPU required for Mamba2 training: {}", e)))?;
info!("Using CUDA device for MAMBA-2 training");
// Create MAMBA-2 model
let config = hyperparameters.to_mamba_config();