gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(ml): GPU OOM detection with automatic CPU fallback in inference engine

Browse Source 
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-02-23 10:06:40 +01:00
parent 3637b49218
commit e36698ef14
1 changed files with 30 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
ml/src/inference.rs
View File

@@ -544,17 +544,37 @@ impl RealMLInferenceEngine {
let device = match self.config.device_preference.as_str() {
"cuda" | "gpu" => match Device::new_cuda(0) {
Ok(cuda_device) => {
info!("✅ Using CUDA device for model: {}", model_id);
cuda_device
},
match crate::memory_optimization::auto_batch_size::detect_gpu_memory() {
Ok((_, free_mb, _)) if free_mb > 500.0 => {
info!(
"Using CUDA device for model: {} (free VRAM: {:.0}MB)",
model_id, free_mb
);
cuda_device
}
Ok((_, free_mb, _)) => {
warn!(
"GPU VRAM too low ({:.0}MB free), falling back to CPU for model: {}",
free_mb, model_id
);
Device::Cpu
}
Err(e) => {
warn!(
"Cannot detect GPU memory ({}), falling back to CPU for model: {}",
e, model_id
);
Device::Cpu
}
}
}
Err(e) => {
return Err(MLSafetyError::from(RealInferenceError::GpuRequired {
reason: format!(
"GPU acceleration required for production model {}: {}",
model_id, e
),
}));
},
warn!(
"CUDA not available ({}), falling back to CPU for model: {}",
e, model_id
);
Device::Cpu
}
},
_ => {
info!("Using CPU device for model: {}", model_id);