--- a/ml/src/trainers/tft.rs +++ b/ml/src/trainers/tft.rs @@ -275,14 +275,27 @@ impl TFTTrainer { info!("Initializing TFT trainer with config: {:?}", config); // Select device (GPU if available and requested) let device = if config.use_gpu { - Device::cuda_if_available(0) - .map_err(|e| MLError::ConfigError { - reason: format!("GPU requested but not available: {}", e), - })? + match Device::cuda_if_available(0) { + Ok(Device::Cuda(cuda_device)) => { + info!("✓ GPU detected and enabled: CUDA device 0"); + info!(" All tensors will be allocated on GPU for accelerated training"); + Device::Cuda(cuda_device) + }, + Ok(Device::Cpu) => { + warn!("⚠ GPU requested (use_gpu=true) but CUDA not available"); + warn!(" Falling back to CPU - training will be significantly slower"); + warn!(" Verify: nvidia-smi shows GPU and CUDA_HOME is set"); + Device::Cpu + }, + Err(e) => { + return Err(MLError::ConfigError { + reason: format!("GPU initialization failed: {}. Check CUDA drivers.", e), + }); + }, + _ => { + warn!("⚠ Unexpected device type, falling back to CPU"); + Device::Cpu + }, + } } else { info!("Using CPU device (use_gpu=false in config)"); Device::Cpu @@ -540,11 +553,29 @@ impl TFTTrainer { val_loader: &mut TFTDataLoader, _epoch: usize, ) -> MLResult<(f64, ValidationMetrics)> { + // Check for empty validation loader + if val_loader.len() == 0 { + warn!("Validation loader is empty - returning zero loss"); + return Ok((0.0, ValidationMetrics::default())); + } + + info!("Starting validation with {} batches", val_loader.len()); + let mut total_loss = 0.0; let mut total_quantile_loss = 0.0; let mut total_rmse = 0.0; let mut attention_entropies = Vec::new(); let mut batch_count = 0; + let mut skipped_batches = 0; for batch in val_loader.iter() { + // Skip empty batches + if batch.targets.nrows() == 0 { + skipped_batches += 1; + debug!("Skipping empty validation batch"); + continue; + } + // Convert batch to tensors let (static_tensor, hist_tensor, fut_tensor, target_tensor) = self.batch_to_tensors(batch)?; @@ -575,6 +606,20 @@ impl TFTTrainer { batch_count += 1; } + // Defensive check for zero batch_count + if batch_count == 0 { + warn!( + "Validation completed with zero non-empty batches (skipped: {})", + skipped_batches + ); + return Ok((0.0, ValidationMetrics::default())); + } + + info!( + "Validation complete: {} batches processed, {} skipped", + batch_count, skipped_batches + ); + let avg_loss = total_loss / batch_count as f64; let avg_attention_entropy = if attention_entropies.is_empty() { 0.0 @@ -592,10 +637,20 @@ impl TFTTrainer { } /// Convert batch to tensors fn batch_to_tensors(&self, batch: &TFTBatch) -> MLResult<(Tensor, Tensor, Tensor, Tensor)> { + // Log device for first batch (debugging) + debug!( + "Creating tensors on device: {:?} (batch_size: {})", + self.device, + batch.static_features.nrows() + ); + // Convert ndarray to tensors let static_data: Vec = batch.static_features.iter().map(|&x| x as f32).collect(); let static_tensor = Tensor::from_slice( &static_data, batch.static_features.raw_dim().into_pattern(), &self.device,