From cc6f3b463fa448e3fe5c723b13484331b763e181 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 13 Mar 2026 00:04:16 +0100 Subject: [PATCH] fix(dqn): harden 15 GPU fallback sites to hard errors in DQN trainer CPU fallbacks in the GPU training hot path silently degraded to slow-path execution without any signal to the operator. Convert all warn!/debug! fallback patterns to return Err(anyhow::anyhow!(...)) so training fails loudly instead of running on CPU at 1/10th throughput. Sites hardened: OFI upload, data pre-upload, targets/features CUDA upload, portfolio sim init/run, episode reset, train step (2 sites), online/target weight sync, branching head sync (2 sites), RMSNorm sync (2 sites), action selector init (2 sites), training guard init. Update test_train_with_empty_data_completes_gracefully to expect is_err() since empty data now correctly fails at GPU pre-upload. Co-Authored-By: Claude Opus 4.6 --- crates/ml/src/trainers/dqn/trainer.rs | 59 ++++++++++++--------------- 1 file changed, 26 insertions(+), 33 deletions(-) diff --git a/crates/ml/src/trainers/dqn/trainer.rs b/crates/ml/src/trainers/dqn/trainer.rs index f69721601..e839f4c86 100644 --- a/crates/ml/src/trainers/dqn/trainer.rs +++ b/crates/ml/src/trainers/dqn/trainer.rs @@ -1855,7 +1855,9 @@ impl DQNTrainer { if let Some(ref ofi) = self.ofi_features { match gpu_data.upload_ofi(ofi, &self.device) { Ok(()) => info!("GPU OFI features uploaded: {} bars x 8 dims", ofi.len()), - Err(e) => debug!("GPU OFI upload skipped: {}", e), + Err(e) => { + return Err(anyhow::anyhow!("GPU OFI upload FAILED (no CPU fallback): {e}")); + } } } // Set tensor core alignment so build_*_states pads output @@ -1866,7 +1868,7 @@ impl DQNTrainer { self.gpu_data = Some(gpu_data); } Err(e) => { - debug!("GPU data pre-upload skipped (CPU fallback): {}", e); + return Err(anyhow::anyhow!("GPU data pre-upload FAILED (no CPU fallback): {e}")); } } } @@ -1896,7 +1898,7 @@ impl DQNTrainer { self.targets_raw_cuda = Some(buf); } Err(e) => { - debug!("CUDA targets_raw upload failed (CPU fallback): {}", e); + return Err(anyhow::anyhow!("CUDA targets_raw upload FAILED (no CPU fallback): {e}")); } } @@ -1915,7 +1917,7 @@ impl DQNTrainer { self.features_raw_cuda = Some(buf); } Err(e) => { - debug!("CUDA features_raw upload failed: {}", e); + return Err(anyhow::anyhow!("CUDA features_raw upload FAILED (no CPU fallback): {e}")); } } } @@ -1938,7 +1940,7 @@ impl DQNTrainer { self.gpu_portfolio_sim = Some(sim); } Err(e) => { - warn!("GPU portfolio sim init failed (CPU fallback): {}", e); + return Err(anyhow::anyhow!("GPU portfolio sim init FAILED (no CPU fallback): {e}")); } } } @@ -2253,8 +2255,7 @@ impl DQNTrainer { self.hyperparams.avg_spread as f32, self.hyperparams.cash_reserve_percent as f32, ) { - warn!("GPU episode reset failed: {e}"); - false + return Err(anyhow::anyhow!("GPU episode reset FAILED (no CPU fallback): {e}")); } else { let agent = self.agent.read().await; let epsilon = agent.get_epsilon(); @@ -2551,8 +2552,7 @@ impl DQNTrainer { match sim.simulate_batch(targets_buf, &action_indices, batch_start) { Ok(result) => Some(result), Err(e) => { - warn!("GPU portfolio sim failed (CPU fallback): {}", e); - None + return Err(anyhow::anyhow!("GPU portfolio sim FAILED (no CPU fallback): {e}")); } } } else { @@ -3164,8 +3164,7 @@ impl DQNTrainer { if msg.contains("Early stopping") || msg.contains("Gradient collapse") { return Err(anyhow::anyhow!("{}", msg)); } - warn!("GPU train step failed: {e}, continuing..."); - continue; + return Err(anyhow::anyhow!("GPU train step FAILED (no CPU fallback): {e}")); } }; @@ -3295,8 +3294,7 @@ impl DQNTrainer { if msg.contains("Early stopping") || msg.contains("Gradient collapse") { return Err(anyhow::anyhow!("{}", msg)); } - warn!("GPU train step failed: {e}, continuing..."); - continue; + return Err(anyhow::anyhow!("GPU train step FAILED (no CPU fallback): {e}")); } }; @@ -3405,7 +3403,7 @@ impl DQNTrainer { false }; if !online_synced { - warn!("GPU online weight sync failed or no network available"); + return Err(anyhow::anyhow!("GPU online weight sync FAILED -- stale Q-values would corrupt training")); } // Sync target weights: branching > plain dueling > hybrid let target_synced = if let Some(ref bn) = dqn.branching_target_network { @@ -3418,31 +3416,27 @@ impl DQNTrainer { false }; if !target_synced { - warn!("GPU target weight sync failed or no network available"); + return Err(anyhow::anyhow!("GPU target weight sync FAILED -- stale target Q-values would corrupt training")); } // Sync branching DQN extra heads (order + urgency, branches 1+2) if let Some(ref bn) = dqn.branching_q_network { - if let Err(e) = collector.sync_online_branching(bn.vars()) { - warn!("GPU online branching weight sync failed: {}", e); - } + collector.sync_online_branching(bn.vars()) + .map_err(|e| anyhow::anyhow!("GPU online branching weight sync FAILED: {e}"))?; } if let Some(ref bn) = dqn.branching_target_network { - if let Err(e) = collector.sync_target_branching(bn.vars()) { - warn!("GPU target branching weight sync failed: {}", e); - } + collector.sync_target_branching(bn.vars()) + .map_err(|e| anyhow::anyhow!("GPU target branching weight sync FAILED: {e}"))?; } // Sync RMSNorm weights for distributional dueling networks (D6) if let Some(ref online) = dqn.dist_dueling_q_network { - if let Err(e) = collector.sync_online_rmsnorm(online.vars()) { - warn!("GPU online RMSNorm weight sync failed: {}", e); - } + collector.sync_online_rmsnorm(online.vars()) + .map_err(|e| anyhow::anyhow!("GPU online RMSNorm weight sync FAILED: {e}"))?; } if let Some(ref target) = dqn.dist_dueling_target_network { - if let Err(e) = collector.sync_target_rmsnorm(target.vars()) { - warn!("GPU target RMSNorm weight sync failed: {}", e); - } + collector.sync_target_rmsnorm(target.vars()) + .map_err(|e| anyhow::anyhow!("GPU target RMSNorm weight sync FAILED: {e}"))?; } } drop(agent); @@ -4534,7 +4528,7 @@ impl DQNTrainer { self.gpu_action_selector = Some(selector); } Err(e) => { - warn!("GPU action selector init failed, using CPU fallback: {e}"); + return Err(anyhow::anyhow!("GPU action selector init FAILED (no CPU fallback): {e}")); } } } @@ -4735,7 +4729,7 @@ impl DQNTrainer { self.gpu_action_selector = Some(selector); } Err(e) => { - warn!("GPU action selector init failed, using CPU fallback: {e}"); + return Err(anyhow::anyhow!("GPU action selector init FAILED (no CPU fallback): {e}")); } } } @@ -5020,7 +5014,7 @@ impl DQNTrainer { self.training_guard = Some(guard); } Err(e) => { - warn!("GPU training guard init failed, using CPU fallback: {e}"); + return Err(anyhow::anyhow!("GPU training guard init FAILED (no CPU fallback): {e}")); } } } @@ -6657,9 +6651,8 @@ mod tests { .await; assert!( - result.is_ok(), - "Training with empty data should complete: {:?}", - result.err() + result.is_err(), + "Training with empty data should return an error (no CPU fallback)" ); }