gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(dqn): harden 15 GPU fallback sites to hard errors in DQN trainer

Browse Source 
CPU fallbacks in the GPU training hot path silently degraded to
slow-path execution without any signal to the operator. Convert all
warn!/debug! fallback patterns to return Err(anyhow::anyhow!(...)) so
training fails loudly instead of running on CPU at 1/10th throughput.

Sites hardened: OFI upload, data pre-upload, targets/features CUDA
upload, portfolio sim init/run, episode reset, train step (2 sites),
online/target weight sync, branching head sync (2 sites), RMSNorm
sync (2 sites), action selector init (2 sites), training guard init.

Update test_train_with_empty_data_completes_gracefully to expect
is_err() since empty data now correctly fails at GPU pre-upload.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-13 00:04:16 +01:00
parent 28471d2ab5
commit cc6f3b463f
1 changed files with 26 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
crates/ml/src/trainers/dqn/trainer.rs
View File

@@ -1855,7 +1855,9 @@ impl DQNTrainer {
if let Some(ref ofi) = self.ofi_features {
match gpu_data.upload_ofi(ofi, &self.device) {
Ok(()) => info!("GPU OFI features uploaded: {} bars x 8 dims", ofi.len()),
Err(e) => debug!("GPU OFI upload skipped: {}", e),
Err(e) => {
return Err(anyhow::anyhow!("GPU OFI upload FAILED (no CPU fallback): {e}"));
}
}
}
// Set tensor core alignment so build_*_states pads output
@@ -1866,7 +1868,7 @@ impl DQNTrainer {
self.gpu_data = Some(gpu_data);
}
Err(e) => {
debug!("GPU data pre-upload skipped (CPU fallback): {}", e);
return Err(anyhow::anyhow!("GPU data pre-upload FAILED (no CPU fallback): {e}"));
}
}
}
@@ -1896,7 +1898,7 @@ impl DQNTrainer {
self.targets_raw_cuda = Some(buf);
}
Err(e) => {
debug!("CUDA targets_raw upload failed (CPU fallback): {}", e);
return Err(anyhow::anyhow!("CUDA targets_raw upload FAILED (no CPU fallback): {e}"));
}
}
@@ -1915,7 +1917,7 @@ impl DQNTrainer {
self.features_raw_cuda = Some(buf);
}
Err(e) => {
debug!("CUDA features_raw upload failed: {}", e);
return Err(anyhow::anyhow!("CUDA features_raw upload FAILED (no CPU fallback): {e}"));
}
}
}
@@ -1938,7 +1940,7 @@ impl DQNTrainer {
self.gpu_portfolio_sim = Some(sim);
}
Err(e) => {
warn!("GPU portfolio sim init failed (CPU fallback): {}", e);
return Err(anyhow::anyhow!("GPU portfolio sim init FAILED (no CPU fallback): {e}"));
}
}
}
@@ -2253,8 +2255,7 @@ impl DQNTrainer {
self.hyperparams.avg_spread as f32,
self.hyperparams.cash_reserve_percent as f32,
) {
warn!("GPU episode reset failed: {e}");
false
return Err(anyhow::anyhow!("GPU episode reset FAILED (no CPU fallback): {e}"));
} else {
let agent = self.agent.read().await;
let epsilon = agent.get_epsilon();
@@ -2551,8 +2552,7 @@ impl DQNTrainer {
match sim.simulate_batch(targets_buf, &action_indices, batch_start) {
Ok(result) => Some(result),
Err(e) => {
warn!("GPU portfolio sim failed (CPU fallback): {}", e);
None
return Err(anyhow::anyhow!("GPU portfolio sim FAILED (no CPU fallback): {e}"));
}
}
} else {
@@ -3164,8 +3164,7 @@ impl DQNTrainer {
if msg.contains("Early stopping") || msg.contains("Gradient collapse") {
return Err(anyhow::anyhow!("{}", msg));
}
warn!("GPU train step failed: {e}, continuing...");
continue;
return Err(anyhow::anyhow!("GPU train step FAILED (no CPU fallback): {e}"));
}
};
@@ -3295,8 +3294,7 @@ impl DQNTrainer {
if msg.contains("Early stopping") || msg.contains("Gradient collapse") {
return Err(anyhow::anyhow!("{}", msg));
}
warn!("GPU train step failed: {e}, continuing...");
continue;
return Err(anyhow::anyhow!("GPU train step FAILED (no CPU fallback): {e}"));
}
};
@@ -3405,7 +3403,7 @@ impl DQNTrainer {
false
};
if !online_synced {
warn!("GPU online weight sync failed or no network available");
return Err(anyhow::anyhow!("GPU online weight sync FAILED -- stale Q-values would corrupt training"));
}
// Sync target weights: branching > plain dueling > hybrid
let target_synced = if let Some(ref bn) = dqn.branching_target_network {
@@ -3418,31 +3416,27 @@ impl DQNTrainer {
false
};
if !target_synced {
warn!("GPU target weight sync failed or no network available");
return Err(anyhow::anyhow!("GPU target weight sync FAILED -- stale target Q-values would corrupt training"));
}
// Sync branching DQN extra heads (order + urgency, branches 1+2)
if let Some(ref bn) = dqn.branching_q_network {
if let Err(e) = collector.sync_online_branching(bn.vars()) {
warn!("GPU online branching weight sync failed: {}", e);
}
collector.sync_online_branching(bn.vars())
.map_err(|e| anyhow::anyhow!("GPU online branching weight sync FAILED: {e}"))?;
}
if let Some(ref bn) = dqn.branching_target_network {
if let Err(e) = collector.sync_target_branching(bn.vars()) {
warn!("GPU target branching weight sync failed: {}", e);
}
collector.sync_target_branching(bn.vars())
.map_err(|e| anyhow::anyhow!("GPU target branching weight sync FAILED: {e}"))?;
}
// Sync RMSNorm weights for distributional dueling networks (D6)
if let Some(ref online) = dqn.dist_dueling_q_network {
if let Err(e) = collector.sync_online_rmsnorm(online.vars()) {
warn!("GPU online RMSNorm weight sync failed: {}", e);
}
collector.sync_online_rmsnorm(online.vars())
.map_err(|e| anyhow::anyhow!("GPU online RMSNorm weight sync FAILED: {e}"))?;
}
if let Some(ref target) = dqn.dist_dueling_target_network {
if let Err(e) = collector.sync_target_rmsnorm(target.vars()) {
warn!("GPU target RMSNorm weight sync failed: {}", e);
}
collector.sync_target_rmsnorm(target.vars())
.map_err(|e| anyhow::anyhow!("GPU target RMSNorm weight sync FAILED: {e}"))?;
}
}
drop(agent);
@@ -4534,7 +4528,7 @@ impl DQNTrainer {
self.gpu_action_selector = Some(selector);
}
Err(e) => {
warn!("GPU action selector init failed, using CPU fallback: {e}");
return Err(anyhow::anyhow!("GPU action selector init FAILED (no CPU fallback): {e}"));
}
}
}
@@ -4735,7 +4729,7 @@ impl DQNTrainer {
self.gpu_action_selector = Some(selector);
}
Err(e) => {
warn!("GPU action selector init failed, using CPU fallback: {e}");
return Err(anyhow::anyhow!("GPU action selector init FAILED (no CPU fallback): {e}"));
}
}
}
@@ -5020,7 +5014,7 @@ impl DQNTrainer {
self.training_guard = Some(guard);
}
Err(e) => {
warn!("GPU training guard init failed, using CPU fallback: {e}");
return Err(anyhow::anyhow!("GPU training guard init FAILED (no CPU fallback): {e}"));
}
}
}
@@ -6657,9 +6651,8 @@ mod tests {
.await;
assert!(
result.is_ok(),
"Training with empty data should complete: {:?}",
result.err()
result.is_err(),
"Training with empty data should return an error (no CPU fallback)"
);
}