gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

diag: disable graph_forward — run forward+backward ungraphed every step

Browse Source 
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-04-09 21:41:06 +02:00
parent ef6fab5f3b
commit 4e95f5142d
1 changed files with 4 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs
View File

@@ -3159,11 +3159,10 @@ impl GpuDqnTrainer {
self.stream.cu_stream(),
);
}
if self.graph_forward.is_none() {
self.capture_training_graphs(online_dueling, online_branching)?;
}
// DIAG: skip ALL graph capture — run forward+backward ungraphed every step.
// If this fixes H100 collapse, the graph recapture path is the root cause.
self.update_stochastic_depth_mask()?;
self.replay_forward()?;
self.submit_forward_ops_main()?;
Ok(FusedTrainScalars { total_loss: 0.0, grad_norm: 0.0 })
}
@@ -3813,6 +3812,7 @@ impl GpuDqnTrainer {
/// Replay graph_adam only (grad_norm → Adam → unflatten).
/// Must be called AFTER replay_forward() and any auxiliary gradient injection.
pub fn replay_adam(&self) -> Result<(), MLError> {
// Keep graph_adam replay — it's captured once and never invalidated
if let Some(ref graph) = self.graph_adam {
graph.0.launch().map_err(|e| {
MLError::ModelError(format!("CUDA graph_adam replay: {e}"))