From 39c2520fca875d54ecc1b8ad60b650d3c4dbb577 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 7 Apr 2026 10:57:58 +0200 Subject: [PATCH] fix: drain CUDA errors between hyperopt trials to prevent cascade MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a trial corrupts GPU state (CUDA_ERROR_ILLEGAL_ADDRESS from extreme training), the inter-trial sync now drains the error instead of propagating it — preventing all subsequent trials from failing. Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/ml/src/hyperopt/adapters/dqn.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index 35a049315..fd3334e28 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -2656,9 +2656,14 @@ impl HyperparameterOptimizable for DQNTrainer { // Synchronize the trial device's stream to ensure all async frees complete. // With forked streams sharing the same context, free_async returns memory // to the context's pool — available for the next trial's allocations. + // If sync fails (CUDA_ERROR_ILLEGAL_ADDRESS from corrupted trial), drain + // the error and continue — the next trial creates a fresh stream. if let MlDevice::Cuda { stream, .. } = &trial_device { - stream.synchronize() - .map_err(|e| MLError::TrainingError(format!("CUDA sync between trials: {e}")))?; + if let Err(e) = stream.synchronize() { + tracing::error!("CUDA sync between trials failed (draining): {e}"); + // Drain context-level error to prevent cascade to next trial + let _ = stream.context().check_err(); + } } info!("Resource cleanup complete");