diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index 35a049315..fd3334e28 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -2656,9 +2656,14 @@ impl HyperparameterOptimizable for DQNTrainer { // Synchronize the trial device's stream to ensure all async frees complete. // With forked streams sharing the same context, free_async returns memory // to the context's pool — available for the next trial's allocations. + // If sync fails (CUDA_ERROR_ILLEGAL_ADDRESS from corrupted trial), drain + // the error and continue — the next trial creates a fresh stream. if let MlDevice::Cuda { stream, .. } = &trial_device { - stream.synchronize() - .map_err(|e| MLError::TrainingError(format!("CUDA sync between trials: {e}")))?; + if let Err(e) = stream.synchronize() { + tracing::error!("CUDA sync between trials failed (draining): {e}"); + // Drain context-level error to prevent cascade to next trial + let _ = stream.context().check_err(); + } } info!("Resource cleanup complete");