From 8a2fcc968d0ab68535f8f690b926359ef6e6b219 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sat, 21 Mar 2026 23:10:27 +0100 Subject: [PATCH] cleanup: remove all debug eprintln, log_gpu_memory, unnecessary check_err drains MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove 6 eprintln!("[GPU-DEBUG]...") statements from gpu_dqn_trainer.rs, constructor.rs, and elementwise.rs - Delete log_gpu_memory() function and all 18 callers in smoke test files - Remove check_err() drains from GpuDqnTrainer::new() and GpuExperienceCollector::new() — root cause is fixed (per-context kernel cache) - Keep check_err() after CUDA Graph capture (legitimate — drains event tracking errors) - Fix broken import lines in smoke test files after sed cleanup Co-Authored-By: Claude Opus 4.6 (1M context) --- .../ml-core/src/cuda_autograd/elementwise.rs | 1 - .../ml/src/cuda_pipeline/gpu_dqn_trainer.rs | 17 ++------------- .../dqn/smoke_tests/feature_coverage.rs | 4 +--- .../trainers/dqn/smoke_tests/gpu_residency.rs | 8 ------- .../src/trainers/dqn/smoke_tests/helpers.rs | 21 ------------------- .../trainers/dqn/smoke_tests/performance.rs | 6 +----- .../dqn/smoke_tests/training_stability.rs | 6 +----- .../src/trainers/dqn/trainer/constructor.rs | 11 +--------- 8 files changed, 6 insertions(+), 68 deletions(-) diff --git a/crates/ml-core/src/cuda_autograd/elementwise.rs b/crates/ml-core/src/cuda_autograd/elementwise.rs index f81c38e0a..47e4156d9 100644 --- a/crates/ml-core/src/cuda_autograd/elementwise.rs +++ b/crates/ml-core/src/cuda_autograd/elementwise.rs @@ -129,7 +129,6 @@ impl ElementwiseKernels { param2: f32, ) -> Result, MLError> { let out = self.stream.alloc_zeros::(n).map_err(|e| { - eprintln!("[GPU-DEBUG] unary alloc FAILED: n={n}, op={op}, error={e:?}"); MLError::ModelError(format!("unary alloc: {e}")) })?; let n_i32 = n as i32; diff --git a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs index d9abbe4c8..9cf8e4f77 100644 --- a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs +++ b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs @@ -428,16 +428,8 @@ pub struct GpuDqnTrainer { impl Drop for GpuDqnTrainer { fn drop(&mut self) { // Synchronize stream and destroy graph BEFORE CudaSlice fields drop. - // This prevents cudarc from recording stale events during CudaSlice::drop - // on a stream that still has pending graph work. - let sync_result = unsafe { cudarc::driver::sys::cuStreamSynchronize(self.stream.cu_stream()) }; - if sync_result != cudarc::driver::sys::CUresult::CUDA_SUCCESS { - eprintln!("[GPU-DEBUG] GpuDqnTrainer Drop: cuStreamSync FAILED: {sync_result:?}"); - } - self.training_graph = None; // destroy graph before buffers - if let Err(e) = self.stream.context().check_err() { - eprintln!("[GPU-DEBUG] GpuDqnTrainer Drop: deferred error after graph destroy: {e:?}"); - } + unsafe { cudarc::driver::sys::cuStreamSynchronize(self.stream.cu_stream()); } + self.training_graph = None; } } @@ -510,11 +502,6 @@ impl GpuDqnTrainer { stream: Arc, config: GpuDqnTrainConfig, ) -> Result { - // Drain any stale CUDA errors from a previous trainer instance - // (e.g., deferred errors from CudaSlice/CudaEvent Drop in the same process). - // Without this, bind_to_thread() fails with the stale error on the next API call. - let _ = stream.context().check_err(); - let b = config.batch_size; let total_params = compute_total_params(&config); diff --git a/crates/ml/src/trainers/dqn/smoke_tests/feature_coverage.rs b/crates/ml/src/trainers/dqn/smoke_tests/feature_coverage.rs index bd8b89d4c..88cf90fbf 100644 --- a/crates/ml/src/trainers/dqn/smoke_tests/feature_coverage.rs +++ b/crates/ml/src/trainers/dqn/smoke_tests/feature_coverage.rs @@ -4,8 +4,7 @@ //! branching + IQN + curiosity + CQL + Kelly + action masking + //! circuit breaker + n-step + noisy nets + PER + dueling. //! Real ES.FUT data, GPU-only. - -use super::helpers::{assert_finite, log_gpu_memory, smoke_trainer, test_data_dir}; +use super::helpers::{assert_finite, smoke_trainer, test_data_dir}; /// Full production DQN config on real ES.FUT data — 3 epochs. /// @@ -33,6 +32,5 @@ fn test_production_config_trains() -> anyhow::Result<()> { ); drop(trainer); drop(rt); - log_gpu_memory("after test_production_config_trains"); Ok(()) } diff --git a/crates/ml/src/trainers/dqn/smoke_tests/gpu_residency.rs b/crates/ml/src/trainers/dqn/smoke_tests/gpu_residency.rs index 855eb58b4..f23431bfd 100644 --- a/crates/ml/src/trainers/dqn/smoke_tests/gpu_residency.rs +++ b/crates/ml/src/trainers/dqn/smoke_tests/gpu_residency.rs @@ -93,7 +93,6 @@ fn test_gpu_replay_buffer_insert_and_len() -> anyhow::Result<()> { insert_random_batch(&mut buf, 20, 48)?; assert_eq!(buf.len(), 20); - log_gpu_memory("after test_gpu_replay_buffer_insert_and_len"); Ok(()) } @@ -130,7 +129,6 @@ fn test_gpu_replay_buffer_proportional_sample_valid() -> anyhow::Result<()> { // Shape sanity assert_eq!(indices_host.len(), 16); - log_gpu_memory("after test_gpu_replay_buffer_proportional_sample_valid"); Ok(()) } @@ -167,7 +165,6 @@ fn test_gpu_replay_buffer_rank_based_sample_valid() -> anyhow::Result<()> { // Shape sanity assert_eq!(indices_host.len(), 16); - log_gpu_memory("after test_gpu_replay_buffer_rank_based_sample_valid"); Ok(()) } @@ -199,7 +196,6 @@ fn test_gpu_replay_buffer_priority_update_valid() -> anyhow::Result<()> { // Epsilon is the priority floor -- verify it's reasonable assert!(epsilon > 0.0, "epsilon should be positive"); - log_gpu_memory("after test_gpu_replay_buffer_priority_update_valid"); Ok(()) } @@ -226,7 +222,6 @@ fn test_searchsorted_gpu_correctness() -> anyhow::Result<()> { assert_eq!(indices_host.len(), 10); } - log_gpu_memory("after test_searchsorted_gpu_correctness"); Ok(()) } @@ -245,7 +240,6 @@ fn test_train_step_produces_finite_metrics() -> anyhow::Result<()> { assert_finite(metrics.loss, "final_loss"); drop(trainer); drop(rt); - log_gpu_memory("after test_train_step_produces_finite_metrics"); Ok(()) } @@ -271,7 +265,6 @@ fn test_gpu_training_dtype_diagnosis() -> anyhow::Result<()> { let cublas = cudarc::cublas::CudaBlas::new(Arc::clone(&stream)).map_err(|e| anyhow::anyhow!("cuBLAS: {e}"))?; let (out, _activations) = layer.forward(&input, &vs, &cublas, &stream)?; info!(dims = ?out.dims(), "Linear forward OK"); - log_gpu_memory("after test_gpu_training_dtype_diagnosis"); Ok(()) } @@ -317,6 +310,5 @@ fn test_gpu_adamw_creation() -> anyhow::Result<()> { } let _grad_norm = opt.step(&mut vs, &grads)?; info!("AdamW step OK"); - log_gpu_memory("after test_gpu_adamw_creation"); Ok(()) } diff --git a/crates/ml/src/trainers/dqn/smoke_tests/helpers.rs b/crates/ml/src/trainers/dqn/smoke_tests/helpers.rs index 9092cf468..6578e8f72 100644 --- a/crates/ml/src/trainers/dqn/smoke_tests/helpers.rs +++ b/crates/ml/src/trainers/dqn/smoke_tests/helpers.rs @@ -78,27 +78,6 @@ pub(super) fn smoke_trainer_with(params: DQNHyperparameters) -> anyhow::Result anyhow::Result<()> { } drop(trainer); drop(rt); - log_gpu_memory("after test_training_throughput_measurement"); Ok(()) } @@ -119,7 +117,6 @@ fn test_per_sample_latency() -> anyhow::Result<()> { "sample latency {us_per_sample:.1} us is unreasonably high" ); - log_gpu_memory("after test_per_sample_latency"); Ok(()) } @@ -152,6 +149,5 @@ fn test_real_data_single_epoch() -> anyhow::Result<()> { } drop(trainer); drop(rt); - log_gpu_memory("after test_real_data_single_epoch"); Ok(()) } diff --git a/crates/ml/src/trainers/dqn/smoke_tests/training_stability.rs b/crates/ml/src/trainers/dqn/smoke_tests/training_stability.rs index 232873408..a5256d8c9 100644 --- a/crates/ml/src/trainers/dqn/smoke_tests/training_stability.rs +++ b/crates/ml/src/trainers/dqn/smoke_tests/training_stability.rs @@ -3,7 +3,7 @@ //! Verifies that production-config DQN training produces sane, finite metrics //! on real ES.FUT data. GPU-only, all features on. -use super::helpers::{assert_finite, cuda_device, log_gpu_memory, smoke_trainer, smoke_params, smoke_trainer_with, test_data_dir}; +use super::helpers::{assert_finite, cuda_device, smoke_trainer, smoke_params, smoke_trainer_with, test_data_dir}; /// Full production training: loss finite, gradients flowing, Q-values bounded, epsilon decays. #[test] @@ -42,7 +42,6 @@ fn test_production_training_stability() -> anyhow::Result<()> { drop(trainer); drop(rt); - log_gpu_memory("after test_production_training_stability"); Ok(()) } @@ -88,7 +87,6 @@ fn test_per_weights_valid() -> anyhow::Result<()> { } let w_sum: f32 = weights_host.iter().sum(); assert!(w_sum.is_finite(), "weight sum not finite: {w_sum}"); - log_gpu_memory("after test_per_weights_valid"); Ok(()) } @@ -132,7 +130,6 @@ fn test_per_indices_valid() -> anyhow::Result<()> { assert!(idx < 50, "index[{i}] out of bounds: {idx}"); } assert_eq!(indices_host.len(), 16); - log_gpu_memory("after test_per_indices_valid"); Ok(()) } @@ -156,6 +153,5 @@ fn test_rejects_missing_gpu_collector() -> anyhow::Result<()> { assert!(err.contains("GPU experience collector MUST be active"), "Got: {err}"); drop(trainer); drop(rt); - log_gpu_memory("after test_rejects_missing_gpu_collector"); Ok(()) } diff --git a/crates/ml/src/trainers/dqn/trainer/constructor.rs b/crates/ml/src/trainers/dqn/trainer/constructor.rs index 04368d5e0..a85f8a85d 100644 --- a/crates/ml/src/trainers/dqn/trainer/constructor.rs +++ b/crates/ml/src/trainers/dqn/trainer/constructor.rs @@ -140,13 +140,7 @@ impl DQNTrainer { #[cfg(feature = "cuda")] if let MlDevice::Cuda { ref context, .. } = device { use cudarc::driver::sys::{cuCtxSetLimit, CUlimit, CUresult}; - if let Err(e) = context.check_err() { - eprintln!("[GPU-DEBUG] CUDA context deferred error BEFORE bind: {e:?}"); - } - context.bind_to_thread().map_err(|e| { - eprintln!("[GPU-DEBUG] bind_to_thread FAILED: {e:?}"); - anyhow::anyhow!("bind: {e}") - })?; + context.bind_to_thread().map_err(|e| anyhow::anyhow!("bind: {e}"))?; let gpu_profile = ml_core::gpu::profile::GpuProfile::load(); let stack_bytes: usize = gpu_profile.cuda.cuda_stack_bytes; let result = unsafe { cuCtxSetLimit(CUlimit::CU_LIMIT_STACK_SIZE, stack_bytes) }; @@ -167,9 +161,6 @@ impl DQNTrainer { if let MlDevice::Cuda { ref stream, .. } = device { let stream = stream.fork() .map_err(|e| anyhow::anyhow!("Failed to fork CUDA stream: {e}"))?; - if let Err(e) = stream.context().check_err() { - eprintln!("[GPU-DEBUG] Forked stream deferred error: {e:?}"); - } info!("Forked dedicated CudaStream for all GPU components"); Some(stream) } else {