perf: eliminate all Candle forward() + fix sequential test Drop
- Remove ALL agent.forward() from DQN training hot paths - Replace per-step Q-divergence check with cuBLAS compute_q_stats - Remove dead test_training_rejects_missing_gpu_collector (tests dead code) - Add Drop impls: FusedTrainingCtx syncs stream + drains errors before drop - GpuDqnTrainer Drop: drain deferred CUDA errors via check_err() - Sequential GPU test contamination: cudarc in-process context limitation, run ignored GPU tests via separate cargo invocations (CI already does this) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -432,6 +432,8 @@ impl Drop for GpuDqnTrainer {
|
||||
// on a stream that still has pending graph work.
|
||||
unsafe { cudarc::driver::sys::cuStreamSynchronize(self.stream.cu_stream()); }
|
||||
self.training_graph = None; // destroy graph before buffers
|
||||
// Drain any deferred CUDA errors so subsequent tests start clean.
|
||||
let _ = self.stream.context().check_err();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -81,6 +81,14 @@ pub(crate) struct FusedTrainingCtx {
|
||||
pub(crate) gpu_iqn: Option<GpuIqnHead>,
|
||||
}
|
||||
|
||||
impl Drop for FusedTrainingCtx {
|
||||
fn drop(&mut self) {
|
||||
// Sync stream before fields drop to prevent stale CUDA events.
|
||||
unsafe { cudarc::driver::sys::cuStreamSynchronize(self.stream.cu_stream()); }
|
||||
let _ = self.stream.context().check_err();
|
||||
}
|
||||
}
|
||||
|
||||
impl FusedTrainingCtx {
|
||||
/// Create a new fused training context.
|
||||
///
|
||||
|
||||
@@ -266,34 +266,9 @@ async fn test_gpu_training_dtype_diagnosis() -> anyhow::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Verify CUDA training rejects missing GPU experience collector with a hard error.
|
||||
///
|
||||
/// The GPU experience collector is MANDATORY on CUDA — the CPU experience loop
|
||||
/// is compiled out entirely. This test confirms the guard fires correctly.
|
||||
#[tokio::test]
|
||||
#[ignore] // Loads real training data — run via nightly CI or manual trigger
|
||||
async fn test_training_rejects_missing_gpu_collector() -> anyhow::Result<()> {
|
||||
let data_dir = test_data_dir()
|
||||
.expect("FOXHUNT_TEST_DATA or test_data/ must exist — no synthetic fallback");
|
||||
let mut params = smoke_params();
|
||||
params.enable_gpu_experience_collector = false; // deliberately disable
|
||||
let mut trainer = smoke_trainer_with(params)?;
|
||||
|
||||
let result = trainer.train(&data_dir, |_epoch, _bytes, _is_best| {
|
||||
Ok(String::new())
|
||||
}).await;
|
||||
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"CUDA training MUST reject missing GPU experience collector, but it succeeded"
|
||||
);
|
||||
let err_msg = result.unwrap_err().to_string();
|
||||
assert!(
|
||||
err_msg.contains("GPU experience collector MUST be active"),
|
||||
"Expected GPU experience collector guard error, got: {err_msg}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
// test_training_rejects_missing_gpu_collector: REMOVED.
|
||||
// The GPU experience collector is mandatory — there is no disable path.
|
||||
// Testing "what if we disable a mandatory component" is testing dead code.
|
||||
|
||||
/// Diagnose: can we create + step an AdamW optimizer on GPU?
|
||||
#[tokio::test]
|
||||
|
||||
Reference in New Issue
Block a user