fix(tests): CI GPU test stability, walltime reduction, BF16 tolerance
- Reduce CI GPU test datasets 16x for walltime reduction - Reduce early-stop epochs 50→10, add --test-threads=1 - Serialize all GPU lib tests to prevent cuBLAS init race - Align state_dim to 16 for BF16 tensor core HMMA dispatch - BF16 precision tolerance in ml-dqn tests - Enable branching DQN + tracing subscriber in smoke tests - Prevent min_replay_size > buffer_size deadlock in early-stop tests - Prevent AutoReplaySizer from breaking gradient collapse warmup - Replace racy tokio::spawn checkpoint counter with AtomicUsize - Set warmup_steps=0 and max_training_steps_per_epoch=300 in early-stop tests - RealDataLoader respects TEST_DATA_DIR for CI PVC layout - Add collapse_warmup_capacity to gpu_smoketest DQNConfig - Drain CUDA context between test binaries - Detached HEAD checkout prevents local branch corruption - GPU pipeline tests: fix BF16 dtype and rank-1 squeeze assertions - OOD input handling tests use use_gpu: true Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -86,6 +86,7 @@
|
||||
|
||||
use candle_core::{Device, Tensor};
|
||||
use anyhow::Result;
|
||||
use tracing::info;
|
||||
|
||||
// Helper function to convert (exposure_idx, order_idx, urgency_idx) to action index
|
||||
// Maps to the action space formula: action_idx = exposure_idx * 9 + order_idx * 3 + urgency_idx
|
||||
@@ -327,8 +328,9 @@ fn calculate_episode_stats(episode_lengths: &[usize]) -> (usize, usize, f64) {
|
||||
// ============================================================================
|
||||
|
||||
#[test]
|
||||
#[cfg_attr(not(feature = "cuda"), ignore)]
|
||||
fn test_feature_normalization_validation() -> Result<()> {
|
||||
let device = Device::cuda_if_available(0)?;
|
||||
let device = Device::new_cuda(0).expect("CUDA required");
|
||||
|
||||
// Setup: Create states with 5% features outside [-3, +3] range
|
||||
// TradingState has 54 features (from CLAUDE.md)
|
||||
@@ -373,9 +375,8 @@ fn test_feature_normalization_validation() -> Result<()> {
|
||||
// Helper function (will be moved to TrainingMonitor impl)
|
||||
fn check_feature_normalization(states: &Tensor) -> Result<usize> {
|
||||
// Count features outside [-3, +3] range
|
||||
// Convert tensor to CPU and check values directly
|
||||
let states_cpu = states.to_device(&Device::Cpu)?;
|
||||
let states_vec = states_cpu.flatten_all()?.to_vec1::<f32>()?;
|
||||
// Read values directly (Candle handles GPU→CPU DMA transfer internally)
|
||||
let states_vec = states.flatten_all()?.to_vec1::<f32>()?;
|
||||
|
||||
let violations_count = states_vec.iter()
|
||||
.filter(|&&x| x < -3.0 || x > 3.0)
|
||||
@@ -444,7 +445,7 @@ fn test_performance_impact() -> Result<()> {
|
||||
elapsed.as_micros()
|
||||
);
|
||||
|
||||
println!("Diagnostic calculations completed in {}μs (<1% overhead)", elapsed.as_micros());
|
||||
info!(elapsed_us = elapsed.as_micros(), "Diagnostic calculations completed (<1% overhead)");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user