fix(tests): CI GPU test stability, walltime reduction, BF16 tolerance

- Reduce CI GPU test datasets 16x for walltime reduction
- Reduce early-stop epochs 50→10, add --test-threads=1
- Serialize all GPU lib tests to prevent cuBLAS init race
- Align state_dim to 16 for BF16 tensor core HMMA dispatch
- BF16 precision tolerance in ml-dqn tests
- Enable branching DQN + tracing subscriber in smoke tests
- Prevent min_replay_size > buffer_size deadlock in early-stop tests
- Prevent AutoReplaySizer from breaking gradient collapse warmup
- Replace racy tokio::spawn checkpoint counter with AtomicUsize
- Set warmup_steps=0 and max_training_steps_per_epoch=300 in early-stop tests
- RealDataLoader respects TEST_DATA_DIR for CI PVC layout
- Add collapse_warmup_capacity to gpu_smoketest DQNConfig
- Drain CUDA context between test binaries
- Detached HEAD checkout prevents local branch corruption
- GPU pipeline tests: fix BF16 dtype and rank-1 squeeze assertions
- OOD input handling tests use use_gpu: true

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-15 12:00:13 +01:00
parent b4178952d4
commit ca4c38d921
139 changed files with 3576 additions and 4462 deletions

View File

@@ -86,6 +86,7 @@
use candle_core::{Device, Tensor};
use anyhow::Result;
use tracing::info;
// Helper function to convert (exposure_idx, order_idx, urgency_idx) to action index
// Maps to the action space formula: action_idx = exposure_idx * 9 + order_idx * 3 + urgency_idx
@@ -327,8 +328,9 @@ fn calculate_episode_stats(episode_lengths: &[usize]) -> (usize, usize, f64) {
// ============================================================================
#[test]
#[cfg_attr(not(feature = "cuda"), ignore)]
fn test_feature_normalization_validation() -> Result<()> {
let device = Device::cuda_if_available(0)?;
let device = Device::new_cuda(0).expect("CUDA required");
// Setup: Create states with 5% features outside [-3, +3] range
// TradingState has 54 features (from CLAUDE.md)
@@ -373,9 +375,8 @@ fn test_feature_normalization_validation() -> Result<()> {
// Helper function (will be moved to TrainingMonitor impl)
fn check_feature_normalization(states: &Tensor) -> Result<usize> {
// Count features outside [-3, +3] range
// Convert tensor to CPU and check values directly
let states_cpu = states.to_device(&Device::Cpu)?;
let states_vec = states_cpu.flatten_all()?.to_vec1::<f32>()?;
// Read values directly (Candle handles GPU→CPU DMA transfer internally)
let states_vec = states.flatten_all()?.to_vec1::<f32>()?;
let violations_count = states_vec.iter()
.filter(|&&x| x < -3.0 || x > 3.0)
@@ -444,7 +445,7 @@ fn test_performance_impact() -> Result<()> {
elapsed.as_micros()
);
println!("Diagnostic calculations completed in {}μs (<1% overhead)", elapsed.as_micros());
info!(elapsed_us = elapsed.as_micros(), "Diagnostic calculations completed (<1% overhead)");
Ok(())
}