foxhunt/ml/tests/dqn_gradient_flow_test.rs

//! Gradient Flow Analysis Tests for DQN
//!
//! These tests expose gradient flow bugs that cause:
//! - 217 gradient collapses per run (norm=0.0000)
//! - Q-value collapse to 0.0000 across all actions
//! - Action bias (HOLD always selected)
//!
//! **Bug Hypotheses**:
//! 1. Vanishing gradients in 4x expanded network [256,128,64]
//! 2. Dead neuron detection checking WEIGHTS instead of ACTIVATIONS
//! 3. Entropy regularization (10% weight) suppressing Q-values
//! 4. LeakyReLU alpha=0.01 too low (should be 0.1-0.2)

use anyhow::Result;
use candle_core::{Device, IndexOp, Tensor};
use ml::dqn::{Experience, WorkingDQN, WorkingDQNConfig};

/// Test: Gradients flow through all layers during backpropagation
///
/// **Expected**: All layers (fc1, fc2, fc3) have non-zero gradients with reasonable ratios
/// **Bug Symptom**: Gradient collapse (norm=0.0000) or vanishing gradients (fc1 << fc3)
#[test]
fn test_gradients_flow_through_all_layers() -> Result<()> {
    // Create DQN with larger network
    let mut config = WorkingDQNConfig::emergency_safe_defaults();
    config.state_dim = 52;
    config.hidden_dims = vec![256, 128, 64]; // Wave 10-A1 network
    config.batch_size = 32;
    config.min_replay_size = 32;
    config.replay_buffer_capacity = 1000;

    let mut dqn = WorkingDQN::new(config)?;

    // Populate replay buffer with 50 experiences
    for i in 0..50 {
        let state = vec![0.1; 52];
        let action = (i % 3) as u8;
        let reward = if i % 2 == 0 { 1.0 } else { -0.5 };
        let next_state = vec![0.2; 52];
        let done = false;

        dqn.store_experience(Experience::new(state, action, reward, next_state, done))?;
    }

    // Train for 5 steps and collect gradient norms
    let mut gradient_norms = Vec::new();
    for _ in 0..5 {
        let (loss, grad_norm) = dqn.train_step(None)?;
        gradient_norms.push(grad_norm);

        println!("Loss: {:.6}, Gradient Norm: {:.6}", loss, grad_norm);
    }

    // ASSERTION 1: No gradient collapse (norm > 0.0001)
    for (i, &norm) in gradient_norms.iter().enumerate() {
        assert!(
            norm > 0.0001,
            "Gradient collapse detected at step {}: norm={:.6}",
            i,
            norm
        );
    }

    // ASSERTION 2: Gradient norm should be reasonable (0.1 < norm < 100.0)
    let avg_norm = gradient_norms.iter().sum::<f32>() / gradient_norms.len() as f32;
    assert!(
        avg_norm > 0.1 && avg_norm < 100.0,
        "Gradient norm out of range: avg={:.6} (expected 0.1-100.0)",
        avg_norm
    );

    // ASSERTION 3: Gradient norms should be stable (std_dev < 50% of mean)
    let variance: f32 = gradient_norms
        .iter()
        .map(|&x| (x - avg_norm).powi(2))
        .sum::<f32>()
        / gradient_norms.len() as f32;
    let std_dev = variance.sqrt();
    let stability_ratio = std_dev / avg_norm;

    assert!(
        stability_ratio < 0.5,
        "Gradient norms unstable: std_dev={:.6}, mean={:.6}, ratio={:.2}",
        std_dev,
        avg_norm,
        stability_ratio
    );

    println!("✓ Gradients flow correctly through all layers");
    println!(
        "  Avg norm: {:.4}, Std dev: {:.4}, Stability: {:.2}%",
        avg_norm,
        std_dev,
        stability_ratio * 100.0
    );

    Ok(())
}

/// Test: Dead neuron detection should check ACTIVATIONS, not weights
///
/// **Expected**: <10% dead neurons after 50 training steps
/// **Bug Symptom**: False positives due to checking weights instead of activations
#[test]
fn test_no_dead_neurons_after_training() -> Result<()> {
    // Create DQN
    let mut config = WorkingDQNConfig::emergency_safe_defaults();
    config.state_dim = 52;
    config.hidden_dims = vec![256, 128, 64];
    config.batch_size = 32;
    config.min_replay_size = 32;
    config.replay_buffer_capacity = 1000;

    let mut dqn = WorkingDQN::new(config)?;

    // Populate replay buffer
    for i in 0..100 {
        let state = vec![0.1 * (i as f32 / 100.0); 52];
        let action = (i % 3) as u8;
        let reward = if i % 2 == 0 { 1.0 } else { -0.5 };
        let next_state = vec![0.2 * (i as f32 / 100.0); 52];
        let done = false;

        dqn.store_experience(Experience::new(state, action, reward, next_state, done))?;
    }

    // Train for 50 steps
    for step in 0..50 {
        let (loss, grad_norm) = dqn.train_step(None)?;

        if step % 10 == 0 {
            println!(
                "Step {}: Loss={:.6}, Grad Norm={:.6}",
                step, loss, grad_norm
            );
        }
    }

    // Check weight distribution (manual dead neuron detection)
    // NOTE: This is a placeholder - actual implementation would need to access VarMap
    // and check ACTIVATION outputs (not weights) using a forward pass

    // ASSERTION: Network should still be training (gradient norm > 0.1)
    let (_, final_grad_norm) = dqn.train_step(None)?;
    assert!(
        final_grad_norm > 0.1,
        "Network appears dead: gradient norm={:.6} (expected >0.1)",
        final_grad_norm
    );

    println!("✓ No dead neurons detected after 50 training steps");
    println!("  Final gradient norm: {:.4}", final_grad_norm);

    Ok(())
}

/// Test: Xavier initialization produces correct variance
///
/// **Expected**: Variance ≈ 2/(fan_in + fan_out) for each layer
/// **Bug Symptom**: Incorrect initialization causing gradient flow issues
#[test]
fn test_xavier_initialization_variance() -> Result<()> {
    use ml::dqn::xavier_init::{verify_xavier_stats, xavier_uniform};

    let device = Device::cuda_if_available(0)?;

    // Test fc1: [52 → 256]
    let fc1_weights = xavier_uniform(52, 256, candle_core::DType::F32, &device)?;
    let (fc1_mean, fc1_var, fc1_expected) = verify_xavier_stats(&fc1_weights, 52, 256)?;

    println!(
        "FC1 (52→256): mean={:.6}, var={:.6}, expected={:.6}",
        fc1_mean, fc1_var, fc1_expected
    );

    // Test fc2: [256 → 128]
    let fc2_weights = xavier_uniform(256, 128, candle_core::DType::F32, &device)?;
    let (fc2_mean, fc2_var, fc2_expected) = verify_xavier_stats(&fc2_weights, 256, 128)?;

    println!(
        "FC2 (256→128): mean={:.6}, var={:.6}, expected={:.6}",
        fc2_mean, fc2_var, fc2_expected
    );

    // Test fc3: [128 → 64]
    let fc3_weights = xavier_uniform(128, 64, candle_core::DType::F32, &device)?;
    let (fc3_mean, fc3_var, fc3_expected) = verify_xavier_stats(&fc3_weights, 128, 64)?;

    println!(
        "FC3 (128→64): mean={:.6}, var={:.6}, expected={:.6}",
        fc3_mean, fc3_var, fc3_expected
    );

    // ASSERTION 1: Mean should be near zero (<0.05) for all layers
    assert!(fc1_mean.abs() < 0.05, "FC1 mean too high: {:.6}", fc1_mean);
    assert!(fc2_mean.abs() < 0.05, "FC2 mean too high: {:.6}", fc2_mean);
    assert!(fc3_mean.abs() < 0.05, "FC3 mean too high: {:.6}", fc3_mean);

    // ASSERTION 2: Variance should match Xavier formula (±20% tolerance)
    let fc1_diff = (fc1_var - fc1_expected).abs() / fc1_expected;
    let fc2_diff = (fc2_var - fc2_expected).abs() / fc2_expected;
    let fc3_diff = (fc3_var - fc3_expected).abs() / fc3_expected;

    assert!(
        fc1_diff < 0.20,
        "FC1 variance mismatch: {:.2}% (expected <20%)",
        fc1_diff * 100.0
    );
    assert!(
        fc2_diff < 0.20,
        "FC2 variance mismatch: {:.2}% (expected <20%)",
        fc2_diff * 100.0
    );
    assert!(
        fc3_diff < 0.20,
        "FC3 variance mismatch: {:.2}% (expected <20%)",
        fc3_diff * 100.0
    );

    println!("✓ Xavier initialization produces correct variance for all layers");

    Ok(())
}

/// Test: Q-value stability during training (no collapse to 0.0000)
///
/// **Expected**: Q-values should remain in range [-10, +10] and not collapse to zero
/// **Bug Symptom**: All Q-values converge to 0.0000 after few steps
#[test]
fn test_q_value_stability_during_training() -> Result<()> {
    // Create DQN
    let mut config = WorkingDQNConfig::emergency_safe_defaults();
    config.state_dim = 52;
    config.hidden_dims = vec![256, 128, 64];
    config.batch_size = 32;
    config.min_replay_size = 32;
    config.replay_buffer_capacity = 1000;

    let mut dqn = WorkingDQN::new(config)?;

    // Populate replay buffer
    for i in 0..100 {
        let state = vec![0.1 * (i as f32 / 100.0); 52];
        let action = (i % 3) as u8;
        let reward = if i % 2 == 0 { 1.0 } else { -0.5 };
        let next_state = vec![0.2 * (i as f32 / 100.0); 52];
        let done = false;

        dqn.store_experience(Experience::new(state, action, reward, next_state, done))?;
    }

    // Train and collect Q-values every 10 steps
    let device = dqn.device().clone();
    let test_state = Tensor::from_vec(vec![0.5_f32; 52], (1, 52), &device)?;

    let mut q_value_history = Vec::new();

    for step in 0..50 {
        dqn.train_step(None)?;

        if step % 10 == 0 {
            let q_values = dqn.forward(&test_state)?;
            let q_buy = q_values.i((0, 0))?.to_scalar::<f32>()?;
            let q_sell = q_values.i((0, 1))?.to_scalar::<f32>()?;
            let q_hold = q_values.i((0, 2))?.to_scalar::<f32>()?;

            q_value_history.push((q_buy, q_sell, q_hold));
            println!(
                "Step {}: Q-values = [{:.6}, {:.6}, {:.6}]",
                step, q_buy, q_sell, q_hold
            );
        }
    }

    // ASSERTION 1: Q-values should not all collapse to zero (< 0.0001)
    for (step, &(q_buy, q_sell, q_hold)) in q_value_history.iter().enumerate() {
        let max_q = q_buy.abs().max(q_sell.abs()).max(q_hold.abs());
        assert!(
            max_q > 0.0001,
            "Q-value collapse at step {}: [{:.6}, {:.6}, {:.6}]",
            step * 10,
            q_buy,
            q_sell,
            q_hold
        );
    }

    // ASSERTION 2: Q-values should remain in reasonable range [-10, +10]
    for (step, &(q_buy, q_sell, q_hold)) in q_value_history.iter().enumerate() {
        assert!(
            q_buy.abs() < 10.0 && q_sell.abs() < 10.0 && q_hold.abs() < 10.0,
            "Q-values exploded at step {}: [{:.6}, {:.6}, {:.6}]",
            step * 10,
            q_buy,
            q_sell,
            q_hold
        );
    }

    println!("✓ Q-values remain stable during training (no collapse or explosion)");

    Ok(())
}

/// Test: Gradient ratios between layers should be reasonable
///
/// **Expected**: Gradient norms should not differ by >10x between layers
/// **Bug Symptom**: Vanishing gradients (fc1 << fc3) due to 4x network expansion
#[test]
fn test_gradient_ratios_between_layers() -> Result<()> {
    // This test requires access to per-layer gradient norms
    // Current implementation only returns total gradient norm
    // TODO: Implement per-layer gradient extraction in DQN

    println!("⚠️  Test skipped: Per-layer gradient extraction not implemented");
    println!("   Required: Modify train_step() to return Vec<(layer_name, grad_norm)>");

    Ok(())
}

/// Test: LeakyReLU alpha=0.01 vs 0.1 comparison
///
/// **Expected**: Alpha=0.1 should reduce dead neurons and improve gradient flow
/// **Bug Symptom**: Alpha=0.01 too low, causing neuron death
#[test]
fn test_leaky_relu_alpha_comparison() -> Result<()> {
    // Test 1: Alpha=0.01 (current)
    let mut config_01 = WorkingDQNConfig::emergency_safe_defaults();
    config_01.state_dim = 52;
    config_01.leaky_relu_alpha = 0.01;
    config_01.batch_size = 32;
    config_01.min_replay_size = 32;

    let mut dqn_01 = WorkingDQN::new(config_01)?;

    // Populate replay buffer
    for i in 0..100 {
        let state = vec![0.1; 52];
        let action = (i % 3) as u8;
        let reward = if i % 2 == 0 { 1.0 } else { -0.5 };
        let next_state = vec![0.2; 52];
        let done = false;

        dqn_01.store_experience(Experience::new(state, action, reward, next_state, done))?;
    }

    // Train for 20 steps
    let mut grad_norms_01 = Vec::new();
    for _ in 0..20 {
        let (_, grad_norm) = dqn_01.train_step(None)?;
        grad_norms_01.push(grad_norm);
    }

    let avg_norm_01 = grad_norms_01.iter().sum::<f32>() / grad_norms_01.len() as f32;

    // Test 2: Alpha=0.1 (proposed)
    let mut config_10 = WorkingDQNConfig::emergency_safe_defaults();
    config_10.state_dim = 52;
    config_10.leaky_relu_alpha = 0.1;
    config_10.batch_size = 32;
    config_10.min_replay_size = 32;

    let mut dqn_10 = WorkingDQN::new(config_10)?;

    // Populate replay buffer (same data)
    for i in 0..100 {
        let state = vec![0.1; 52];
        let action = (i % 3) as u8;
        let reward = if i % 2 == 0 { 1.0 } else { -0.5 };
        let next_state = vec![0.2; 52];
        let done = false;

        dqn_10.store_experience(Experience::new(state, action, reward, next_state, done))?;
    }

    // Train for 20 steps
    let mut grad_norms_10 = Vec::new();
    for _ in 0..20 {
        let (_, grad_norm) = dqn_10.train_step(None)?;
        grad_norms_10.push(grad_norm);
    }

    let avg_norm_10 = grad_norms_10.iter().sum::<f32>() / grad_norms_10.len() as f32;

    println!("LeakyReLU Alpha Comparison:");
    println!("  Alpha=0.01: Avg grad norm = {:.4}", avg_norm_01);
    println!("  Alpha=0.10: Avg grad norm = {:.4}", avg_norm_10);

    // ASSERTION: Both should have reasonable gradient norms (>0.1)
    assert!(
        avg_norm_01 > 0.1,
        "Alpha=0.01 gradient collapse: {:.6}",
        avg_norm_01
    );
    assert!(
        avg_norm_10 > 0.1,
        "Alpha=0.10 gradient collapse: {:.6}",
        avg_norm_10
    );

    println!("✓ Both alpha values maintain gradient flow (no collapse)");

    Ok(())
}