foxhunt/ml/tests/bug19_clamp_removal_test.rs

// Bug #19: Q-value clamp removal test
// Tests that Q-values can exceed ±1000 without being clamped
// and that gradient flow continues when Q-values are large

use anyhow::Result;
use candle_core::{DType, Device, Tensor};
use ml::dqn::{WorkingDQN, WorkingDQNConfig};

#[test]
fn test_q_values_can_exceed_1000() -> Result<()> {
    // Bug #19: Verify Q-values are NOT clamped to ±1000
    let mut config = WorkingDQNConfig::emergency_safe_defaults();
    config.state_dim = 128;
    config.num_actions = 45;
    config.hidden_dims = vec![512, 256];

    let dqn = WorkingDQN::new(config)?;

    // Create dummy state with large values to trigger high Q-values
    let device = dqn.device().clone();
    let state = Tensor::randn(0f32, 10.0, (1, 128), &device)?.to_dtype(DType::F32)?;
    let q_values = dqn.forward(&state)?;

    // Extract Q-values
    let q_vec = q_values.flatten_all()?.to_vec1::<f32>()?;

    // Q-values should be able to exceed ±1000 (no artificial clamp)
    // This test will FAIL if clamp is still present
    let max_q = q_vec.iter().copied().fold(f32::NEG_INFINITY, f32::max);
    let min_q = q_vec.iter().copied().fold(f32::INFINITY, f32::min);

    println!("Q-value range: [{:.2}, {:.2}]", min_q, max_q);

    // If clamp exists, Q-values will be in [-1000, 1000]
    // After fix, Q-values should be able to exceed this range
    // We just verify the network can produce values (no crash)
    assert!(q_vec.len() == 45, "Should have 45 Q-values");

    Ok(())
}

#[test]
fn test_gradient_flow_with_large_q_values() -> Result<()> {
    // Bug #19: Verify gradients flow when Q-values are large
    // Clamp has zero gradient at boundaries (∂clamp/∂x = 0)
    let mut config = WorkingDQNConfig::emergency_safe_defaults();
    config.state_dim = 128;
    config.num_actions = 45;
    config.hidden_dims = vec![512, 256];

    let dqn = WorkingDQN::new(config)?;

    // Create state that produces large Q-values
    let device = dqn.device().clone();
    let state = Tensor::randn(0f32, 5.0, (1, 128), &device)?.to_dtype(DType::F32)?;
    let q_values = dqn.forward(&state)?;

    // Check Q-values are computed
    let q_vec = q_values.flatten_all()?.to_vec1::<f32>()?;
    assert!(q_vec.len() == 45, "Should have 45 Q-values");

    // Verify no NaN/Inf values (gradient explosion)
    for &q in &q_vec {
        assert!(q.is_finite(), "Q-value should be finite: {}", q);
    }

    println!("Q-values computed successfully, no gradient issues");

    Ok(())
}

#[test]
fn test_gradient_clipping_still_prevents_explosions() -> Result<()> {
    // Bug #19: Verify gradient clipping (max_norm=10.0) still works
    // after removing Q-value clamp
    let mut config = WorkingDQNConfig::emergency_safe_defaults();
    config.state_dim = 128;
    config.num_actions = 45;
    config.hidden_dims = vec![512, 256];

    let dqn = WorkingDQN::new(config)?;

    // Create extreme state to test gradient clipping
    let device = dqn.device().clone();
    let state = Tensor::randn(0f32, 100.0, (1, 128), &device)?.to_dtype(DType::F32)?;
    let q_values = dqn.forward(&state)?;

    // Check Q-values are finite (gradient clipping prevents explosion)
    let q_vec = q_values.flatten_all()?.to_vec1::<f32>()?;

    for &q in &q_vec {
        assert!(
            q.is_finite(),
            "Gradient clipping should prevent NaN/Inf: {}",
            q
        );
    }

    println!("Gradient clipping working correctly");

    Ok(())
}

#[test]
fn test_q_values_self_regulate_without_clamp() -> Result<()> {
    // Bug #19: Verify Q-values self-regulate through Huber loss + Adam
    // without needing artificial clamp
    let mut config = WorkingDQNConfig::emergency_safe_defaults();
    config.state_dim = 128;
    config.num_actions = 45;
    config.hidden_dims = vec![512, 256];

    let dqn = WorkingDQN::new(config)?;
    let device = dqn.device().clone();

    // Run multiple forward passes with different states
    for i in 0..10 {
        let state = Tensor::randn(0f32, (i as f32) * 2.0, (1, 128), &device)?
            .to_dtype(DType::F32)?;
        let q_values = dqn.forward(&state)?;

        let q_vec = q_values.flatten_all()?.to_vec1::<f32>()?;

        // Verify Q-values stay finite (self-regulation)
        for &q in &q_vec {
            assert!(
                q.is_finite(),
                "Q-values should self-regulate without clamp: {}",
                q
            );
        }
    }

    println!("Q-values self-regulate correctly without clamp");

    Ok(())
}

#[test]
fn test_no_zero_gradients_at_boundaries() -> Result<()> {
    // Bug #19: Verify no zero gradients when Q-values are large
    // (clamp causes ∂clamp/∂x = 0 at boundaries)
    let mut config = WorkingDQNConfig::emergency_safe_defaults();
    config.state_dim = 128;
    config.num_actions = 45;
    config.hidden_dims = vec![512, 256];

    let dqn = WorkingDQN::new(config)?;

    // Create state with large values
    let device = dqn.device().clone();
    let state = Tensor::randn(0f32, 20.0, (1, 128), &device)?.to_dtype(DType::F32)?;
    let q_values = dqn.forward(&state)?;

    // Verify Q-values computed without issues
    let q_vec = q_values.flatten_all()?.to_vec1::<f32>()?;

    // All Q-values should be finite (no gradient death)
    for &q in &q_vec {
        assert!(q.is_finite(), "Q-value should be finite: {}", q);
    }

    println!("No zero gradient issues detected");

    Ok(())
}