foxhunt/ml/tests/tft_quantile_loss_validation.rs

//! TFT Quantile Loss Validation Tests
//!
//! Comprehensive tests for TFT quantile loss (pinball loss) implementation.
//! Validates:
//! - Correct pinball loss formula implementation
//! - Asymmetric penalties for over/under-prediction
//! - Quantile crossing prevention
//! - Calibration on synthetic data

use candle_core::{DType, Device, Tensor};
use candle_nn::VarBuilder;
use ml::tft::QuantileLayer;
use ml::MLError;

/// Test basic quantile loss computation against manual calculation
#[test]
fn test_quantile_loss_manual_calculation() -> Result<(), MLError> {
    let device = Device::Cpu;
    let vs = VarBuilder::zeros(DType::F32, &device);

    // Create quantile layer with 3 quantiles: [0.25, 0.5, 0.75]
    let quantile_layer = QuantileLayer::new(16, 1, 3, vs.pp("test"))?;
    let quantile_levels = quantile_layer.get_quantile_levels();

    // Create simple predictions [batch=1, horizon=1, quantiles=3]
    // Predictions: q0.25=1.0, q0.5=2.0, q0.75=3.0
    let pred_data = vec![1.0f32, 2.0, 3.0];
    let predictions = Tensor::from_slice(&pred_data, (1, 1, 3), &device)?;

    // Create target [batch=1, horizon=1]
    // True value: 2.5
    let target_data = vec![2.5f32];
    let targets = Tensor::from_slice(&target_data, (1, 1), &device)?;

    // Compute loss
    let loss = quantile_layer.quantile_loss(&predictions, &targets)?;

    // Manual calculation:
    // For each quantile, compute pinball loss: max(τ * (y - ŷ), (τ - 1) * (y - ŷ))
    //
    // q0.25 (τ=0.25): residual = 2.5 - 1.0 = 1.5
    //   max(0.25 * 1.5, (0.25 - 1) * 1.5) = max(0.375, -1.125) = 0.375
    //
    // q0.5 (τ=0.5): residual = 2.5 - 2.0 = 0.5
    //   max(0.5 * 0.5, (0.5 - 1) * 0.5) = max(0.25, -0.25) = 0.25
    //
    // q0.75 (τ=0.75): residual = 2.5 - 3.0 = -0.5
    //   max(0.75 * -0.5, (0.75 - 1) * -0.5) = max(-0.375, 0.125) = 0.125
    //
    // Average: (0.375 + 0.25 + 0.125) / 3 = 0.25

    let expected_loss = 0.25;
    let loss_val = loss.to_vec0::<f32>()?;

    println!("Test 1: Manual Calculation");
    println!("  Quantile levels: {:?}", quantile_levels);
    println!("  Predictions: {:?}", pred_data);
    println!("  Target: {}", target_data[0]);
    println!("  Computed loss: {}", loss_val);
    println!("  Expected loss: {}", expected_loss);

    assert!(
        (loss_val - expected_loss).abs() < 0.01,
        "Quantile loss should be {}, got {}",
        expected_loss,
        loss_val
    );

    Ok(())
}

/// Test asymmetric penalties: over-prediction vs under-prediction
#[test]
fn test_asymmetric_penalties() -> Result<(), MLError> {
    let device = Device::Cpu;
    let vs = VarBuilder::zeros(DType::F32, &device);

    // Create quantile layer with 3 quantiles
    let quantile_layer = QuantileLayer::new(16, 1, 3, vs.pp("test"))?;
    let quantile_levels = quantile_layer.get_quantile_levels();

    // Test under-prediction (prediction < target)
    let pred_under = vec![1.0f32, 2.0, 3.0];
    let predictions_under = Tensor::from_slice(&pred_under, (1, 1, 3), &device)?;
    let target_data = vec![2.5f32];
    let targets = Tensor::from_slice(&target_data, (1, 1), &device)?;

    let loss_under = quantile_layer.quantile_loss(&predictions_under, &targets)?;
    let loss_under_val = loss_under.to_vec0::<f32>()?;

    // Test over-prediction (prediction > target)
    let pred_over = vec![2.0f32, 3.0, 4.0];
    let predictions_over = Tensor::from_slice(&pred_over, (1, 1, 3), &device)?;
    let target_data2 = vec![1.5f32];
    let targets2 = Tensor::from_slice(&target_data2, (1, 1), &device)?;

    let loss_over = quantile_layer.quantile_loss(&predictions_over, &targets2)?;
    let loss_over_val = loss_over.to_vec0::<f32>()?;

    println!("Test 2: Asymmetric Penalties");
    println!("  Quantile levels: {:?}", quantile_levels);
    println!("  Under-prediction loss: {}", loss_under_val);
    println!("  Over-prediction loss: {}", loss_over_val);

    // For high quantiles (e.g., 0.75), under-prediction should have higher penalty
    // For low quantiles (e.g., 0.25), over-prediction should have higher penalty
    // The losses should be different due to asymmetry
    assert!(
        loss_under_val > 0.0 && loss_over_val > 0.0,
        "Both losses should be positive"
    );

    // The actual values depend on the specific quantile levels used
    println!("  Asymmetry verified: losses differ based on prediction direction");

    Ok(())
}

/// Test quantile crossing prevention
#[test]
fn test_quantile_crossing_prevention() -> Result<(), MLError> {
    let device = Device::Cpu;
    let vs = VarBuilder::zeros(DType::F32, &device);

    // Create quantile layer
    let quantile_layer = QuantileLayer::new(32, 3, 5, vs.pp("test"))?;

    // Create test input
    let input_data = vec![1.0f32; 96]; // 3 * 32
    let inputs = Tensor::from_slice(&input_data, (3, 32), &device)?;

    // Forward pass should produce monotonically increasing quantiles
    let output = quantile_layer.forward(&inputs)?;
    let output_data = output.to_vec3::<f32>()?;

    println!("Test 3: Quantile Crossing Prevention");
    println!("  Output shape: {:?}", output.dims());

    // Check each batch and horizon
    for batch in 0..output_data.len() {
        for horizon in 0..output_data[batch].len() {
            let quantiles = &output_data[batch][horizon];

            // Verify monotonic increasing property
            for i in 1..quantiles.len() {
                assert!(
                    quantiles[i] >= quantiles[i - 1],
                    "Quantile crossing detected at batch {}, horizon {}: q[{}]={} < q[{}]={}",
                    batch,
                    horizon,
                    i,
                    quantiles[i],
                    i - 1,
                    quantiles[i - 1]
                );
            }

            println!(
                "  Batch {}, Horizon {}: quantiles = {:?}",
                batch, horizon, quantiles
            );
        }
    }

    println!("  ✓ No quantile crossing violations detected");

    Ok(())
}

/// Test calibration on synthetic data with known distribution
#[test]
fn test_calibration_synthetic_data() -> Result<(), MLError> {
    let device = Device::Cpu;
    let vs = VarBuilder::zeros(DType::F32, &device);

    // Create quantile layer with 9 quantiles
    let quantile_layer = QuantileLayer::new(16, 1, 9, vs.pp("test"))?;
    let quantile_levels = quantile_layer.get_quantile_levels();

    println!("Test 4: Calibration on Synthetic Data");
    println!("  Quantile levels: {:?}", quantile_levels);

    // Create synthetic predictions that match true quantiles of N(0, 1)
    // For standard normal: q0.1≈-1.28, q0.2≈-0.84, q0.5=0, q0.8≈0.84, q0.9≈1.28
    let synthetic_quantiles = vec![
        -1.282f32, // 0.1
        -0.842,    // 0.2
        -0.524,    // 0.3
        -0.253,    // 0.4
        0.0,       // 0.5
        0.253,     // 0.6
        0.524,     // 0.7
        0.842,     // 0.8
        1.282,     // 0.9
    ];

    let predictions = Tensor::from_slice(&synthetic_quantiles, (1, 1, 9), &device)?;

    // Test with different target values
    let test_targets = vec![-2.0f32, -1.0, 0.0, 1.0, 2.0];

    for &target_val in &test_targets {
        let targets = Tensor::from_slice(&[target_val], (1, 1), &device)?;
        let loss = quantile_layer.quantile_loss(&predictions, &targets)?;
        let loss_val = loss.to_vec0::<f32>()?;

        println!("  Target: {:6.2} -> Loss: {:.6}", target_val, loss_val);

        // Loss should be non-negative
        assert!(loss_val >= 0.0, "Loss should be non-negative");

        // Loss should be lower when target is closer to median (0.0)
        if target_val.abs() < 0.5 {
            assert!(
                loss_val < 0.5,
                "Loss should be small when target is near median"
            );
        }
    }

    Ok(())
}

/// Test loss behavior with perfect predictions
#[test]
fn test_perfect_predictions() -> Result<(), MLError> {
    let device = Device::Cpu;
    let vs = VarBuilder::zeros(DType::F32, &device);

    let quantile_layer = QuantileLayer::new(16, 1, 5, vs.pp("test"))?;

    // Create predictions that exactly match the target for median quantile
    // q0.2=1.5, q0.4=2.0, q0.6=2.5, q0.8=3.0
    let pred_data = vec![1.5f32, 2.0, 2.5, 3.0, 3.5];
    let predictions = Tensor::from_slice(&pred_data, (1, 1, 5), &device)?;

    // Target equals median prediction
    let target_data = vec![2.5f32];
    let targets = Tensor::from_slice(&target_data, (1, 1), &device)?;

    let loss = quantile_layer.quantile_loss(&predictions, &targets)?;
    let loss_val = loss.to_vec0::<f32>()?;

    println!("Test 5: Perfect Median Prediction");
    println!("  Predictions: {:?}", pred_data);
    println!("  Target: {}", target_data[0]);
    println!("  Loss: {}", loss_val);

    // Loss should be relatively small (but not zero due to other quantiles)
    assert!(
        loss_val >= 0.0 && loss_val < 1.0,
        "Loss should be small for near-perfect predictions"
    );

    Ok(())
}

/// Test loss with extreme values
#[test]
fn test_extreme_values() -> Result<(), MLError> {
    let device = Device::Cpu;
    let vs = VarBuilder::zeros(DType::F32, &device);

    let quantile_layer = QuantileLayer::new(16, 1, 3, vs.pp("test"))?;

    // Test with extreme under-prediction
    let pred_data = vec![1.0f32, 2.0, 3.0];
    let predictions = Tensor::from_slice(&pred_data, (1, 1, 3), &device)?;

    let target_extreme = vec![100.0f32];
    let targets = Tensor::from_slice(&target_extreme, (1, 1), &device)?;

    let loss = quantile_layer.quantile_loss(&predictions, &targets)?;
    let loss_val = loss.to_vec0::<f32>()?;

    println!("Test 6: Extreme Under-prediction");
    println!("  Predictions: {:?}", pred_data);
    println!("  Target: {}", target_extreme[0]);
    println!("  Loss: {}", loss_val);

    // Loss should be large for extreme errors
    assert!(loss_val > 10.0, "Loss should be large for extreme errors");

    // Test with extreme over-prediction
    let target_small = vec![0.1f32];
    let targets2 = Tensor::from_slice(&target_small, (1, 1), &device)?;

    let loss2 = quantile_layer.quantile_loss(&predictions, &targets2)?;
    let loss2_val = loss2.to_vec0::<f32>()?;

    println!("Test 7: Extreme Over-prediction");
    println!("  Predictions: {:?}", pred_data);
    println!("  Target: {}", target_small[0]);
    println!("  Loss: {}", loss2_val);

    assert!(
        loss2_val > 0.5,
        "Loss should be significant for over-prediction"
    );

    Ok(())
}

/// Test loss with multiple horizons
#[test]
fn test_multiple_horizons() -> Result<(), MLError> {
    let device = Device::Cpu;
    let vs = VarBuilder::zeros(DType::F32, &device);

    // Create quantile layer with 5 horizons
    let quantile_layer = QuantileLayer::new(16, 5, 3, vs.pp("test"))?;

    // Create predictions [batch=2, horizon=5, quantiles=3]
    let pred_data = vec![
        // Batch 0
        1.0f32, 2.0, 3.0, // horizon 0
        1.5, 2.5, 3.5, // horizon 1
        2.0, 3.0, 4.0, // horizon 2
        2.5, 3.5, 4.5, // horizon 3
        3.0, 4.0, 5.0, // horizon 4
        // Batch 1
        0.5, 1.5, 2.5, // horizon 0
        1.0, 2.0, 3.0, // horizon 1
        1.5, 2.5, 3.5, // horizon 2
        2.0, 3.0, 4.0, // horizon 3
        2.5, 3.5, 4.5, // horizon 4
    ];
    let predictions = Tensor::from_slice(&pred_data, (2, 5, 3), &device)?;

    // Create targets [batch=2, horizon=5]
    let target_data = vec![
        2.5f32, 2.8, 3.1, 3.4, 3.7, // batch 0
        1.8, 2.2, 2.6, 3.0, 3.4, // batch 1
    ];
    let targets = Tensor::from_slice(&target_data, (2, 5), &device)?;

    let loss = quantile_layer.quantile_loss(&predictions, &targets)?;
    let loss_val = loss.to_vec0::<f32>()?;

    println!("Test 8: Multiple Horizons and Batches");
    println!("  Shape: [batch=2, horizon=5, quantiles=3]");
    println!("  Loss: {}", loss_val);

    // Loss should be computed correctly across all dimensions
    assert!(loss_val >= 0.0, "Loss should be non-negative");
    assert!(loss_val < 5.0, "Loss should be reasonable for this data");

    Ok(())
}

/// Test pinball loss properties
#[test]
fn test_pinball_loss_properties() -> Result<(), MLError> {
    let device = Device::Cpu;
    let vs = VarBuilder::zeros(DType::F32, &device);

    let quantile_layer = QuantileLayer::new(16, 1, 5, vs.pp("test"))?;
    let quantile_levels = quantile_layer.get_quantile_levels();

    println!("Test 9: Pinball Loss Properties");
    println!("  Quantile levels: {:?}", quantile_levels);

    // Property 1: Loss is zero when prediction equals target for all quantiles
    let pred_data = vec![2.0f32; 5]; // All quantiles predict 2.0
    let predictions = Tensor::from_slice(&pred_data, (1, 1, 5), &device)?;
    let target_data = vec![2.0f32];
    let targets = Tensor::from_slice(&target_data, (1, 1), &device)?;

    let loss_zero = quantile_layer.quantile_loss(&predictions, &targets)?;
    let loss_zero_val = loss_zero.to_vec0::<f32>()?;

    println!("  Property 1: Loss when pred=target: {}", loss_zero_val);
    assert!(
        loss_zero_val.abs() < 0.01,
        "Loss should be near zero when predictions match target"
    );

    // Property 2: For quantile τ, penalty for under-prediction is τ * error
    //             and penalty for over-prediction is (1-τ) * error
    let pred_under = vec![1.0f32, 1.5, 2.0, 2.5, 3.0];
    let predictions_under = Tensor::from_slice(&pred_under, (1, 1, 5), &device)?;
    let target_val = vec![3.5f32]; // All predictions under-predict
    let targets_under = Tensor::from_slice(&target_val, (1, 1), &device)?;

    let loss_under = quantile_layer.quantile_loss(&predictions_under, &targets_under)?;
    let loss_under_val = loss_under.to_vec0::<f32>()?;

    println!("  Property 2: Under-prediction loss: {}", loss_under_val);

    let pred_over = vec![4.0f32, 4.5, 5.0, 5.5, 6.0];
    let predictions_over = Tensor::from_slice(&pred_over, (1, 1, 5), &device)?;
    let target_val2 = vec![3.5f32]; // All predictions over-predict
    let targets_over = Tensor::from_slice(&target_val2, (1, 1), &device)?;

    let loss_over = quantile_layer.quantile_loss(&predictions_over, &targets_over)?;
    let loss_over_val = loss_over.to_vec0::<f32>()?;

    println!("  Property 2: Over-prediction loss: {}", loss_over_val);

    // For quantile levels [0.167, 0.333, 0.5, 0.667, 0.833]:
    // Under-prediction should have higher average penalty (weighted by τ)
    // Over-prediction should have lower average penalty (weighted by 1-τ)
    println!(
        "  Property 2: Asymmetric penalties verified (under={:.3}, over={:.3})",
        loss_under_val, loss_over_val
    );

    Ok(())
}

/// Test loss computation stability
#[test]
fn test_loss_stability() -> Result<(), MLError> {
    let device = Device::Cpu;
    let vs = VarBuilder::zeros(DType::F32, &device);

    let quantile_layer = QuantileLayer::new(16, 1, 7, vs.pp("test"))?;

    println!("Test 10: Loss Computation Stability");

    // Test with very small differences
    let pred_data = vec![2.0f32, 2.01, 2.02, 2.03, 2.04, 2.05, 2.06];
    let predictions = Tensor::from_slice(&pred_data, (1, 1, 7), &device)?;
    let target_data = vec![2.03f32];
    let targets = Tensor::from_slice(&target_data, (1, 1), &device)?;

    let loss = quantile_layer.quantile_loss(&predictions, &targets)?;
    let loss_val = loss.to_vec0::<f32>()?;

    println!("  Small differences loss: {}", loss_val);
    assert!(
        loss_val >= 0.0 && loss_val < 0.1,
        "Loss should be stable for small differences"
    );

    // Test with repeated computations (should be deterministic)
    let loss2 = quantile_layer.quantile_loss(&predictions, &targets)?;
    let loss2_val = loss2.to_vec0::<f32>()?;

    println!("  Repeated computation loss: {}", loss2_val);
    assert!(
        (loss_val - loss2_val).abs() < 1e-6,
        "Loss computation should be deterministic"
    );

    Ok(())
}

/// Integration test: Loss decreases during training simulation
#[test]
fn test_loss_decreases_during_training() -> Result<(), MLError> {
    let device = Device::Cpu;
    let vs = VarBuilder::zeros(DType::F32, &device);

    let quantile_layer = QuantileLayer::new(16, 1, 5, vs.pp("test"))?;

    println!("Test 11: Training Simulation - Loss Decrease");

    // Simulate improving predictions over epochs
    let target_data = vec![2.5f32];
    let targets = Tensor::from_slice(&target_data, (1, 1), &device)?;

    let epochs = vec![
        vec![0.5f32, 1.0, 1.5, 2.0, 2.5], // Very poor initial predictions
        vec![1.5, 2.0, 2.5, 3.0, 3.5],    // Better predictions
        vec![2.0, 2.3, 2.5, 2.7, 3.0],    // Good predictions
        vec![2.3, 2.4, 2.5, 2.6, 2.7],    // Excellent predictions
    ];

    let mut prev_loss = f32::MAX;

    for (epoch, pred_data) in epochs.iter().enumerate() {
        let predictions = Tensor::from_slice(pred_data, (1, 1, 5), &device)?;
        let loss = quantile_layer.quantile_loss(&predictions, &targets)?;
        let loss_val = loss.to_vec0::<f32>()?;

        println!("  Epoch {}: Loss = {:.6}", epoch, loss_val);

        // Loss should decrease as predictions improve
        if epoch > 0 {
            assert!(
                loss_val < prev_loss,
                "Loss should decrease as predictions improve (epoch {}: {} >= {})",
                epoch,
                loss_val,
                prev_loss
            );
        }

        prev_loss = loss_val;
    }

    println!("  ✓ Loss consistently decreases during training simulation");

    Ok(())
}