foxhunt/ml/tests/mamba2_accuracy_fix_test.rs

//! Test suite to verify MAMBA-2 accuracy calculation fix
//!
//! This test suite validates the fix for the accuracy calculation bug where
//! mean_all() was incorrectly used on incompatible tensor shapes, causing
//! 99% error rates and 3-12% "accuracy" despite normal loss convergence.

use candle_core::{Device, IndexOp, Tensor};

/// Test accuracy calculation with single-value target (basic case)
#[test]
fn test_accuracy_calculation_single_value() {
    let device = Device::Cpu;

    // Simulate normalized predictions and targets
    let pred = Tensor::new(&[[[0.48]]], &device).unwrap(); // Predict 0.48
    let target = Tensor::new(&[[[0.50]]], &device).unwrap(); // Target 0.50

    // Expected MAPE: |0.48 - 0.50| / 0.50 = 0.04 = 4% error
    // Should be CORRECT with 30% threshold

    // NEW FIX (scalar extraction):
    let pred_val = pred.i((0, 0, 0)).unwrap().to_scalar::<f64>().unwrap(); // 0.48
    let target_val = target.i((0, 0, 0)).unwrap().to_scalar::<f64>().unwrap(); // 0.50
    let error = ((pred_val - target_val) / target_val).abs();

    assert!(
        (error - 0.04).abs() < 1e-6,
        "Expected 4% error, got {}%",
        error * 100.0
    );

    // ✅ With 30% threshold, this should be marked "correct"
    assert!(
        error < 0.3,
        "4% error should be considered correct with 30% threshold"
    );

    // Also verify it passes the stricter 10% threshold
    assert!(
        error < 0.1,
        "4% error should also pass 10% threshold (but 10% is too strict for production)"
    );
}

/// Test accuracy calculation with multi-dimensional output (realistic MAMBA-2 case)
#[test]
fn test_accuracy_calculation_multi_dim_output() {
    let device = Device::Cpu;

    // Simulate realistic MAMBA-2 output: [1, 1, 54]
    let mut output_data = vec![0.0; 54];
    output_data[0] = 0.48; // First feature is regression target

    let pred = Tensor::from_vec(output_data.clone(), (1, 1, 54), &device).unwrap();
    let target = Tensor::new(&[[[0.50]]], &device).unwrap();

    // OLD BUG (mean_all): Would give 99% error
    let old_pred_mean = pred.mean_all().unwrap().to_scalar::<f64>().unwrap();
    // old_pred_mean ≈ 0.48/54 ≈ 0.0021
    let old_target_mean = target.mean_all().unwrap().to_scalar::<f64>().unwrap(); // 0.50
    let old_error = ((old_pred_mean - old_target_mean) / old_target_mean).abs();

    println!(
        "OLD BUG: pred_mean={:.6}, target_mean={:.6}, error={:.2}%",
        old_pred_mean,
        old_target_mean,
        old_error * 100.0
    );
    assert!(
        old_error > 0.9,
        "OLD BUG: Should show ~99% error due to mean_all() on 54-dim output"
    );

    // NEW FIX (scalar extraction from first feature):
    let new_pred_val = pred.i((0, 0, 0)).unwrap().to_scalar::<f64>().unwrap(); // 0.48
    let new_target_val = target.i((0, 0, 0)).unwrap().to_scalar::<f64>().unwrap(); // 0.50
    let new_error = ((new_pred_val - new_target_val) / new_target_val).abs();

    println!(
        "NEW FIX: pred_val={:.6}, target_val={:.6}, error={:.2}%",
        new_pred_val,
        new_target_val,
        new_error * 100.0
    );
    assert!(
        (new_error - 0.04).abs() < 1e-6,
        "NEW FIX: Should show 4% error, got {}%",
        new_error * 100.0
    );

    // ✅ NEW: 4% error → "correct" with 30% threshold
    assert!(
        new_error < 0.3,
        "NEW FIX: 4% error should be considered correct"
    );
}

/// Test edge case: target near zero (avoid division by zero)
#[test]
fn test_accuracy_calculation_near_zero_target() {
    let device = Device::Cpu;

    let pred = Tensor::new(&[[[0.02]]], &device).unwrap();
    let target = Tensor::new(&[[[1e-9]]], &device).unwrap(); // Very near zero (below 1e-8)

    // Extract scalar values
    let pred_val = pred.i((0, 0, 0)).unwrap().to_scalar::<f64>().unwrap();
    let target_val = target.i((0, 0, 0)).unwrap().to_scalar::<f64>().unwrap();

    // For targets near zero (< 1e-8), use absolute error instead of percentage
    let error = if target_val.abs() > 1e-8 {
        ((pred_val - target_val) / target_val).abs()
    } else {
        (pred_val - target_val).abs()
    };

    println!(
        "Near-zero target: pred={:.6}, target={:.9}, error={:.6}, using_absolute_error={}",
        pred_val,
        target_val,
        error,
        target_val.abs() <= 1e-8
    );

    // Should use absolute error (0.02 - 1e-9 ≈ 0.02)
    assert!(
        (error - 0.02).abs() < 1e-6,
        "Expected absolute error ~0.02, got {}",
        error
    );
}

/// Test threshold sensitivity: 10% vs 30%
#[test]
fn test_threshold_comparison() {
    let device = Device::Cpu;

    // Test different error levels
    let test_cases = vec![
        (0.48, 0.50, 0.04), // 4% error - should pass both thresholds
        (0.42, 0.50, 0.16), // 16% error - should pass 30% but fail 10%
        (0.30, 0.50, 0.40), // 40% error - should fail both thresholds
    ];

    for (pred_val, target_val, expected_error) in test_cases {
        let pred = Tensor::new(&[[[pred_val]]], &device).unwrap();
        let target = Tensor::new(&[[[target_val]]], &device).unwrap();

        let pred_scalar = pred.i((0, 0, 0)).unwrap().to_scalar::<f64>().unwrap();
        let target_scalar = target.i((0, 0, 0)).unwrap().to_scalar::<f64>().unwrap();
        let error = ((pred_scalar - target_scalar) / target_scalar).abs();

        println!(
            "Pred={:.2}, Target={:.2}, Error={:.2}% (expected {:.2}%)",
            pred_val,
            target_val,
            error * 100.0,
            expected_error * 100.0
        );

        assert!(
            (error - expected_error).abs() < 1e-6,
            "Expected {:.2}% error, got {:.2}%",
            expected_error * 100.0,
            error * 100.0
        );

        // Check threshold behavior
        let passes_10 = error < 0.1;
        let passes_30 = error < 0.3;

        match expected_error {
            e if e < 0.1 => {
                assert!(
                    passes_10,
                    "Error {:.2}% should pass 10% threshold",
                    e * 100.0
                );
                assert!(
                    passes_30,
                    "Error {:.2}% should pass 30% threshold",
                    e * 100.0
                );
            },
            e if e < 0.3 => {
                assert!(
                    !passes_10,
                    "Error {:.2}% should fail 10% threshold",
                    e * 100.0
                );
                assert!(
                    passes_30,
                    "Error {:.2}% should pass 30% threshold",
                    e * 100.0
                );
            },
            e => {
                assert!(
                    !passes_10,
                    "Error {:.2}% should fail 10% threshold",
                    e * 100.0
                );
                assert!(
                    !passes_30,
                    "Error {:.2}% should fail 30% threshold",
                    e * 100.0
                );
            },
        }
    }
}

/// Test realistic ES futures price prediction scenario
#[test]
fn test_realistic_futures_prediction() {
    let device = Device::Cpu;

    // ES futures: price range $5000-$5200 (normalized to 0.0-1.0)
    // Example: predict $5095, actual $5100
    // Normalized: predict 0.475, actual 0.5
    // Error: $5 out of $200 range = 2.5% in price space
    // MAPE: |0.475 - 0.5| / 0.5 = 5% in normalized space

    let pred = Tensor::new(&[[[0.475]]], &device).unwrap();
    let target = Tensor::new(&[[[0.50]]], &device).unwrap();

    let pred_val = pred.i((0, 0, 0)).unwrap().to_scalar::<f64>().unwrap();
    let target_val = target.i((0, 0, 0)).unwrap().to_scalar::<f64>().unwrap();
    let error_pct = ((pred_val - target_val) / target_val).abs();

    println!(
        "Realistic ES prediction: pred={:.3}, target={:.3}, error={:.2}%",
        pred_val,
        target_val,
        error_pct * 100.0
    );

    // 5% error should be considered EXCELLENT for financial prediction
    assert!(error_pct < 0.1, "5% error should easily pass 10% threshold");
    assert!(error_pct < 0.3, "5% error should easily pass 30% threshold");

    // In price terms: $5 error on $5100 = 0.098% in absolute terms
    // This is EXCELLENT prediction accuracy for intraday futures
}

/// Test batch of predictions to estimate accuracy rate
#[test]
fn test_batch_accuracy_estimation() {
    let device = Device::Cpu;

    // Simulate 100 predictions with varying errors
    let mut errors = vec![];

    // Generate predictions with normal distribution around target
    for i in 0..100 {
        let target_val = 0.5;
        // Add noise: ±15% RMSE → most predictions within ±30%
        let noise = (i as f64 / 100.0 - 0.5) * 0.3; // -15% to +15%
        let pred_val = target_val + noise;

        let pred = Tensor::new(&[[[pred_val]]], &device).unwrap();
        let target = Tensor::new(&[[[target_val]]], &device).unwrap();

        let pred_scalar = pred.i((0, 0, 0)).unwrap().to_scalar::<f64>().unwrap();
        let target_scalar = target.i((0, 0, 0)).unwrap().to_scalar::<f64>().unwrap();
        let error = ((pred_scalar - target_scalar) / target_scalar).abs();

        errors.push(error);
    }

    // Count how many predictions are "correct" with different thresholds
    let correct_10 = errors.iter().filter(|&&e| e < 0.1).count();
    let correct_30 = errors.iter().filter(|&&e| e < 0.3).count();

    let accuracy_10 = correct_10 as f64 / 100.0;
    let accuracy_30 = correct_30 as f64 / 100.0;

    println!(
        "Batch accuracy estimation (100 samples): 10% threshold={:.1}%, 30% threshold={:.1}%",
        accuracy_10 * 100.0,
        accuracy_30 * 100.0
    );

    // With ±15% noise, expect:
    // - 10% threshold: ~33% accuracy (1/3 within ±10%)
    // - 30% threshold: ~100% accuracy (all within ±15%)
    assert!(
        accuracy_10 > 0.20 && accuracy_10 < 0.50,
        "Expected 20-50% accuracy with 10% threshold, got {:.1}%",
        accuracy_10 * 100.0
    );
    assert!(
        accuracy_30 > 0.90,
        "Expected >90% accuracy with 30% threshold, got {:.1}%",
        accuracy_30 * 100.0
    );
}

/// Integration test: verify fix aligns accuracy with loss
#[test]
fn test_accuracy_loss_alignment() {
    // Given: Training loss = 0.071 (MSE in normalized space)
    // RMSE = sqrt(0.071) = 0.266 = 26.6% error
    //
    // With 30% MAPE threshold:
    // - Predictions with <30% error marked "correct"
    // - RMSE 26.6% means ~68% of predictions within ±30% (assuming normal distribution)
    // - Expected accuracy: ~68-75%

    let expected_rmse = 0.266;
    let threshold = 0.3;

    // Approximate: for RMSE R and threshold T, accuracy ≈ erf(T/R*sqrt(2))
    // For R=0.266, T=0.3: accuracy ≈ erf(1.13*sqrt(2)) ≈ erf(1.6) ≈ 0.976
    // But this assumes normal distribution centered at target
    //
    // More conservative estimate: if RMSE=26.6%, about 68-75% within ±30%

    println!(
        "Loss-Accuracy Alignment: RMSE={:.1}%, Threshold={:.1}%",
        expected_rmse * 100.0,
        threshold * 100.0
    );
    println!("Expected accuracy: 68-75% (most predictions within threshold)");

    // Verify threshold is reasonable for this RMSE
    assert!(
        threshold > expected_rmse,
        "Threshold ({:.1}%) should be greater than RMSE ({:.1}%) for reasonable accuracy",
        threshold * 100.0,
        expected_rmse * 100.0
    );
}