foxhunt/ml/tests/preprocessing_bessel_integration.rs

//! Integration Test: Preprocessing Module Uses Bessel's Correction
//!
//! Validates that the windowed_normalize function in the preprocessing module
//! correctly applies Bessel's correction when computing variance.

use candle_core::{Device, Tensor};
use ml::preprocessing::windowed_normalize;

/// Manually compute expected z-scores with Bessel's correction
fn compute_expected_zscore_unbiased(data: &[f32], window_size: usize) -> Vec<f32> {
    let mut result = Vec::with_capacity(data.len());

    for i in 0..data.len() {
        let start = if i + 1 >= window_size {
            i + 1 - window_size
        } else {
            0
        };

        let window = &data[start..=i];

        // Compute mean
        let mean: f32 = window.iter().sum::<f32>() / window.len() as f32;

        // Compute variance with Bessel's correction (N-1)
        let variance: f32 = if window.len() > 1 {
            window.iter().map(|&x| (x - mean).powi(2)).sum::<f32>() / (window.len() - 1) as f32
        } else {
            0.0
        };
        let std = variance.sqrt();

        // Compute z-score
        let eps = 1e-8;
        let z_score = if std > eps {
            (data[i] - mean) / std
        } else {
            0.0
        };

        result.push(z_score);
    }

    result
}

#[test]
fn test_preprocessing_uses_bessel_correction() {
    // Given: Simple test data
    let data = vec![1.0f32, 2.0, 3.0, 4.0, 5.0];
    let window_size = 3;

    // When: Apply windowed normalization from preprocessing module
    let data_tensor = Tensor::from_slice(&data, (data.len(),), &Device::Cpu).unwrap();
    let normalized = windowed_normalize(&data_tensor, window_size as i64).unwrap();
    let normalized_vec: Vec<f32> = normalized.to_vec1().unwrap();

    // Then: Should match manual calculation with Bessel's correction
    let expected = compute_expected_zscore_unbiased(&data, window_size);

    for (i, (&actual, &expected)) in normalized_vec.iter().zip(expected.iter()).enumerate() {
        let diff = (actual - expected).abs();
        assert!(
            diff < 1e-5,
            "Index {}: actual={}, expected={}, diff={}",
            i,
            actual,
            expected,
            diff
        );
    }
}

#[test]
fn test_preprocessing_bessel_vs_biased() {
    // Given: Data where Bessel's correction makes a significant difference
    let data = vec![100.0f32, 110.0, 105.0]; // Small sample (N=3)
    let window_size = 3;

    // When: Apply windowed normalization (should use Bessel's correction)
    let data_tensor = Tensor::from_slice(&data, (data.len(),), &Device::Cpu).unwrap();
    let normalized = windowed_normalize(&data_tensor, window_size as i64).unwrap();
    let normalized_vec: Vec<f32> = normalized.to_vec1().unwrap();

    // Then: Verify it matches unbiased calculation
    let expected_unbiased = compute_expected_zscore_unbiased(&data, window_size);

    // Last value should be properly normalized with unbiased estimator
    let last_actual = normalized_vec[2];
    let last_expected = expected_unbiased[2];

    assert!(
        (last_actual - last_expected).abs() < 1e-5,
        "Preprocessing should use unbiased estimator. actual={}, expected={}",
        last_actual,
        last_expected
    );

    // Verify that it's different from biased calculation (for documentation)
    // Mean of [100, 110, 105] = 105
    // Variance (biased): [(100-105)^2 + (110-105)^2 + (105-105)^2] / 3 = 50/3 ≈ 16.67
    // Variance (unbiased): 50 / 2 = 25.0
    // Std (biased): sqrt(16.67) ≈ 4.08
    // Std (unbiased): sqrt(25.0) = 5.0
    // Z-score for 105: (105-105)/std = 0.0 (same for both, but demonstrates difference)
}

#[test]
fn test_preprocessing_edge_case_n1() {
    // Given: Single element
    let data = vec![42.0f32];
    let window_size = 1;

    // When: Apply windowed normalization
    let data_tensor = Tensor::from_slice(&data, (data.len(),), &Device::Cpu).unwrap();
    let normalized = windowed_normalize(&data_tensor, window_size as i64).unwrap();
    let normalized_vec: Vec<f32> = normalized.to_vec1().unwrap();

    // Then: Should handle N=1 gracefully (variance=0, z-score=0)
    assert_eq!(normalized_vec[0], 0.0);
}

#[test]
fn test_preprocessing_realistic_prices() {
    // Given: Realistic price data
    let prices = vec![
        5000.0f32, 5010.0, 5020.0, 5015.0, 5025.0, 5030.0, 5028.0, 5035.0,
    ];
    let window_size = 5;

    // When: Apply windowed normalization
    let data_tensor = Tensor::from_slice(&prices, (prices.len(),), &Device::Cpu).unwrap();
    let normalized = windowed_normalize(&data_tensor, window_size as i64).unwrap();
    let normalized_vec: Vec<f32> = normalized.to_vec1().unwrap();

    // Then: Should match manual unbiased calculation
    let expected = compute_expected_zscore_unbiased(&prices, window_size);

    for (i, (&actual, &expected)) in normalized_vec.iter().zip(expected.iter()).enumerate() {
        let diff = (actual - expected).abs();
        assert!(
            diff < 1e-4,
            "Index {}: Price={}, Z-score actual={}, expected={}, diff={}",
            i,
            prices[i],
            actual,
            expected,
            diff
        );
    }
}

#[test]
fn test_preprocessing_variance_difference() {
    // Given: Small window to maximize Bessel's correction impact
    let data = vec![1.0f32, 2.0]; // N=2 shows maximum 2x difference
    let window_size = 2;

    // When: Apply windowed normalization
    let data_tensor = Tensor::from_slice(&data, (data.len(),), &Device::Cpu).unwrap();
    let normalized = windowed_normalize(&data_tensor, window_size as i64).unwrap();
    let normalized_vec: Vec<f32> = normalized.to_vec1().unwrap();

    // Then: Verify matches unbiased calculation
    // For index 1 (window [1.0, 2.0]):
    // Mean = 1.5
    // Biased variance: [(1-1.5)^2 + (2-1.5)^2] / 2 = 0.5 / 2 = 0.25
    // Unbiased variance: 0.5 / 1 = 0.5
    // Biased std: sqrt(0.25) = 0.5
    // Unbiased std: sqrt(0.5) ≈ 0.707
    // Z-score (biased): (2.0 - 1.5) / 0.5 = 1.0
    // Z-score (unbiased): (2.0 - 1.5) / 0.707 ≈ 0.707

    let expected = compute_expected_zscore_unbiased(&data, window_size);
    let last_zscore = normalized_vec[1];
    let expected_zscore = expected[1];

    // Should match unbiased calculation (~0.707)
    assert!(
        (last_zscore - expected_zscore).abs() < 1e-5,
        "Should use unbiased estimator. actual={}, expected={}",
        last_zscore,
        expected_zscore
    );

    // Verify it's NOT the biased value (1.0)
    assert!(
        (last_zscore - 1.0).abs() > 0.2,
        "Should not use biased estimator (1.0), got {}",
        last_zscore
    );
}