foxhunt/crates/ml/tests/preprocessing_bessel_integration.rs

#![allow(
    clippy::assertions_on_constants,
    clippy::assertions_on_result_states,
    clippy::clone_on_copy,
    clippy::decimal_literal_representation,
    clippy::doc_markdown,
    clippy::empty_line_after_doc_comments,
    clippy::field_reassign_with_default,
    clippy::get_unwrap,
    clippy::identity_op,
    clippy::inconsistent_digit_grouping,
    clippy::indexing_slicing,
    clippy::integer_division,
    clippy::len_zero,
    clippy::let_underscore_must_use,
    clippy::manual_div_ceil,
    clippy::manual_let_else,
    clippy::manual_range_contains,
    clippy::modulo_arithmetic,
    clippy::needless_range_loop,
    clippy::non_ascii_literal,
    clippy::redundant_clone,
    clippy::shadow_reuse,
    clippy::shadow_same,
    clippy::shadow_unrelated,
    clippy::single_match_else,
    clippy::str_to_string,
    clippy::string_slice,
    clippy::tests_outside_test_module,
    clippy::too_many_lines,
    clippy::unnecessary_wraps,
    clippy::unseparated_literal_suffix,
    clippy::use_debug,
    clippy::useless_vec,
    clippy::wildcard_enum_match_arm,
    clippy::else_if_without_else,
    clippy::expect_used,
    clippy::missing_const_for_fn,
    clippy::similar_names,
    clippy::type_complexity,
    clippy::collapsible_else_if,
    clippy::doc_lazy_continuation,
    clippy::items_after_test_module,
    clippy::map_clone,
    clippy::multiple_unsafe_ops_per_block,
    clippy::unwrap_or_default,
    clippy::assign_op_pattern,
    clippy::needless_borrow,
    clippy::println_empty_string,
    clippy::unnecessary_cast,
    clippy::used_underscore_binding,
    clippy::create_dir,
    clippy::implicit_saturating_sub,
    clippy::exit,
    clippy::expect_fun_call,
    clippy::too_many_arguments,
    clippy::unnecessary_map_or,
    clippy::unwrap_used,
    dead_code,
    unused_imports,
    unused_variables,
    clippy::cloned_ref_to_slice_refs,
    clippy::neg_multiply,
    clippy::while_let_loop,
    clippy::bool_assert_comparison,
    clippy::excessive_precision,
    clippy::trivially_copy_pass_by_ref,
    clippy::op_ref,
    clippy::redundant_closure,
    clippy::unnecessary_lazy_evaluations,
    clippy::if_then_some_else_none,
    clippy::unnecessary_to_owned,
    clippy::single_component_path_imports,
)]
//! Integration Test: Preprocessing Module Uses Bessel's Correction
//!
//! Validates that the windowed_normalize function in the preprocessing module
//! correctly applies Bessel's correction when computing variance.

// candle eliminated — test uses native APIs
use ml::preprocessing::windowed_normalize;

/// Manually compute expected z-scores with Bessel's correction
fn compute_expected_zscore_unbiased(data: &[f32], window_size: usize) -> Vec<f32> {
    let mut result = Vec::with_capacity(data.len());

    for i in 0..data.len() {
        let start = if i + 1 >= window_size {
            i + 1 - window_size
        } else {
            0
        };

        let window = &data[start..=i];

        // Compute mean
        let mean: f32 = window.iter().sum::<f32>() / window.len() as f32;

        // Compute variance with Bessel's correction (N-1)
        let variance: f32 = if window.len() > 1 {
            window.iter().map(|&x| (x - mean).powi(2)).sum::<f32>() / (window.len() - 1) as f32
        } else {
            0.0
        };
        let std = variance.sqrt();

        // Compute z-score
        let eps = 1e-8;
        let z_score = if std > eps {
            (data[i] - mean) / std
        } else {
            0.0
        };

        result.push(z_score);
    }

    result
}

#[test]
fn test_preprocessing_uses_bessel_correction() {
    // Given: Simple test data
    let data = vec![1.0f32, 2.0, 3.0, 4.0, 5.0];
    let window_size = 3;

    // When: Apply windowed normalization from preprocessing module
    let data_tensor = Tensor::from_slice(&data, (data.len(),), &Device::new_cuda(0).expect("CUDA required")).unwrap();
    let normalized = windowed_normalize(&data_tensor, window_size as i64).unwrap();

    // Then: Should match manual calculation with Bessel's correction
    let expected = compute_expected_zscore_unbiased(&data, window_size);

    for (i, &exp) in expected.iter().enumerate() {
        let actual = normalized.narrow(0, i, 1).unwrap().squeeze(0).unwrap().to_scalar::<f32>().unwrap();
        let diff = (actual - exp).abs();
        assert!(
            diff < 1e-5,
            "Index {}: actual={}, expected={}, diff={}",
            i,
            actual,
            exp,
            diff
        );
    }
}

#[test]
fn test_preprocessing_bessel_vs_biased() {
    // Given: Data where Bessel's correction makes a significant difference
    let data = vec![100.0f32, 110.0, 105.0]; // Small sample (N=3)
    let window_size = 3;

    // When: Apply windowed normalization (should use Bessel's correction)
    let data_tensor = Tensor::from_slice(&data, (data.len(),), &Device::new_cuda(0).expect("CUDA required")).unwrap();
    let normalized = windowed_normalize(&data_tensor, window_size as i64).unwrap();

    // Then: Verify it matches unbiased calculation
    let expected_unbiased = compute_expected_zscore_unbiased(&data, window_size);

    // Last value should be properly normalized with unbiased estimator
    let last_actual = normalized.narrow(0, 2, 1).unwrap().squeeze(0).unwrap().to_scalar::<f32>().unwrap();
    let last_expected = expected_unbiased[2];

    assert!(
        (last_actual - last_expected).abs() < 1e-5,
        "Preprocessing should use unbiased estimator. actual={}, expected={}",
        last_actual,
        last_expected
    );

    // Verify that it's different from biased calculation (for documentation)
    // Mean of [100, 110, 105] = 105
    // Variance (biased): [(100-105)^2 + (110-105)^2 + (105-105)^2] / 3 = 50/3 ≈ 16.67
    // Variance (unbiased): 50 / 2 = 25.0
    // Std (biased): sqrt(16.67) ≈ 4.08
    // Std (unbiased): sqrt(25.0) = 5.0
    // Z-score for 105: (105-105)/std = 0.0 (same for both, but demonstrates difference)
}

#[test]
fn test_preprocessing_edge_case_n1() {
    // Given: Single element
    let data = vec![42.0f32];
    let window_size = 1;

    // When: Apply windowed normalization
    let data_tensor = Tensor::from_slice(&data, (data.len(),), &Device::new_cuda(0).expect("CUDA required")).unwrap();
    let normalized = windowed_normalize(&data_tensor, window_size as i64).unwrap();

    // Then: Should handle N=1 gracefully (variance=0, z-score=0)
    let val0 = normalized.narrow(0, 0, 1).unwrap().squeeze(0).unwrap().to_scalar::<f32>().unwrap();
    assert_eq!(val0, 0.0);
}

#[test]
fn test_preprocessing_realistic_prices() {
    // Given: Realistic price data
    let prices = vec![
        5000.0f32, 5010.0, 5020.0, 5015.0, 5025.0, 5030.0, 5028.0, 5035.0,
    ];
    let window_size = 5;

    // When: Apply windowed normalization
    let data_tensor = Tensor::from_slice(&prices, (prices.len(),), &Device::new_cuda(0).expect("CUDA required")).unwrap();
    let normalized = windowed_normalize(&data_tensor, window_size as i64).unwrap();

    // Then: Should match manual unbiased calculation
    let expected = compute_expected_zscore_unbiased(&prices, window_size);

    for (i, &exp) in expected.iter().enumerate() {
        let actual = normalized.narrow(0, i, 1).unwrap().squeeze(0).unwrap().to_scalar::<f32>().unwrap();
        let diff = (actual - exp).abs();
        assert!(
            diff < 1e-4,
            "Index {}: Price={}, Z-score actual={}, expected={}, diff={}",
            i,
            prices[i],
            actual,
            exp,
            diff
        );
    }
}

#[test]
fn test_preprocessing_variance_difference() {
    // Given: Small window to maximize Bessel's correction impact
    let data = vec![1.0f32, 2.0]; // N=2 shows maximum 2x difference
    let window_size = 2;

    // When: Apply windowed normalization
    let data_tensor = Tensor::from_slice(&data, (data.len(),), &Device::new_cuda(0).expect("CUDA required")).unwrap();
    let normalized = windowed_normalize(&data_tensor, window_size as i64).unwrap();

    // Then: Verify matches unbiased calculation
    // For index 1 (window [1.0, 2.0]):
    // Mean = 1.5
    // Biased variance: [(1-1.5)^2 + (2-1.5)^2] / 2 = 0.5 / 2 = 0.25
    // Unbiased variance: 0.5 / 1 = 0.5
    // Biased std: sqrt(0.25) = 0.5
    // Unbiased std: sqrt(0.5) ≈ 0.707
    // Z-score (biased): (2.0 - 1.5) / 0.5 = 1.0
    // Z-score (unbiased): (2.0 - 1.5) / 0.707 ≈ 0.707

    let expected = compute_expected_zscore_unbiased(&data, window_size);
    let last_zscore = normalized.narrow(0, 1, 1).unwrap().squeeze(0).unwrap().to_scalar::<f32>().unwrap();
    let expected_zscore = expected[1];

    // Should match unbiased calculation (~0.707)
    assert!(
        (last_zscore - expected_zscore).abs() < 1e-5,
        "Should use unbiased estimator. actual={}, expected={}",
        last_zscore,
        expected_zscore
    );

    // Verify it's NOT the biased value (1.0)
    assert!(
        (last_zscore - 1.0).abs() > 0.2,
        "Should not use biased estimator (1.0), got {}",
        last_zscore
    );
}