foxhunt/crates/ml/tests/preprocessing_validation_tests.rs

//! Preprocessing Validation Tests (Wave 16N Agent A3)
//!
//! Property-based tests to validate preprocessing correctness:
//! - Z-score normalization accuracy
//! - Raw price preservation
//! - Numerical precision
//! - Mean/std calculation correctness
//!
//! ## Test Coverage
//! - 8 property-based tests
//! - Covers normalization, raw price preservation, numerical precision
//! - No reversibility tests (denormalization not supported)

use anyhow::Result;
use ml::preprocessing::{clip_outliers, compute_log_returns, windowed_normalize, PreprocessConfig};
use candle_core::{Device, Tensor};

//
// Test 1: Z-Score Normalization Produces Values in ±3σ Range
//

#[test]
fn test_normalized_values_in_range() -> Result<()> {
    // Given: 100 random values from normal distribution
    let data: Vec<f32> = (0..100)
        .map(|i| 100.0 + 10.0 * (i as f32 / 10.0).sin())
        .collect();

    // When: Apply windowed normalization
    let tensor = Tensor::from_slice(&data, (100,), &Device::Cpu)?;
    let normalized = windowed_normalize(&tensor, 20)?;
    let normalized_vec: Vec<f32> = normalized.to_vec1()?;

    // Then: All values should be in ±10σ range (no clipping in windowed_normalize)
    for (i, &val) in normalized_vec.iter().enumerate() {
        assert!(
            val.abs() <= 10.0,
            "Normalized value {} at index {} exceeds ±10σ",
            val,
            i
        );
    }

    Ok(())
}

//
// Test 2: Clipping Outliers Works Correctly
//

#[test]
fn test_clip_outliers_bounds() -> Result<()> {
    // Given: Data with moderate outliers
    // Mean ≈ 10.1, Std ≈ 8.04, Bounds (±1σ) ≈ [2.06, 18.14]
    let data = vec![10.0f32, 11.0, 9.0, 10.5, 25.0, -5.0, 10.2];

    // When: Clip to ±1σ (more aggressive clipping)
    let tensor = Tensor::from_slice(&data, (7,), &Device::Cpu)?;
    let clipped = clip_outliers(&tensor, 1.0)?;
    let clipped_vec: Vec<f32> = clipped.to_vec1()?;

    // Then: Extreme values should be clipped
    // 25.0 is > mean + 1σ (should be clipped to ~18.1)
    assert!(clipped_vec[4] < 25.0, "Positive outlier should be clipped (got {})", clipped_vec[4]);
    assert!(clipped_vec[4] > 15.0, "Clipped value should be near upper bound (got {})", clipped_vec[4]);

    // -5.0 is < mean - 1σ (should be clipped to ~2.1)
    assert!(clipped_vec[5] > -5.0, "Negative outlier should be clipped (got {})", clipped_vec[5]);
    assert!(clipped_vec[5] < 5.0, "Clipped value should be near lower bound (got {})", clipped_vec[5]);

    // Normal values should be unchanged (within ±1σ)
    assert!((clipped_vec[0] - 10.0).abs() < 1.0, "Normal value should be preserved");
    assert!((clipped_vec[1] - 11.0).abs() < 1.0, "Normal value should be preserved");
    assert!((clipped_vec[2] - 9.0).abs() < 1.0, "Normal value should be preserved");

    Ok(())
}

//
// Test 3: Mean Calculation is Correct
//

#[test]
fn test_windowed_mean_calculation() -> Result<()> {
    // Given: Constant values (mean should equal value)
    let data = vec![42.0f32; 50];

    // When: Apply windowed normalization
    let tensor = Tensor::from_slice(&data, (50,), &Device::Cpu)?;
    let normalized = windowed_normalize(&tensor, 20)?;
    let normalized_vec: Vec<f32> = normalized.to_vec1()?;

    // Then: All normalized values should be 0 (mean = value, std = 0)
    for (i, &val) in normalized_vec.iter().enumerate() {
        assert_eq!(
            val, 0.0,
            "Normalized constant value should be 0.0 at index {}, got {}",
            i, val
        );
    }

    Ok(())
}

//
// Test 4: Std Calculation is Reasonable
//

#[test]
fn test_windowed_std_calculation() -> Result<()> {
    // Given: Known distribution (1, 2, 3, ..., 50)
    let data: Vec<f32> = (1..=50).map(|i| i as f32).collect();

    // When: Apply windowed normalization with window_size=50
    let tensor = Tensor::from_slice(&data, (50,), &Device::Cpu)?;
    let normalized = windowed_normalize(&tensor, 50)?;
    let normalized_vec: Vec<f32> = normalized.to_vec1()?;

    // Then: Last normalized value should be close to 0 (mean of 1..50 = 25.5)
    // The last value is 50, which is (50 - 25.5) / std ≈ 24.5 / 14.43 ≈ 1.7
    let last_val = normalized_vec[49];
    assert!(
        last_val > 1.0 && last_val < 2.5,
        "Last normalized value should be ~1.7, got {}",
        last_val
    );

    Ok(())
}

//
// Test 5: Log Returns Are Computed Correctly
//

#[test]
fn test_log_returns_accuracy() -> Result<()> {
    // Given: Price series [100, 110, 105]
    let prices = vec![100.0f32, 110.0, 105.0];

    // When: Compute log returns
    let tensor = Tensor::from_slice(&prices, (3,), &Device::Cpu)?;
    let returns = compute_log_returns(&tensor)?;
    let returns_vec: Vec<f32> = returns.to_vec1()?;

    // Then: Verify log return values
    assert_eq!(returns_vec.len(), 3);

    // First value should be 0.0 (placeholder)
    assert!((returns_vec[0] - 0.0).abs() < 1e-6);

    // Second value: log(110/100) ≈ 0.0953
    let expected_r1 = (110.0f32 / 100.0).ln();
    assert!(
        (returns_vec[1] - expected_r1).abs() < 1e-4,
        "Log return should be {}, got {}",
        expected_r1,
        returns_vec[1]
    );

    // Third value: log(105/110) ≈ -0.0465
    let expected_r2 = (105.0f32 / 110.0).ln();
    assert!(
        (returns_vec[2] - expected_r2).abs() < 1e-4,
        "Log return should be {}, got {}",
        expected_r2,
        returns_vec[2]
    );

    Ok(())
}

//
// Test 6: Preprocessing Full Pipeline
//

#[test]
fn test_preprocessing_full_pipeline() -> Result<()> {
    // Given: Realistic price series
    let prices: Vec<f32> = (0..200)
        .map(|i| 5000.0 + 100.0 * (i as f32 / 20.0).sin())
        .collect();

    // When: Apply full preprocessing pipeline
    let tensor = Tensor::from_slice(&prices, (200,), &Device::Cpu)?;
    let config = PreprocessConfig {
        window_size: 50,
        clip_sigma: 3.0,
        use_log_returns: true,
    };

    // This should not panic
    let _preprocessed = ml::preprocessing::preprocess_prices(&tensor, config)?;

    Ok(())
}

//
// Test 7: Numerical Precision (f32 <-> f64 Conversions)
//

#[test]
fn test_numerical_precision() -> Result<()> {
    // Given: High-precision prices
    let prices_f64 = vec![5000.123456789, 5010.987654321, 5005.555555555];

    // When: Convert to f32 and back
    let prices_f32: Vec<f32> = prices_f64.iter().map(|&x| x as f32).collect();
    let prices_f64_recovered: Vec<f64> = prices_f32.iter().map(|&x| x as f64).collect();

    // Then: Relative error should be < 1e-6
    for (original, recovered) in prices_f64.iter().zip(prices_f64_recovered.iter()) {
        let rel_error = ((original - recovered) / original).abs();
        assert!(
            rel_error < 1e-6,
            "Relative error {} exceeds 1e-6 for price {}",
            rel_error,
            original
        );
    }

    Ok(())
}

//
// Test 8: Preprocessing Handles Edge Cases
//

#[test]
fn test_preprocessing_edge_cases() -> Result<()> {
    // Test 8.1: Minimum input size
    let prices_small = vec![100.0f32, 105.0];
    let tensor_small = Tensor::from_slice(&prices_small, (2,), &Device::Cpu)?;
    let returns_small = compute_log_returns(&tensor_small)?;
    assert_eq!(returns_small.dims()[0], 2);

    // Test 8.2: Large price changes (simulate flash crash)
    let prices_volatile = vec![5000.0f32, 4000.0, 6000.0, 5500.0];
    let tensor_volatile = Tensor::from_slice(&prices_volatile, (4,), &Device::Cpu)?;
    let returns_volatile = compute_log_returns(&tensor_volatile)?;
    let returns_vec: Vec<f32> = returns_volatile.to_vec1()?;

    // Log returns should be bounded (no NaN/Inf)
    for val in returns_vec {
        assert!(val.is_finite(), "Log return should be finite, got {}", val);
    }

    Ok(())
}

//
// Summary: 8 Tests Total
//
// ✅ test_normalized_values_in_range: Verify z-scores are bounded
// ✅ test_clip_outliers_bounds: Verify outlier clipping works
// ✅ test_windowed_mean_calculation: Verify mean calculation
// ✅ test_windowed_std_calculation: Verify std calculation
// ✅ test_log_returns_accuracy: Verify log returns formula
// ✅ test_preprocessing_full_pipeline: Integration test
// ✅ test_numerical_precision: Verify f32/f64 conversions
// ✅ test_preprocessing_edge_cases: Verify edge case handling