foxhunt/crates/ml/tests/preprocessing_validation_tests.rs

#![allow(
    clippy::assertions_on_constants,
    clippy::assertions_on_result_states,
    clippy::clone_on_copy,
    clippy::decimal_literal_representation,
    clippy::doc_markdown,
    clippy::empty_line_after_doc_comments,
    clippy::field_reassign_with_default,
    clippy::get_unwrap,
    clippy::identity_op,
    clippy::inconsistent_digit_grouping,
    clippy::indexing_slicing,
    clippy::integer_division,
    clippy::len_zero,
    clippy::let_underscore_must_use,
    clippy::manual_div_ceil,
    clippy::manual_let_else,
    clippy::manual_range_contains,
    clippy::modulo_arithmetic,
    clippy::needless_range_loop,
    clippy::non_ascii_literal,
    clippy::redundant_clone,
    clippy::shadow_reuse,
    clippy::shadow_same,
    clippy::shadow_unrelated,
    clippy::single_match_else,
    clippy::str_to_string,
    clippy::string_slice,
    clippy::tests_outside_test_module,
    clippy::too_many_lines,
    clippy::unnecessary_wraps,
    clippy::unseparated_literal_suffix,
    clippy::use_debug,
    clippy::useless_vec,
    clippy::wildcard_enum_match_arm,
    clippy::else_if_without_else,
    clippy::expect_used,
    clippy::missing_const_for_fn,
    clippy::similar_names,
    clippy::type_complexity,
    clippy::collapsible_else_if,
    clippy::doc_lazy_continuation,
    clippy::items_after_test_module,
    clippy::map_clone,
    clippy::multiple_unsafe_ops_per_block,
    clippy::unwrap_or_default,
    clippy::assign_op_pattern,
    clippy::needless_borrow,
    clippy::println_empty_string,
    clippy::unnecessary_cast,
    clippy::used_underscore_binding,
    clippy::create_dir,
    clippy::implicit_saturating_sub,
    clippy::exit,
    clippy::expect_fun_call,
    clippy::too_many_arguments,
    clippy::unnecessary_map_or,
    clippy::unwrap_used,
    dead_code,
    unused_imports,
    unused_variables,
    clippy::cloned_ref_to_slice_refs,
    clippy::neg_multiply,
    clippy::while_let_loop,
    clippy::bool_assert_comparison,
    clippy::excessive_precision,
    clippy::trivially_copy_pass_by_ref,
    clippy::op_ref,
    clippy::redundant_closure,
    clippy::unnecessary_lazy_evaluations,
    clippy::if_then_some_else_none,
    clippy::unnecessary_to_owned,
    clippy::single_component_path_imports,
)]
//! Preprocessing Validation Tests (Wave 16N Agent A3)
//!
//! Property-based tests to validate preprocessing correctness:
//! - Z-score normalization accuracy
//! - Raw price preservation
//! - Numerical precision
//! - Mean/std calculation correctness
//!
//! ## Test Coverage
//! - 8 property-based tests
//! - Covers normalization, raw price preservation, numerical precision
//! - No reversibility tests (denormalization not supported)

use anyhow::Result;
use ml::preprocessing::{clip_outliers, compute_log_returns, windowed_normalize, PreprocessConfig};
use candle_core::{Device, Tensor};

//
// Test 1: Z-Score Normalization Produces Values in ±3σ Range
//

#[test]
fn test_normalized_values_in_range() -> Result<()> {
    // Given: 100 random values from normal distribution
    let data: Vec<f32> = (0..100)
        .map(|i| 100.0 + 10.0 * (i as f32 / 10.0).sin())
        .collect();

    // When: Apply windowed normalization
    let tensor = Tensor::from_slice(&data, (100,), &Device::new_cuda(0).expect("CUDA required"))?;
    let normalized = windowed_normalize(&tensor, 20)?;

    // Then: All values should be in ±10σ range (no clipping in windowed_normalize)
    let max_abs = normalized.abs()?.max(0)?.to_scalar::<f32>()?;
    assert!(
        max_abs <= 10.0,
        "Max absolute normalized value {} exceeds ±10σ",
        max_abs
    );

    Ok(())
}

//
// Test 2: Clipping Outliers Works Correctly
//

#[test]
fn test_clip_outliers_bounds() -> Result<()> {
    // Given: Data with moderate outliers
    // Mean ≈ 10.1, Std ≈ 8.04, Bounds (±1σ) ≈ [2.06, 18.14]
    let data = vec![10.0f32, 11.0, 9.0, 10.5, 25.0, -5.0, 10.2];

    // When: Clip to ±1σ (more aggressive clipping)
    let tensor = Tensor::from_slice(&data, (7,), &Device::new_cuda(0).expect("CUDA required"))?;
    let clipped = clip_outliers(&tensor, 1.0)?;

    // Then: Extreme values should be clipped
    // 25.0 is > mean + 1σ (should be clipped to ~18.1)
    let val4 = clipped.narrow(0, 4, 1)?.squeeze(0)?.to_scalar::<f32>()?;
    assert!(val4 < 25.0, "Positive outlier should be clipped (got {})", val4);
    assert!(val4 > 15.0, "Clipped value should be near upper bound (got {})", val4);

    // -5.0 is < mean - 1σ (should be clipped to ~2.1)
    let val5 = clipped.narrow(0, 5, 1)?.squeeze(0)?.to_scalar::<f32>()?;
    assert!(val5 > -5.0, "Negative outlier should be clipped (got {})", val5);
    assert!(val5 < 5.0, "Clipped value should be near lower bound (got {})", val5);

    // Normal values should be unchanged (within ±1σ)
    let val0 = clipped.narrow(0, 0, 1)?.squeeze(0)?.to_scalar::<f32>()?;
    let val1 = clipped.narrow(0, 1, 1)?.squeeze(0)?.to_scalar::<f32>()?;
    let val2 = clipped.narrow(0, 2, 1)?.squeeze(0)?.to_scalar::<f32>()?;
    assert!((val0 - 10.0).abs() < 1.0, "Normal value should be preserved");
    assert!((val1 - 11.0).abs() < 1.0, "Normal value should be preserved");
    assert!((val2 - 9.0).abs() < 1.0, "Normal value should be preserved");

    Ok(())
}

//
// Test 3: Mean Calculation is Correct
//

#[test]
fn test_windowed_mean_calculation() -> Result<()> {
    // Given: Constant values (mean should equal value)
    let data = vec![42.0f32; 50];

    // When: Apply windowed normalization
    let tensor = Tensor::from_slice(&data, (50,), &Device::new_cuda(0).expect("CUDA required"))?;
    let normalized = windowed_normalize(&tensor, 20)?;

    // Then: All normalized values should be 0 (mean = value, std = 0)
    // Check via sum_all and max_abs — if all are exactly 0.0, both should be 0.0
    let sum_all = normalized.sum_all()?.to_scalar::<f32>()?;
    assert_eq!(sum_all, 0.0, "Sum of all normalized constant values should be 0.0, got {}", sum_all);
    let max_abs = normalized.abs()?.max(0)?.to_scalar::<f32>()?;
    assert_eq!(max_abs, 0.0, "Max absolute normalized constant value should be 0.0, got {}", max_abs);

    Ok(())
}

//
// Test 4: Std Calculation is Reasonable
//

#[test]
fn test_windowed_std_calculation() -> Result<()> {
    // Given: Known distribution (1, 2, 3, ..., 50)
    let data: Vec<f32> = (1..=50).map(|i| i as f32).collect();

    // When: Apply windowed normalization with window_size=50
    let tensor = Tensor::from_slice(&data, (50,), &Device::new_cuda(0).expect("CUDA required"))?;
    let normalized = windowed_normalize(&tensor, 50)?;

    // Then: Last normalized value should be close to 0 (mean of 1..50 = 25.5)
    // The last value is 50, which is (50 - 25.5) / std ≈ 24.5 / 14.43 ≈ 1.7
    let last_val = normalized.narrow(0, 49, 1)?.squeeze(0)?.to_scalar::<f32>()?;
    assert!(
        last_val > 1.0 && last_val < 2.5,
        "Last normalized value should be ~1.7, got {}",
        last_val
    );

    Ok(())
}

//
// Test 5: Log Returns Are Computed Correctly
//

#[test]
fn test_log_returns_accuracy() -> Result<()> {
    // Given: Price series [100, 110, 105]
    let prices = vec![100.0f32, 110.0, 105.0];

    // When: Compute log returns
    let tensor = Tensor::from_slice(&prices, (3,), &Device::new_cuda(0).expect("CUDA required"))?;
    let returns = compute_log_returns(&tensor)?;

    // Then: Verify log return values
    assert_eq!(returns.dims()[0], 3);

    // First value should be 0.0 (placeholder)
    let val0 = returns.narrow(0, 0, 1)?.squeeze(0)?.to_scalar::<f32>()?;
    assert!((val0 - 0.0).abs() < 1e-6);

    // Second value: log(110/100) ≈ 0.0953
    let expected_r1 = (110.0f32 / 100.0).ln();
    let val1 = returns.narrow(0, 1, 1)?.squeeze(0)?.to_scalar::<f32>()?;
    assert!(
        (val1 - expected_r1).abs() < 1e-4,
        "Log return should be {}, got {}",
        expected_r1,
        val1
    );

    // Third value: log(105/110) ≈ -0.0465
    let expected_r2 = (105.0f32 / 110.0).ln();
    let val2 = returns.narrow(0, 2, 1)?.squeeze(0)?.to_scalar::<f32>()?;
    assert!(
        (val2 - expected_r2).abs() < 1e-4,
        "Log return should be {}, got {}",
        expected_r2,
        val2
    );

    Ok(())
}

//
// Test 6: Preprocessing Full Pipeline
//

#[test]
fn test_preprocessing_full_pipeline() -> Result<()> {
    // Given: Realistic price series
    let prices: Vec<f32> = (0..200)
        .map(|i| 5000.0 + 100.0 * (i as f32 / 20.0).sin())
        .collect();

    // When: Apply full preprocessing pipeline
    let tensor = Tensor::from_slice(&prices, (200,), &Device::new_cuda(0).expect("CUDA required"))?;
    let config = PreprocessConfig {
        window_size: 50,
        clip_sigma: 3.0,
        use_log_returns: true,
    };

    // This should not panic
    let _preprocessed = ml::preprocessing::preprocess_prices(&tensor, config)?;

    Ok(())
}

//
// Test 7: Numerical Precision (f32 <-> f64 Conversions)
//

#[test]
fn test_numerical_precision() -> Result<()> {
    // Given: High-precision prices
    let prices_f64 = vec![5000.123456789, 5010.987654321, 5005.555555555];

    // When: Convert to f32 and back
    let prices_f32: Vec<f32> = prices_f64.iter().map(|&x| x as f32).collect();
    let prices_f64_recovered: Vec<f64> = prices_f32.iter().map(|&x| x as f64).collect();

    // Then: Relative error should be < 1e-6
    for (original, recovered) in prices_f64.iter().zip(prices_f64_recovered.iter()) {
        let rel_error = ((original - recovered) / original).abs();
        assert!(
            rel_error < 1e-6,
            "Relative error {} exceeds 1e-6 for price {}",
            rel_error,
            original
        );
    }

    Ok(())
}

//
// Test 8: Preprocessing Handles Edge Cases
//

#[test]
fn test_preprocessing_edge_cases() -> Result<()> {
    // Test 8.1: Minimum input size
    let prices_small = vec![100.0f32, 105.0];
    let tensor_small = Tensor::from_slice(&prices_small, (2,), &Device::new_cuda(0).expect("CUDA required"))?;
    let returns_small = compute_log_returns(&tensor_small)?;
    assert_eq!(returns_small.dims()[0], 2);

    // Test 8.2: Large price changes (simulate flash crash)
    let prices_volatile = vec![5000.0f32, 4000.0, 6000.0, 5500.0];
    let tensor_volatile = Tensor::from_slice(&prices_volatile, (4,), &Device::new_cuda(0).expect("CUDA required"))?;
    let returns_volatile = compute_log_returns(&tensor_volatile)?;

    // Log returns should be bounded (no NaN/Inf) — sum would be NaN/Inf if any element is
    let sum_all = returns_volatile.sum_all()?.to_scalar::<f32>()?;
    assert!(sum_all.is_finite(), "Log returns should all be finite, sum_all = {}", sum_all);

    Ok(())
}

//
// Summary: 8 Tests Total
//
// ✅ test_normalized_values_in_range: Verify z-scores are bounded
// ✅ test_clip_outliers_bounds: Verify outlier clipping works
// ✅ test_windowed_mean_calculation: Verify mean calculation
// ✅ test_windowed_std_calculation: Verify std calculation
// ✅ test_log_returns_accuracy: Verify log returns formula
// ✅ test_preprocessing_full_pipeline: Integration test
// ✅ test_numerical_precision: Verify f32/f64 conversions
// ✅ test_preprocessing_edge_cases: Verify edge case handling