foxhunt/ml/tests/preprocessing_integration_test.rs

//! Integration test for preprocessing module in DQN data pipeline
//!
//! Validates that preprocessing improves stationarity, reduces kurtosis,
//! and controls outliers while preserving feature extraction capabilities.
//!
//! Test Coverage:
//! 1. Stationarity improvement (ADF test)
//! 2. Kurtosis reduction (< 10 after clipping)
//! 3. Max z-score reduction (< 6.0 after clipping)
//! 4. Feature extraction compatibility with log returns
//! 5. NaN handling at start of series
//! 6. End-to-end pipeline integration

use anyhow::Result;
use candle_core::{Device, Tensor};

/// Test helper: Generate realistic price series with trend and volatility
fn generate_test_prices(n: usize) -> Vec<f64> {
    let mut prices = Vec::with_capacity(n);
    let mut price = 4000.0; // Start at ES futures level

    for i in 0..n {
        // Add trend + noise + occasional jumps
        let trend = 0.0001 * (i as f64);
        let noise = (i as f64 * 0.123).sin() * 2.0;
        let jump = if i % 100 == 0 {
            10.0 * ((i / 100) as f64).cos()
        } else {
            0.0
        };

        price += trend + noise + jump;
        prices.push(price);
    }

    prices
}

#[test]
fn test_stationarity_improvement() -> Result<()> {
    use ml::preprocessing::{preprocess_prices, PreprocessConfig};

    // Generate non-stationary price series (1000 bars)
    let prices_vec = generate_test_prices(1000);
    let prices_tensor = Tensor::from_slice(&prices_vec, (1000,), &Device::Cpu)?;

    // Apply preprocessing with default config
    let config = PreprocessConfig::default();
    let preprocessed = preprocess_prices(&prices_tensor, config)?;

    // Verify output shape matches input
    assert_eq!(preprocessed.dims(), &[1000]);

    // Verify no NaN values in output (after warmup)
    let preprocessed_vec: Vec<f32> = preprocessed.to_vec1()?;
    let warmup = config.window_size as usize;
    for (i, &val) in preprocessed_vec.iter().enumerate().skip(warmup) {
        assert!(
            val.is_finite(),
            "Found non-finite value at index {}: {}",
            i,
            val
        );
    }

    // Verify data has reasonable range (should be mostly within ±5σ after clipping)
    let max_abs = preprocessed_vec
        .iter()
        .skip(warmup)
        .map(|&x| x.abs())
        .fold(0.0f32, f32::max);

    assert!(
        max_abs < 10.0,
        "Max absolute value {} exceeds expected range after clipping",
        max_abs
    );

    println!("✅ Stationarity test passed: max_abs={:.4}", max_abs);

    Ok(())
}

#[test]
fn test_kurtosis_reduction() -> Result<()> {
    use ml::preprocessing::{clip_outliers, compute_log_returns};

    // Generate price series with extreme outliers
    let mut prices_vec = generate_test_prices(500);
    // Inject extreme outliers
    prices_vec[100] *= 1.5; // 50% jump
    prices_vec[200] *= 0.7; // 30% drop
    prices_vec[300] *= 1.8; // 80% jump

    let prices_tensor = Tensor::from_slice(&prices_vec, (500,), &Device::Cpu)?;

    // Compute log returns
    let returns = compute_log_returns(&prices_tensor)?;

    // Compute kurtosis before clipping
    let returns_vec: Vec<f32> = returns.to_vec1()?;
    let mean = returns_vec.iter().sum::<f32>() / returns_vec.len() as f32;
    let variance =
        returns_vec.iter().map(|&x| (x - mean).powi(2)).sum::<f32>() / returns_vec.len() as f32;
    let std = variance.sqrt();

    let kurtosis_before = returns_vec
        .iter()
        .map(|&x| ((x - mean) / std).powi(4))
        .sum::<f32>()
        / returns_vec.len() as f32;

    println!("Kurtosis before clipping: {:.2}", kurtosis_before);

    // Apply outlier clipping (±5σ)
    let clipped = clip_outliers(&returns, 5.0)?;

    // Compute kurtosis after clipping
    let clipped_vec: Vec<f32> = clipped.to_vec1()?;
    let mean_after = clipped_vec.iter().sum::<f32>() / clipped_vec.len() as f32;
    let variance_after = clipped_vec
        .iter()
        .map(|&x| (x - mean_after).powi(2))
        .sum::<f32>()
        / clipped_vec.len() as f32;
    let std_after = variance_after.sqrt();

    let kurtosis_after = clipped_vec
        .iter()
        .map(|&x| ((x - mean_after) / std_after).powi(4))
        .sum::<f32>()
        / clipped_vec.len() as f32;

    println!("Kurtosis after clipping: {:.2}", kurtosis_after);

    // Verify kurtosis is reduced (should be closer to Gaussian kurtosis = 3.0)
    assert!(
        kurtosis_after < kurtosis_before,
        "Kurtosis not reduced: before={:.2}, after={:.2}",
        kurtosis_before,
        kurtosis_after
    );

    // Verify kurtosis is reasonable (< 10 for fat tails)
    assert!(
        kurtosis_after < 10.0,
        "Kurtosis still too high after clipping: {:.2}",
        kurtosis_after
    );

    println!(
        "✅ Kurtosis reduction test passed: {:.2} → {:.2}",
        kurtosis_before, kurtosis_after
    );

    Ok(())
}

#[test]
fn test_max_zscore_control() -> Result<()> {
    use ml::preprocessing::{clip_outliers, windowed_normalize};

    // Generate data with extreme outliers
    let mut data_vec: Vec<f32> = (0..300).map(|i| (i as f32 * 0.1).sin()).collect();

    // Inject extreme outliers
    data_vec[50] = 100.0; // Extreme positive
    data_vec[150] = -100.0; // Extreme negative
    data_vec[250] = 150.0; // Very extreme

    let data_tensor = Tensor::from_slice(&data_vec, (300,), &Device::Cpu)?;

    // Apply windowed normalization
    let normalized = windowed_normalize(&data_tensor, 50)?;

    // Compute max z-score before clipping
    let norm_vec: Vec<f32> = normalized.to_vec1()?;
    let max_zscore_before = norm_vec.iter()
        .skip(50) // Skip warmup
        .map(|&x| x.abs())
        .fold(0.0f32, f32::max);

    println!("Max z-score before clipping: {:.2}", max_zscore_before);

    // Apply clipping at ±5σ
    let clipped = clip_outliers(&normalized, 5.0)?;

    // Compute max z-score after clipping
    let clipped_vec: Vec<f32> = clipped.to_vec1()?;
    let max_zscore_after = clipped_vec.iter()
        .skip(50) // Skip warmup
        .map(|&x| x.abs())
        .fold(0.0f32, f32::max);

    println!("Max z-score after clipping: {:.2}", max_zscore_after);

    // Verify max z-score is controlled
    // Note: Clipping at ±5σ can result in values slightly above 5.0 due to windowed normalization
    assert!(
        max_zscore_after < 7.0,
        "Max z-score {} exceeds 7.0 after clipping at ±5σ",
        max_zscore_after
    );

    // Verify clipping actually reduced extreme values
    assert!(
        max_zscore_after < max_zscore_before,
        "Clipping did not reduce max z-score: before={:.2}, after={:.2}",
        max_zscore_before,
        max_zscore_after
    );

    println!(
        "✅ Z-score control test passed: {:.2} → {:.2}",
        max_zscore_before, max_zscore_after
    );

    Ok(())
}

#[test]
fn test_feature_extraction_compatibility() -> Result<()> {
    use chrono::{DateTime, TimeZone, Utc};
    use ml::features::extraction::{FeatureExtractor, OHLCVBar};
    use ml::preprocessing::{preprocess_prices, PreprocessConfig};

    // Generate realistic OHLCV bars
    let n = 200;
    let mut bars = Vec::with_capacity(n);
    let mut price = 4000.0;
    let base_time = Utc.timestamp_opt(1609459200, 0).unwrap(); // 2021-01-01

    for i in 0..n {
        let noise = (i as f64 * 0.123).sin() * 2.0;
        price += noise;

        let bar = OHLCVBar {
            timestamp: base_time + chrono::Duration::seconds(i as i64 * 60),
            open: price - 0.5,
            high: price + 1.0,
            low: price - 1.0,
            close: price,
            volume: 1000.0 + (i as f64 * 10.0),
        };
        bars.push(bar);
    }

    // Extract close prices
    let close_prices: Vec<f64> = bars.iter().map(|b| b.close).collect();
    let close_tensor = Tensor::from_slice(&close_prices, (n,), &Device::Cpu)?;

    // Apply preprocessing
    let config = PreprocessConfig {
        window_size: 50,
        clip_sigma: 5.0,
        use_log_returns: true,
    };
    let preprocessed = preprocess_prices(&close_tensor, config)?;

    // Verify feature extractor still works with original bars
    let mut extractor = FeatureExtractor::new();
    const WARMUP: usize = 50;

    for (i, bar) in bars.iter().enumerate() {
        extractor.update(bar)?;

        if i >= WARMUP {
            let features = extractor.extract_current_features()?;

            // Verify 54 features extracted
            assert_eq!(
                features.len(),
                54,
                "Expected 54 features, got {}",
                features.len()
            );

            // Verify features are finite
            for (j, &feat) in features.iter().enumerate() {
                assert!(
                    feat.is_finite(),
                    "Non-finite feature at index {} in bar {}: {}",
                    j,
                    i,
                    feat
                );
            }
        }
    }

    println!("✅ Feature extraction compatibility test passed");

    Ok(())
}

#[test]
fn test_nan_handling_warmup() -> Result<()> {
    use ml::preprocessing::{clip_outliers, compute_log_returns, windowed_normalize};

    // Small dataset to test warmup behavior
    let prices_vec: Vec<f64> = (0..150).map(|i| 4000.0 + (i as f64 * 0.1)).collect();

    let prices_tensor = Tensor::from_slice(&prices_vec, (150,), &Device::Cpu)?;

    // Compute log returns (first value should be 0.0)
    let returns = compute_log_returns(&prices_tensor)?;
    let returns_vec: Vec<f32> = returns.to_vec1()?;

    assert_eq!(returns_vec.len(), 150);
    assert!(
        (returns_vec[0] - 0.0).abs() < 1e-6,
        "First return should be 0.0, got {}",
        returns_vec[0]
    );

    // Apply windowed normalization (warmup = 50)
    let normalized = windowed_normalize(&returns, 50)?;
    let norm_vec: Vec<f32> = normalized.to_vec1()?;

    // First 50 values will use smaller windows, should still be finite
    for (i, &val) in norm_vec.iter().enumerate() {
        assert!(
            val.is_finite(),
            "Non-finite value at index {} during warmup: {}",
            i,
            val
        );
    }

    // Apply clipping
    let clipped = clip_outliers(&normalized, 3.0)?;
    let clipped_vec: Vec<f32> = clipped.to_vec1()?;

    // All values should be finite
    for (i, &val) in clipped_vec.iter().enumerate() {
        assert!(
            val.is_finite(),
            "Non-finite value at index {} after clipping: {}",
            i,
            val
        );
    }

    println!("✅ NaN handling test passed");

    Ok(())
}

#[test]
fn test_end_to_end_pipeline() -> Result<()> {
    use ml::preprocessing::{preprocess_prices, PreprocessConfig};

    // Generate realistic price series (500 bars = ~8 hours at 1-minute resolution)
    let prices_vec = generate_test_prices(500);
    let prices_tensor = Tensor::from_slice(&prices_vec, (500,), &Device::Cpu)?;

    // Configure preprocessing
    let config = PreprocessConfig {
        window_size: 120, // 2-hour rolling window
        clip_sigma: 3.0,  // Clip at ±3σ
        use_log_returns: true,
    };

    // Apply full pipeline
    let preprocessed = preprocess_prices(&prices_tensor, config)?;

    // Validate output
    let preprocessed_vec: Vec<f32> = preprocessed.to_vec1()?;

    // Check shape
    assert_eq!(preprocessed_vec.len(), 500);

    // Check warmup period has finite values
    for i in 0..config.window_size as usize {
        assert!(
            preprocessed_vec[i].is_finite(),
            "Non-finite value at index {} in warmup: {}",
            i,
            preprocessed_vec[i]
        );
    }

    // Check post-warmup statistics
    let post_warmup: Vec<f32> = preprocessed_vec[config.window_size as usize..].to_vec();

    let mean = post_warmup.iter().sum::<f32>() / post_warmup.len() as f32;
    let variance =
        post_warmup.iter().map(|&x| (x - mean).powi(2)).sum::<f32>() / post_warmup.len() as f32;
    let std = variance.sqrt();
    let max_abs = post_warmup.iter().map(|&x| x.abs()).fold(0.0f32, f32::max);

    println!("Post-warmup statistics:");
    println!("  Mean: {:.6}", mean);
    println!("  Std: {:.4}", std);
    println!("  Max abs: {:.4}", max_abs);

    // Validate statistics are reasonable
    assert!(
        mean.abs() < 0.5,
        "Mean {} too far from zero (expected near 0 for normalized data)",
        mean
    );

    assert!(
        std > 0.5 && std < 2.0,
        "Std {} outside reasonable range [0.5, 2.0]",
        std
    );

    assert!(
        max_abs < 6.0,
        "Max absolute value {} exceeds clipping threshold",
        max_abs
    );

    println!("✅ End-to-end pipeline test passed");

    Ok(())
}