foxhunt/data/tests/feature_extraction_tests.rs

//! Feature Extraction and Engineering Tests
//!
//! Comprehensive tests for feature engineering pipeline covering:
//! - Technical indicators (MA, RSI, MACD, Bollinger Bands)
//! - Market microstructure features
//! - Temporal features
//! - Feature normalization and scaling
//! - Feature vector construction
//!
//! Tests use REAL market data from BTC-USD and ETH-USD Parquet files
//! to ensure feature calculations work with production-quality data.

#![allow(unused_crate_dependencies)]

mod real_data_helpers;

use chrono::{Datelike, Timelike, Utc};
use data::features::{FeatureMetadata, FeatureVector, PricePoint};
use real_data_helpers::RealDataLoader;
use std::collections::HashMap;

// ============================================================================
// PricePoint Tests
// ============================================================================

#[test]
fn test_price_point_construction() {
    let point = PricePoint {
        timestamp: Utc::now(),
        open: 100.0,
        high: 102.0,
        low: 99.0,
        close: 101.0,
    };

    assert!(point.high >= point.low);
    assert!(point.high >= point.open);
    assert!(point.high >= point.close);
    assert!(point.low <= point.open);
    assert!(point.low <= point.close);
}

#[test]
fn test_price_point_edge_cases() {
    // Test equal OHLC values
    let point = PricePoint {
        timestamp: Utc::now(),
        open: 100.0,
        high: 100.0,
        low: 100.0,
        close: 100.0,
    };

    assert_eq!(point.open, point.close);
    assert_eq!(point.high, point.low);
}

#[test]
fn test_price_point_validation() {
    let points = vec![
        PricePoint {
            timestamp: Utc::now(),
            open: -1.0,
            high: 100.0,
            low: 50.0,
            close: 75.0,
        },
        PricePoint {
            timestamp: Utc::now(),
            open: 100.0,
            high: 50.0,
            low: 100.0,
            close: 75.0,
        },
        PricePoint {
            timestamp: Utc::now(),
            open: f64::NAN,
            high: 100.0,
            low: 50.0,
            close: 75.0,
        },
    ];

    for point in points {
        let is_valid = point.open > 0.0
            && point.high >= point.low
            && point.open.is_finite()
            && point.high.is_finite()
            && point.low.is_finite()
            && point.close.is_finite();

        assert!(!is_valid);
    }
}

// ============================================================================
// Moving Average Tests (with REAL BTC data)
// ============================================================================

#[tokio::test]
async fn test_simple_moving_average_real_data() {
    let loader = RealDataLoader::new();

    // Skip if real data files don't exist
    if !loader.files_exist() {
        eprintln!("⚠️  Skipping test: Real Parquet data files not found");
        return;
    }

    // Load real BTC close prices
    let prices = loader
        .load_btc_close_prices(100)
        .await
        .expect("Failed to load BTC prices");

    assert!(!prices.is_empty(), "Should load BTC prices");
    println!("Loaded {} BTC close prices", prices.len());

    let window = 20;
    let mut smas = Vec::new();

    for i in window - 1..prices.len() {
        let sum: f64 = prices[i - window + 1..=i].iter().sum();
        let sma = sum / window as f64;
        smas.push(sma);
    }

    assert_eq!(smas.len(), prices.len() - window + 1);

    // Verify SMA is finite and reasonable
    for sma in &smas {
        assert!(sma.is_finite(), "SMA should be finite");
        assert!(*sma > 0.0, "SMA should be positive for BTC prices");
        // BTC prices typically in 10K-70K range
        assert!(*sma > 1000.0 && *sma < 200_000.0, "SMA should be in reasonable BTC price range");
    }

    println!("✅ SMA-20 range: ${:.2} - ${:.2}",
        smas.iter().cloned().fold(f64::INFINITY, f64::min),
        smas.iter().cloned().fold(f64::NEG_INFINITY, f64::max)
    );
}

#[tokio::test]
async fn test_exponential_moving_average_real_data() {
    let loader = RealDataLoader::new();

    // Skip if real data files don't exist
    if !loader.files_exist() {
        eprintln!("⚠️  Skipping test: Real Parquet data files not found");
        return;
    }

    // Load real ETH close prices
    let prices = loader
        .load_eth_close_prices(50)
        .await
        .expect("Failed to load ETH prices");

    assert!(!prices.is_empty(), "Should load ETH prices");
    println!("Loaded {} ETH close prices", prices.len());

    let alpha = 0.2; // 20% smoothing factor
    let mut ema: f64 = prices[0];

    for &price in &prices[1..] {
        ema = alpha * price + (1.0 - alpha) * ema;
    }

    assert!(ema.is_finite(), "EMA should be finite");
    assert!(ema > 0.0, "EMA should be positive for ETH prices");
    // ETH prices typically in 1K-5K range
    assert!(ema > 500.0 && ema < 20_000.0, "EMA should be in reasonable ETH price range");

    println!("✅ EMA calculation: ${:.2} (alpha={})", ema, alpha);
}

// ============================================================================
// RSI (Relative Strength Index) Tests (with REAL BTC data)
// ============================================================================

#[tokio::test]
async fn test_rsi_calculation_real_data() {
    let loader = RealDataLoader::new();

    // Skip if real data files don't exist
    if !loader.files_exist() {
        eprintln!("⚠️  Skipping test: Real Parquet data files not found");
        return;
    }

    // Load real BTC close prices
    let prices = loader
        .load_btc_close_prices(50)
        .await
        .expect("Failed to load BTC prices");

    assert!(prices.len() >= 15, "Need at least 15 prices for RSI-14");
    println!("Loaded {} BTC close prices for RSI calculation", prices.len());

    let period = 14;
    let mut gains = Vec::new();
    let mut losses = Vec::new();

    for i in 1..prices.len() {
        let change = prices[i] - prices[i - 1];
        if change > 0.0 {
            gains.push(change);
            losses.push(0.0);
        } else {
            gains.push(0.0);
            losses.push(-change);
        }
    }

    assert!(gains.len() >= period, "Should have enough data points");

    let avg_gain: f64 = gains[..period].iter().sum::<f64>() / period as f64;
    let avg_loss: f64 = losses[..period].iter().sum::<f64>() / period as f64;

    println!("RSI calculation: avg_gain={:.4}, avg_loss={:.4}", avg_gain, avg_loss);

    if avg_loss > 0.0 {
        let rs = avg_gain / avg_loss;
        let rsi = 100.0 - (100.0 / (1.0 + rs));

        assert!(rsi >= 0.0 && rsi <= 100.0, "RSI should be between 0 and 100, got {}", rsi);

        // Typical RSI ranges (not extreme)
        // RSI < 30 = oversold, RSI > 70 = overbought
        // For BTC, expect values typically between 20-80 in normal conditions
        assert!(rsi.is_finite(), "RSI should be finite");

        println!("✅ RSI-14 = {:.2} (0=oversold, 50=neutral, 100=overbought)", rsi);
    } else {
        // All gains scenario
        println!("⚠️  All gains detected (RSI = 100)");
    }
}

#[tokio::test]
async fn test_rsi_with_eth_data() {
    let loader = RealDataLoader::new();

    // Skip if real data files don't exist
    if !loader.files_exist() {
        eprintln!("⚠️  Skipping test: Real Parquet data files not found");
        return;
    }

    // Load real ETH close prices
    let prices = loader
        .load_eth_close_prices(50)
        .await
        .expect("Failed to load ETH prices");

    assert!(prices.len() >= 15, "Need at least 15 prices for RSI-14");

    let period = 14;
    let mut rsi_values = Vec::new();

    // Calculate RSI for a sliding window
    for start_idx in 0..(prices.len() - period - 1) {
        let window = &prices[start_idx..start_idx + period + 1];

        let mut gains = 0.0;
        let mut losses = 0.0;

        for i in 1..window.len() {
            let change = window[i] - window[i - 1];
            if change > 0.0 {
                gains += change;
            } else {
                losses += -change;
            }
        }

        let avg_gain = gains / period as f64;
        let avg_loss = losses / period as f64;

        let rsi = if avg_loss > 0.0 {
            let rs = avg_gain / avg_loss;
            100.0 - (100.0 / (1.0 + rs))
        } else {
            100.0 // All gains
        };

        rsi_values.push(rsi);
    }

    // Verify all RSI values are in valid range
    for (i, rsi) in rsi_values.iter().enumerate() {
        assert!(
            *rsi >= 0.0 && *rsi <= 100.0,
            "RSI at index {} should be between 0 and 100, got {}",
            i,
            rsi
        );
    }

    let avg_rsi = rsi_values.iter().sum::<f64>() / rsi_values.len() as f64;
    let min_rsi = rsi_values.iter().cloned().fold(f64::INFINITY, f64::min);
    let max_rsi = rsi_values.iter().cloned().fold(f64::NEG_INFINITY, f64::max);

    println!("✅ RSI statistics: avg={:.2}, min={:.2}, max={:.2} ({} samples)",
        avg_rsi, min_rsi, max_rsi, rsi_values.len());
}

// ============================================================================
// Bollinger Bands Tests (with REAL BTC data)
// ============================================================================

#[tokio::test]
async fn test_bollinger_bands_real_data() {
    let loader = RealDataLoader::new();

    // Skip if real data files don't exist
    if !loader.files_exist() {
        eprintln!("⚠️  Skipping test: Real Parquet data files not found");
        return;
    }

    // Load real BTC close prices
    let prices = loader
        .load_btc_close_prices(100)
        .await
        .expect("Failed to load BTC prices");

    assert!(prices.len() >= 20, "Need at least 20 prices for Bollinger Bands");
    println!("Loaded {} BTC close prices for Bollinger Bands", prices.len());

    let period = 20;
    let num_std = 2.0;

    // Calculate Bollinger Bands for each window
    let mut bb_results = Vec::new();

    for start_idx in 0..(prices.len() - period + 1) {
        let window = &prices[start_idx..start_idx + period];
        let mean: f64 = window.iter().sum::<f64>() / period as f64;
        let variance: f64 = window.iter().map(|x| (x - mean).powi(2)).sum::<f64>() / period as f64;
        let std_dev = variance.sqrt();

        let upper_band = mean + (num_std * std_dev);
        let lower_band = mean - (num_std * std_dev);
        let middle_band = mean;

        // Verify band relationships
        assert!(
            upper_band > middle_band,
            "Upper band should be above middle band"
        );
        assert!(
            lower_band < middle_band,
            "Lower band should be below middle band"
        );
        assert!(upper_band > lower_band, "Upper band should be above lower band");

        // Verify all values are finite
        assert!(upper_band.is_finite(), "Upper band should be finite");
        assert!(lower_band.is_finite(), "Lower band should be finite");
        assert!(middle_band.is_finite(), "Middle band should be finite");

        bb_results.push((upper_band, middle_band, lower_band));
    }

    // Calculate average band width (volatility indicator)
    let avg_band_width: f64 = bb_results
        .iter()
        .map(|(upper, _, lower)| (upper - lower) / lower * 100.0)
        .sum::<f64>()
        / bb_results.len() as f64;

    println!(
        "✅ Bollinger Bands (period={}, std={}): {} samples, avg band width={:.2}%",
        period,
        num_std,
        bb_results.len(),
        avg_band_width
    );
}

// ============================================================================
// MACD Tests (with REAL ETH data)
// ============================================================================

#[tokio::test]
async fn test_macd_calculation_real_data() {
    let loader = RealDataLoader::new();

    // Skip if real data files don't exist
    if !loader.files_exist() {
        eprintln!("⚠️  Skipping test: Real Parquet data files not found");
        return;
    }

    // Load real ETH close prices
    let prices = loader
        .load_eth_close_prices(50)
        .await
        .expect("Failed to load ETH prices");

    assert!(prices.len() >= 26, "Need at least 26 prices for MACD (12,26,9)");
    println!("Loaded {} ETH close prices for MACD calculation", prices.len());

    // Standard MACD parameters
    let fast_period = 12;
    let slow_period = 26;
    let signal_period = 9;

    // Calculate EMAs
    let alpha_fast = 2.0 / (fast_period as f64 + 1.0);
    let alpha_slow = 2.0 / (slow_period as f64 + 1.0);
    let alpha_signal = 2.0 / (signal_period as f64 + 1.0);

    let mut ema_fast = prices[0];
    let mut ema_slow = prices[0];
    let mut macd_values = Vec::new();

    for &price in &prices[1..] {
        ema_fast = alpha_fast * price + (1.0 - alpha_fast) * ema_fast;
        ema_slow = alpha_slow * price + (1.0 - alpha_slow) * ema_slow;

        let macd = ema_fast - ema_slow;
        macd_values.push(macd);
    }

    // Calculate signal line (EMA of MACD)
    let mut signal_line = macd_values[0];
    let mut signal_values = vec![signal_line];

    for &macd in &macd_values[1..] {
        signal_line = alpha_signal * macd + (1.0 - alpha_signal) * signal_line;
        signal_values.push(signal_line);
    }

    // Calculate MACD histogram (MACD - Signal)
    let histogram: Vec<f64> = macd_values
        .iter()
        .zip(signal_values.iter())
        .map(|(macd, signal)| macd - signal)
        .collect();

    // Verify all values are finite
    for (i, &macd) in macd_values.iter().enumerate() {
        assert!(
            macd.is_finite(),
            "MACD value at index {} should be finite",
            i
        );
    }

    for (i, &signal) in signal_values.iter().enumerate() {
        assert!(
            signal.is_finite(),
            "Signal value at index {} should be finite",
            i
        );
    }

    for (i, &hist) in histogram.iter().enumerate() {
        assert!(
            hist.is_finite(),
            "Histogram value at index {} should be finite",
            i
        );
    }

    // Calculate statistics
    let avg_macd = macd_values.iter().sum::<f64>() / macd_values.len() as f64;
    let avg_histogram = histogram.iter().sum::<f64>() / histogram.len() as f64;

    println!(
        "✅ MACD calculation: avg MACD={:.4}, avg histogram={:.4} ({} samples)",
        avg_macd,
        avg_histogram,
        macd_values.len()
    );
}

// ============================================================================
// Temporal Feature Tests
// ============================================================================

#[test]
fn test_temporal_hour_of_day() {
    let now = Utc::now();
    let hour = now.hour();

    assert!(hour < 24);
}

#[test]
fn test_temporal_day_of_week() {
    let now = Utc::now();
    let day = now.weekday().number_from_monday();

    assert!(day >= 1 && day <= 7);
}

#[test]
fn test_temporal_market_session() {
    let hour = 14; // 2 PM

    let session = if hour >= 9 && hour < 16 {
        "regular_hours"
    } else if hour >= 4 && hour < 9 {
        "pre_market"
    } else {
        "after_hours"
    };

    assert_eq!(session, "regular_hours");
}

#[test]
fn test_temporal_cyclical_encoding() {
    let hour = 15;
    let hour_sin = ((hour as f64 / 24.0) * 2.0 * std::f64::consts::PI).sin();
    let hour_cos = ((hour as f64 / 24.0) * 2.0 * std::f64::consts::PI).cos();

    assert!(hour_sin.abs() <= 1.0);
    assert!(hour_cos.abs() <= 1.0);
}

// ============================================================================
// Feature Normalization Tests
// ============================================================================

#[test]
fn test_min_max_normalization() {
    let values = vec![10.0, 20.0, 30.0, 40.0, 50.0];
    let min = values.iter().cloned().fold(f64::INFINITY, f64::min);
    let max = values.iter().cloned().fold(f64::NEG_INFINITY, f64::max);

    let normalized: Vec<f64> = values
        .iter()
        .map(|&v| (v - min) / (max - min))
        .collect();

    for &val in &normalized {
        assert!(val >= 0.0 && val <= 1.0);
    }
    assert_eq!(normalized[0], 0.0);
    assert_eq!(normalized[normalized.len() - 1], 1.0);
}

#[test]
fn test_z_score_normalization() {
    let values = vec![10.0, 20.0, 30.0, 40.0, 50.0];
    let mean: f64 = values.iter().sum::<f64>() / values.len() as f64;
    let variance: f64 = values
        .iter()
        .map(|&x| (x - mean).powi(2))
        .sum::<f64>()
        / values.len() as f64;
    let std_dev = variance.sqrt();

    let normalized: Vec<f64> = values.iter().map(|&v| (v - mean) / std_dev).collect();

    let normalized_mean: f64 = normalized.iter().sum::<f64>() / normalized.len() as f64;
    assert!((normalized_mean).abs() < 0.0001); // Should be close to 0
}

// ============================================================================
// Market Microstructure Tests
// ============================================================================

#[test]
fn test_bid_ask_spread() {
    let bid = 100.0;
    let ask = 100.5;

    let spread = ask - bid;
    let spread_bps = (spread / bid) * 10000.0;

    assert!(spread > 0.0);
    assert!(spread_bps > 0.0);
}

#[test]
fn test_order_imbalance() {
    let bid_volume = 10000.0;
    let ask_volume = 8000.0;

    let total_volume = bid_volume + ask_volume;
    let imbalance = (bid_volume - ask_volume) / total_volume;

    assert!(imbalance >= -1.0 && imbalance <= 1.0);
}

#[test]
fn test_effective_spread() {
    let trade_price: f64 = 100.25;
    let mid_price: f64 = 100.0;

    let effective_spread: f64 = 2.0 * (trade_price - mid_price).abs();
    assert!(effective_spread >= 0.0);
}

// ============================================================================
// Volume-Based Features Tests
// ============================================================================

#[test]
fn test_volume_weighted_average_price() {
    let prices = vec![100.0, 101.0, 102.0];
    let volumes = vec![1000.0, 1500.0, 2000.0];

    let total_value: f64 = prices
        .iter()
        .zip(volumes.iter())
        .map(|(p, v)| p * v)
        .sum();
    let total_volume: f64 = volumes.iter().sum();

    let vwap = total_value / total_volume;
    assert!(vwap > prices[0] && vwap < prices[prices.len() - 1]);
}

#[test]
fn test_volume_profile() {
    let volumes = vec![1000.0, 1500.0, 2000.0, 1800.0, 1200.0];
    let avg_volume: f64 = volumes.iter().sum::<f64>() / volumes.len() as f64;

    for &vol in &volumes {
        let volume_ratio = vol / avg_volume;
        assert!(volume_ratio > 0.0);
    }
}

// ============================================================================
// Feature Vector Tests
// ============================================================================

#[test]
fn test_feature_vector_construction() {
    let mut features = HashMap::new();
    features.insert("sma_20".to_string(), 100.5);
    features.insert("rsi_14".to_string(), 65.0);
    features.insert("volume_ratio".to_string(), 1.2);

    let vector = FeatureVector {
        timestamp: Utc::now(),
        symbol: "AAPL".to_string(),
        features,
        metadata: FeatureMetadata {
            feature_descriptions: HashMap::new(),
            feature_categories: HashMap::new(),
            quality_indicators: HashMap::new(),
            symbol: "AAPL".to_string(),
            timestamp: Utc::now(),
            feature_count: 3,
            categories: Vec::new(),
        },
    };

    assert_eq!(vector.symbol, "AAPL");
    assert_eq!(vector.features.len(), 3);
}

#[test]
fn test_feature_vector_serialization() {
    use serde_json;

    let mut features = HashMap::new();
    features.insert("price".to_string(), 100.0);
    features.insert("volume".to_string(), 1000.0);

    let vector = FeatureVector {
        timestamp: Utc::now(),
        symbol: "AAPL".to_string(),
        features,
        metadata: FeatureMetadata {
            feature_descriptions: HashMap::new(),
            feature_categories: HashMap::new(),
            quality_indicators: HashMap::new(),
            symbol: "AAPL".to_string(),
            timestamp: Utc::now(),
            feature_count: 2,
            categories: Vec::new(),
        },
    };

    let json = serde_json::to_string(&vector).unwrap();
    let deserialized: FeatureVector = serde_json::from_str(&json).unwrap();

    assert_eq!(vector.symbol, deserialized.symbol);
    assert_eq!(vector.features.len(), deserialized.features.len());
}

// ============================================================================
// Missing Data Handling Tests
// ============================================================================

#[test]
fn test_missing_data_forward_fill() {
    let values = vec![Some(10.0), None, None, Some(20.0)];
    let mut filled = Vec::new();
    let mut last_valid = 0.0;

    for val in values {
        match val {
            Some(v) => {
                filled.push(v);
                last_valid = v;
            }
            None => filled.push(last_valid),
        }
    }

    assert_eq!(filled.len(), 4);
    assert_eq!(filled[1], 10.0);
    assert_eq!(filled[2], 10.0);
}

#[test]
fn test_missing_data_interpolation() {
    let values = vec![10.0, f64::NAN, f64::NAN, 20.0];
    let mut filled = Vec::new();

    for i in 0..values.len() {
        if values[i].is_nan() {
            if i > 0 && i < values.len() - 1 && !values[i - 1].is_nan() && !values[i + 1].is_nan()
            {
                let interpolated = (values[i - 1] + values[i + 1]) / 2.0;
                filled.push(interpolated);
            } else {
                filled.push(0.0); // Default fallback
            }
        } else {
            filled.push(values[i]);
        }
    }

    assert!(filled[1] > 10.0 && filled[1] < 20.0);
}

// ============================================================================
// Feature Importance Tests
// ============================================================================

#[test]
fn test_feature_correlation() {
    let feature1 = vec![1.0, 2.0, 3.0, 4.0, 5.0];
    let feature2 = vec![2.0, 4.0, 6.0, 8.0, 10.0];

    let mean1: f64 = feature1.iter().sum::<f64>() / feature1.len() as f64;
    let mean2: f64 = feature2.iter().sum::<f64>() / feature2.len() as f64;

    let covariance: f64 = feature1
        .iter()
        .zip(feature2.iter())
        .map(|(x, y)| (x - mean1) * (y - mean2))
        .sum::<f64>()
        / feature1.len() as f64;

    assert!(covariance > 0.0); // Should be positively correlated
}

// ============================================================================
// Performance Tests
// ============================================================================

#[test]
fn test_feature_calculation_performance() {
    let prices: Vec<f64> = (0..1000).map(|i| 100.0 + i as f64 * 0.1).collect();

    let start = std::time::Instant::now();

    // Calculate simple moving average
    let window = 20;
    let mut smas = Vec::new();
    for i in window - 1..prices.len() {
        let sum: f64 = prices[i - window + 1..=i].iter().sum();
        let sma = sum / window as f64;
        smas.push(sma);
    }

    let duration = start.elapsed();
    assert!(duration.as_millis() < 1000); // Should complete in under 1 second
    assert!(!smas.is_empty());
}

// ============================================================================
// Edge Case Tests
// ============================================================================

#[test]
fn test_division_by_zero_protection() {
    let numerator = 100.0;
    let denominator = 0.0;

    let result = if denominator != 0.0 {
        numerator / denominator
    } else {
        0.0 // Default value
    };

    assert_eq!(result, 0.0);
}

#[test]
fn test_infinity_handling() {
    let values = vec![f64::INFINITY, f64::NEG_INFINITY, 100.0];
    let finite_values: Vec<f64> = values.into_iter().filter(|v| v.is_finite()).collect();

    assert_eq!(finite_values.len(), 1);
    assert_eq!(finite_values[0], 100.0);
}

#[test]
fn test_nan_handling() {
    let values = vec![f64::NAN, 100.0, f64::NAN, 200.0];
    let valid_values: Vec<f64> = values.into_iter().filter(|v| !v.is_nan()).collect();

    assert_eq!(valid_values.len(), 2);
}