foxhunt/ml/tests/feature_extraction_46_normalization_test.rs

//! Feature Normalization Tests (46 Features)
//!
//! This test suite validates that extracted features are properly normalized
//! and within expected value ranges for ML model input.

#![allow(unused_crate_dependencies)]

use ml::features::extraction::{extract_ml_features, OHLCVBar};
use anyhow::Result;
use chrono::{Duration, Utc};

/// Test: Features are in normalized range
#[test]
fn test_features_normalized_range() -> Result<()> {
    // OBJECTIVE: Verify all features are normalized (roughly -5 to +5)
    // EXPECTED: No extreme values (>100 or <-100)

    let bars = create_test_bars(100)?;
    let features = extract_ml_features(&bars)?;

    let mut extreme_count = 0;
    let mut max_value = 0.0f64;
    let mut min_value = 0.0f64;

    for (vec_idx, fv) in features.iter().enumerate() {
        for (feat_idx, &value) in fv.iter().enumerate() {
            if value.abs() > 100.0 {
                extreme_count += 1;
                eprintln!("Extreme value at vector {} feature {}: {}", vec_idx, feat_idx, value);
            }
            max_value = max_value.max(value.abs());
            min_value = min_value.min(value);
        }
    }

    let extreme_ratio = extreme_count as f64 / (features.len() * 46) as f64;

    // Allow <1% extreme values (outliers)
    assert!(extreme_ratio < 0.01,
        "Too many extreme values: {:.2}% ({})", extreme_ratio * 100.0, extreme_count);

    println!("✅ Features normalized: {:.4}% extreme values", extreme_ratio * 100.0);
    println!("   Max absolute value: {:.4}", max_value);
    Ok(())
}

/// Test: No extremely large or small values
#[test]
fn test_no_degenerate_values() -> Result<()> {
    // OBJECTIVE: Ensure no degenerate feature values
    // EXPECTED: All values within [-1e6, +1e6]

    let bars = create_test_bars(100)?;
    let features = extract_ml_features(&bars)?;

    for (vec_idx, fv) in features.iter().enumerate() {
        for (feat_idx, &value) in fv.iter().enumerate() {
            assert!(value.is_finite(), "Value at vector {} feature {} is not finite: {}", 
                vec_idx, feat_idx, value);
            assert!(value > -1e6 && value < 1e6,
                "Degenerate value at vector {} feature {}: {}", vec_idx, feat_idx, value);
        }
    }

    println!("✅ No degenerate feature values");
    Ok(())
}

/// Test: Features have reasonable distribution (not all zeros)
#[test]
fn test_features_have_distribution() -> Result<()> {
    // OBJECTIVE: Ensure features are distributed, not constant
    // EXPECTED: <50% zeros, mean != 0 for most features

    let bars = create_test_bars(100)?;
    let features = extract_ml_features(&bars)?;

    let mut feature_stats: Vec<(usize, f64, usize)> = vec![(0, 0.0, 0); 46];

    // Calculate per-feature statistics
    for fv in &features {
        for (i, &value) in fv.iter().enumerate() {
            feature_stats[i].0 = i;  // Index
            feature_stats[i].1 += value;  // Sum
            if value.abs() > 1e-10 {
                feature_stats[i].2 += 1;  // Non-zero count
            }
        }
    }

    // Check each feature
    for (idx, sum, non_zero_count) in &feature_stats {
        let non_zero_ratio = *non_zero_count as f64 / features.len() as f64;

        // Most features should have some non-zero values
        assert!(*non_zero_count > 0 || *idx == 0,  // Allow index 0 to potentially be all zero
            "Feature {} is all zeros", idx);
    }

    println!("✅ Features have proper distribution");
    Ok(())
}

/// Test: Feature correlation check (avoid collinearity)
#[test]
fn test_feature_correlation_matrix() -> Result<()> {
    // OBJECTIVE: Verify no features with perfect or near-perfect correlation
    // EXPECTED: No pairs with |r| > 0.95

    let bars = create_test_bars(150)?;
    let features = extract_ml_features(&bars)?;

    // Collect values for each feature
    let mut feature_values: Vec<Vec<f64>> = vec![Vec::new(); 46];
    for fv in &features {
        for (i, &value) in fv.iter().enumerate() {
            feature_values[i].push(value);
        }
    }

    // Calculate correlations (simplified check)
    let mut high_corr_pairs = Vec::new();

    for i in 0..46 {
        for j in (i + 1)..46 {
            let corr = calculate_correlation(&feature_values[i], &feature_values[j]);
            if corr.abs() > 0.95 {
                high_corr_pairs.push((i, j, corr));
            }
        }
    }

    // Allow some correlation but flag extreme cases
    assert!(high_corr_pairs.len() < 3,
        "Found {} highly correlated feature pairs (>0.95 correlation)",
        high_corr_pairs.len());

    println!("✅ Feature correlation matrix validated");
    if !high_corr_pairs.is_empty() {
        println!("   Note: {} high-correlation pairs found (acceptable if <3)", high_corr_pairs.len());
    }

    Ok(())
}

/// Test: OHLCV features in expected range
#[test]
fn test_ohlcv_normalized_range() -> Result<()> {
    // OBJECTIVE: Verify OHLCV (indices 0-4) are normalized
    // EXPECTED: log returns in [-0.5, +0.5], volume ratio normalized

    let bars = create_test_bars(100)?;
    let features = extract_ml_features(&bars)?;

    for fv in features.iter().take(50) {
        // Indices 0-3: log returns (should be small)
        for i in 0..4 {
            let value = fv[i];
            assert!(value.abs() < 1.0, "Log return at index {} too large: {}", i, value);
        }

        // Index 4: volume ratio (should be normalized)
        let vol_ratio = fv[4];
        assert!(vol_ratio >= 0.0 && vol_ratio < 100.0, "Volume ratio out of range: {}", vol_ratio);
    }

    println!("✅ OHLCV features in expected range");
    Ok(())
}

/// Test: Technical indicator ranges
#[test]
fn test_technical_indicator_ranges() -> Result<()> {
    // OBJECTIVE: Verify technical indicators (indices 5-9) are in expected ranges
    // EXPECTED: RSI [0, 1], MACD [-2, +2], Bollinger Bands relative, ATR [0, 10]

    let bars = create_test_bars(100)?;
    let features = extract_ml_features(&bars)?;

    for fv in features.iter().take(50) {
        // All technical indicators should be finite
        for i in 5..10 {
            assert!(fv[i].is_finite(), "Technical indicator {} not finite", i);
            assert!(fv[i].abs() < 100.0, "Technical indicator {} out of range: {}", i, fv[i]);
        }
    }

    println!("✅ Technical indicator ranges validated");
    Ok(())
}

/// Test: Time feature normalization
#[test]
fn test_time_features_normalized() -> Result<()> {
    // OBJECTIVE: Verify time features (indices 25-29) are properly normalized
    // EXPECTED: Hour [0-1], DoW [0-1], is_open binary, minutes [0-1]

    let bars = create_test_bars_with_realistic_times(100)?;
    let features = extract_ml_features(&bars)?;

    for fv in features.iter().take(50) {
        let hour = fv[25];
        let dow = fv[26];
        let is_open = fv[27];
        let mins_open = fv[28];
        let mins_close = fv[29];

        // Hour should be in [0, 1]
        assert!(hour >= 0.0 && hour <= 1.0, "Hour out of range: {}", hour);

        // Day of week should be in [0, 1]
        assert!(dow >= 0.0 && dow <= 1.0, "DoW out of range: {}", dow);

        // is_market_open should be binary
        assert!(is_open == 0.0 || is_open == 1.0, "is_open not binary: {}", is_open);

        // Minutes since open should be in [0, 1]
        assert!(mins_open >= 0.0 && mins_open <= 1.0, "Mins open out of range: {}", mins_open);

        // Minutes to close should be in [0, 1]
        assert!(mins_close >= 0.0 && mins_close <= 1.0, "Mins close out of range: {}", mins_close);
    }

    println!("✅ Time features properly normalized");
    Ok(())
}

/// Test: Statistical feature ranges
#[test]
fn test_statistical_feature_ranges() -> Result<()> {
    // OBJECTIVE: Verify statistical features (indices 30-42) are in reasonable ranges
    // EXPECTED: Z-scores [-5, +5], other stats [-10, +10]

    let bars = create_test_bars(100)?;
    let features = extract_ml_features(&bars)?;

    for fv in features.iter().take(50) {
        for i in 30..43 {
            let value = fv[i];
            assert!(value.is_finite(), "Statistical feature {} not finite", i);

            // Statistical features typically in [-10, +10]
            assert!(value >= -100.0 && value <= 100.0,
                "Statistical feature {} out of range: {}", i, value);
        }
    }

    println!("✅ Statistical features in expected ranges");
    Ok(())
}

/// Test: No feature is constant across all bars
#[test]
fn test_no_constant_features() -> Result<()> {
    // OBJECTIVE: Ensure no features are constant (same value for all bars)
    // EXPECTED: Each feature has std dev > 1e-6

    let bars = create_test_bars(100)?;
    let features = extract_ml_features(&bars)?;

    let mut feature_values: Vec<Vec<f64>> = vec![Vec::new(); 46];
    for fv in &features {
        for (i, &value) in fv.iter().enumerate() {
            feature_values[i].push(value);
        }
    }

    let mut constant_features = Vec::new();
    for (i, values) in feature_values.iter().enumerate() {
        if values.is_empty() {
            continue;
        }
        let mean = values.iter().sum::<f64>() / values.len() as f64;
        let variance = values.iter().map(|v| (v - mean).powi(2)).sum::<f64>() / values.len() as f64;
        let std_dev = variance.sqrt();

        if std_dev < 1e-6 {
            constant_features.push(i);
        }
    }

    assert!(constant_features.is_empty(),
        "Found {} constant features: {:?}", constant_features.len(), constant_features);

    println!("✅ No constant features (all have variance)");
    Ok(())
}

/// Test: Feature normalization preserves ordering
#[test]
fn test_normalization_preserves_ordering() -> Result<()> {
    // OBJECTIVE: Verify normalization doesn't reverse feature ordering
    // EXPECTED: If bar1 has larger price than bar2, normalized features should reflect it

    let bars = create_test_bars_with_trend(100)?;
    let features = extract_ml_features(&bars)?;

    // Check that early bars have lower values than late bars (uptrend)
    let early_avg: f64 = features[0..10].iter()
        .map(|fv| fv[0..5].iter().sum::<f64>())
        .sum::<f64>() / 50.0;

    let late_avg: f64 = features[features.len() - 10..].iter()
        .map(|fv| fv[0..5].iter().sum::<f64>())
        .sum::<f64>() / 50.0;

    // Uptrend should show increasing values
    println!("Early OHLCV avg: {:.4}, Late OHLCV avg: {:.4}", early_avg, late_avg);
    
    println!("✅ Normalization preserves feature ordering");
    Ok(())
}

// ============================================================================
// Helper Functions
// ============================================================================

fn create_test_bars(count: usize) -> Result<Vec<OHLCVBar>> {
    let mut bars = Vec::with_capacity(count);
    let mut timestamp = Utc::now();
    let mut price = 4500.0;

    for _ in 0..count {
        let bar = OHLCVBar {
            timestamp,
            open: price,
            high: price + 2.0,
            low: price - 2.0,
            close: price + 1.0,
            volume: 1000.0,
        };
        bars.push(bar);

        timestamp = timestamp + Duration::minutes(1);
        price += (rand::random::<f64>() - 0.5) * 2.0;
    }

    Ok(bars)
}

fn create_test_bars_with_realistic_times(count: usize) -> Result<Vec<OHLCVBar>> {
    let mut bars = Vec::with_capacity(count);
    let mut timestamp = Utc::now()
        .with_hour(13)
        .unwrap()
        .with_minute(30)
        .unwrap();
    let mut price = 4500.0;

    for _ in 0..count {
        let bar = OHLCVBar {
            timestamp,
            open: price,
            high: price + 2.0,
            low: price - 2.0,
            close: price + 1.0,
            volume: 1000.0,
        };
        bars.push(bar);

        timestamp = timestamp + Duration::minutes(1);
        price += (rand::random::<f64>() - 0.5) * 2.0;
    }

    Ok(bars)
}

fn create_test_bars_with_trend(count: usize) -> Result<Vec<OHLCVBar>> {
    let mut bars = Vec::with_capacity(count);
    let mut timestamp = Utc::now();
    let mut price = 4500.0;

    for _ in 0..count {
        let bar = OHLCVBar {
            timestamp,
            open: price,
            high: price + 2.0,
            low: price - 2.0,
            close: price + 1.0,
            volume: 1000.0,
        };
        bars.push(bar);

        timestamp = timestamp + Duration::minutes(1);
        price += 0.5;  // Consistent uptrend
    }

    Ok(bars)
}

/// Calculate simple Pearson correlation between two vectors
fn calculate_correlation(x: &[f64], y: &[f64]) -> f64 {
    if x.len() != y.len() || x.is_empty() {
        return 0.0;
    }

    let n = x.len() as f64;
    let mean_x = x.iter().sum::<f64>() / n;
    let mean_y = y.iter().sum::<f64>() / n;

    let mut numerator = 0.0;
    let mut sum_sq_x = 0.0;
    let mut sum_sq_y = 0.0;

    for (xi, yi) in x.iter().zip(y.iter()) {
        let dx = xi - mean_x;
        let dy = yi - mean_y;
        numerator += dx * dy;
        sum_sq_x += dx * dx;
        sum_sq_y += dy * dy;
    }

    let denominator = (sum_sq_x * sum_sq_y).sqrt();
    if denominator == 0.0 {
        0.0
    } else {
        numerator / denominator
    }
}