foxhunt/ml/tests/feature_normalization_test.rs

//! Feature Normalization Tests
//!
//! Tests for percentile-based feature clipping to prevent outliers
//! from crushing the feature distribution during min-max normalization.
//!
//! ## Problem
//!
//! OBV (On-Balance Volume) features accumulate signed volume over time,
//! leading to extreme outliers (e.g., -863K to +863K). When using
//! min-max normalization, these outliers compress 222/225 other features
//! into a narrow range [0.48, 0.52], making them indistinguishable.
//!
//! ## Solution
//!
//! Apply percentile clipping (1st to 99th percentile) BEFORE min-max
//! normalization. This preserves 98% of data while preventing outliers
//! from dominating the normalization scale.

use std::f64;

/// Compute percentile value from sorted data
fn percentile(sorted_data: &[f64], p: f64) -> f64 {
    assert!(!sorted_data.is_empty(), "Cannot compute percentile of empty data");
    assert!(p >= 0.0 && p <= 1.0, "Percentile must be in [0, 1]");

    let idx = (sorted_data.len() as f64 * p).round() as usize;
    let idx = idx.min(sorted_data.len() - 1);
    sorted_data[idx]
}

/// Apply percentile clipping to features
fn clip_features_by_percentile(features: &[f64], p_low: f64, p_high: f64) -> Vec<f64> {
    let mut sorted = features.to_vec();
    sorted.sort_by(|a, b| a.partial_cmp(b).unwrap());

    let p1 = percentile(&sorted, p_low);
    let p99 = percentile(&sorted, p_high);

    println!("Percentile p1 ({:.2}): {:.2}", p_low, p1);
    println!("Percentile p99 ({:.2}): {:.2}", p_high, p99);

    features.iter()
        .map(|&x| x.clamp(p1, p99))
        .collect()
}

/// Normalize features to [0, 1] range
fn normalize_min_max(features: &[f64]) -> Vec<f64> {
    let min = features.iter().copied().fold(f64::INFINITY, f64::min);
    let max = features.iter().copied().fold(f64::NEG_INFINITY, f64::max);
    let range = max - min;

    if range.abs() < 1e-10 {
        // All values are the same, return 0.5
        return vec![0.5; features.len()];
    }

    features.iter()
        .map(|&x| (x - min) / range)
        .collect()
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_percentile_computation() {
        let data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];

        // Test extremes
        assert!((percentile(&data, 0.0) - 1.0).abs() < 1e-10);
        assert!((percentile(&data, 1.0) - 10.0).abs() < 1e-10);

        // Test median (50th percentile)
        let p50 = percentile(&data, 0.5);
        assert!(p50 >= 5.0 && p50 <= 6.0, "Median should be ~5.5, got {}", p50);

        // Test 99th percentile
        let p99 = percentile(&data, 0.99);
        assert!(p99 >= 9.0 && p99 <= 10.0, "99th percentile should be ~10, got {}", p99);
    }

    #[test]
    fn test_clip_features_without_outliers() {
        // Data without outliers - clipping should have minimal effect
        let features = vec![10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0];
        let clipped = clip_features_by_percentile(&features, 0.01, 0.99);

        // Most values should be unchanged
        for i in 1..9 {
            assert!((clipped[i] - features[i]).abs() < 1.0,
                    "Value {} should be mostly unchanged", i);
        }
    }

    #[test]
    fn test_clip_features_with_extreme_outliers() {
        // Test that clipping works when percentiles exclude outliers
        // Key insight: outliers must be OUTSIDE the 1st-99th percentile range
        let mut features = Vec::new();

        // Add 100 normal values in range [-100, 100]
        for i in -50..50 {
            features.push(i as f64 * 2.0);
        }

        // Add extreme outliers at beginning and end
        // These will be at the 0.5% and 99.5% positions
        features.insert(0, -863_000.0);
        features.push(863_000.0);

        println!("Total features: {}", features.len());

        let clipped = clip_features_by_percentile(&features, 0.02, 0.98);

        // Extreme outliers should be clipped to 2nd and 98th percentile values
        let min_clipped = clipped.iter().copied().fold(f64::INFINITY, f64::min);
        let max_clipped = clipped.iter().copied().fold(f64::NEG_INFINITY, f64::max);

        println!("Min clipped: {}, Max clipped: {}", min_clipped, max_clipped);

        // After clipping, outliers should be replaced with percentile boundary values
        // which are within the normal range
        assert!(max_clipped < 200.0, "Max should be clipped to reasonable range, got {}", max_clipped);
        assert!(min_clipped > -200.0, "Min should be clipped to reasonable range, got {}", min_clipped);
    }

    #[test]
    fn test_normalize_min_max_basic() {
        let features = vec![0.0, 25.0, 50.0, 75.0, 100.0];
        let normalized = normalize_min_max(&features);

        // Check bounds
        assert!((normalized[0] - 0.0).abs() < 1e-10, "Min should map to 0");
        assert!((normalized[4] - 1.0).abs() < 1e-10, "Max should map to 1");

        // Check midpoint
        assert!((normalized[2] - 0.5).abs() < 1e-10, "Midpoint should map to 0.5");

        // Check all values in [0, 1]
        for val in &normalized {
            assert!(*val >= 0.0 && *val <= 1.0, "Normalized value {} out of range", val);
        }
    }

    #[test]
    fn test_normalize_constant_features() {
        // All values the same - should return 0.5
        let features = vec![42.0; 10];
        let normalized = normalize_min_max(&features);

        for val in &normalized {
            assert!((val - 0.5).abs() < 1e-10, "Constant features should normalize to 0.5");
        }
    }

    #[test]
    fn test_full_pipeline_with_outliers() {
        // Simulate realistic scenario: 225 features with OBV outliers
        let mut features = Vec::new();

        // 222 normal features (range: 0-100)
        for _ in 0..222 {
            for i in 0..10 {
                features.push(i as f64 * 10.0);
            }
        }

        // 3 OBV features with extreme outliers
        for _ in 0..3 {
            features.push(-863_000.0);
            features.push(863_000.0);
            for i in -5..5 {
                features.push(i as f64 * 100.0);
            }
        }

        println!("\n=== Feature Normalization Test ===");
        println!("Total features: {}", features.len());

        // BEFORE: Direct normalization (broken)
        let normalized_before = normalize_min_max(&features);
        let min_before = normalized_before.iter().copied().fold(f64::INFINITY, f64::min);
        let max_before = normalized_before.iter().copied().fold(f64::NEG_INFINITY, f64::max);

        println!("\nBEFORE percentile clipping:");
        println!("  Feature range: {:.2} to {:.2}",
                 features.iter().copied().fold(f64::INFINITY, f64::min),
                 features.iter().copied().fold(f64::NEG_INFINITY, f64::max));
        println!("  Normalized range: [{:.6}, {:.6}]", min_before, max_before);

        // Count how many values are in narrow range [0.48, 0.52]
        let crushed_before = normalized_before.iter()
            .filter(|&&x| x >= 0.48 && x <= 0.52)
            .count();
        println!("  Values crushed to [0.48, 0.52]: {} ({:.1}%)",
                 crushed_before,
                 100.0 * crushed_before as f64 / normalized_before.len() as f64);

        // AFTER: Percentile clipping + normalization (fixed)
        let clipped = clip_features_by_percentile(&features, 0.01, 0.99);
        let normalized_after = normalize_min_max(&clipped);
        let min_after = normalized_after.iter().copied().fold(f64::INFINITY, f64::min);
        let max_after = normalized_after.iter().copied().fold(f64::NEG_INFINITY, f64::max);

        println!("\nAFTER percentile clipping:");
        println!("  Clipped range: {:.2} to {:.2}",
                 clipped.iter().copied().fold(f64::INFINITY, f64::min),
                 clipped.iter().copied().fold(f64::NEG_INFINITY, f64::max));
        println!("  Normalized range: [{:.6}, {:.6}]", min_after, max_after);

        // Count distribution after fix
        let crushed_after = normalized_after.iter()
            .filter(|&&x| x >= 0.48 && x <= 0.52)
            .count();
        println!("  Values crushed to [0.48, 0.52]: {} ({:.1}%)",
                 crushed_after,
                 100.0 * crushed_after as f64 / normalized_after.len() as f64);

        // Assert fix works
        assert!(crushed_after < crushed_before / 2,
                "Percentile clipping should reduce feature crushing significantly");

        // Verify full utilization of [0, 1] range
        assert!((min_after - 0.0).abs() < 0.1, "Min should be close to 0 after fix");
        assert!((max_after - 1.0).abs() < 0.1, "Max should be close to 1 after fix");

        println!("\n=== Fix Validated ===");
        println!("Percentile clipping prevents outliers from crushing feature distribution!");
    }

    #[test]
    fn test_obv_realistic_scenario() {
        // Realistic OBV outlier scenario from ES_FUT_180d.parquet
        let mut features = Vec::new();

        // Generate OBV-like data: accumulates over time
        let mut obv = 0.0;
        for i in 0..1000 {
            let volume = 100.0 + (i as f64 % 50.0);
            let direction = if i % 3 == 0 { 1.0 } else { -1.0 };
            obv += volume * direction;
            features.push(obv);
        }

        // Add other normal features (RSI, MACD, etc.)
        for _ in 0..224 {
            for i in 0..1000 {
                features.push((i % 100) as f64);
            }
        }

        println!("\n=== OBV Realistic Scenario ===");
        let orig_min = features.iter().copied().fold(f64::INFINITY, f64::min);
        let orig_max = features.iter().copied().fold(f64::NEG_INFINITY, f64::max);
        println!("Original feature range: [{:.0}, {:.0}]", orig_min, orig_max);

        // Apply percentile clipping
        let clipped = clip_features_by_percentile(&features, 0.01, 0.99);
        let clipped_min = clipped.iter().copied().fold(f64::INFINITY, f64::min);
        let clipped_max = clipped.iter().copied().fold(f64::NEG_INFINITY, f64::max);
        println!("Clipped feature range: [{:.0}, {:.0}]", clipped_min, clipped_max);

        // Range should be much smaller after clipping
        let orig_range = orig_max - orig_min;
        let clipped_range = clipped_max - clipped_min;
        println!("Range reduction: {:.0} → {:.0} ({:.1}% reduction)",
                 orig_range, clipped_range,
                 100.0 * (1.0 - clipped_range / orig_range));

        assert!(clipped_range < orig_range * 0.5,
                "Clipping should reduce range by at least 50%");
    }

    #[test]
    fn test_edge_case_all_same_value() {
        let features = vec![42.0; 100];
        let clipped = clip_features_by_percentile(&features, 0.01, 0.99);
        let normalized = normalize_min_max(&clipped);

        // All values should normalize to 0.5
        for val in &normalized {
            assert!((val - 0.5).abs() < 1e-10);
        }
    }

    #[test]
    fn test_edge_case_two_values() {
        let features = vec![0.0, 100.0];
        let clipped = clip_features_by_percentile(&features, 0.01, 0.99);
        let normalized = normalize_min_max(&clipped);

        assert!((normalized[0] - 0.0).abs() < 1e-10);
        assert!((normalized[1] - 1.0).abs() < 1e-10);
    }

    #[test]
    fn test_preserves_98_percent_of_data() {
        // Generate 10000 normal values + 200 outliers
        let mut features = Vec::new();

        // 98% normal (0-100)
        for i in 0..9800 {
            features.push((i % 100) as f64);
        }

        // 2% outliers (-100000, +100000)
        for _ in 0..100 {
            features.push(-100_000.0);
            features.push(100_000.0);
        }

        let clipped = clip_features_by_percentile(&features, 0.01, 0.99);

        // Count how many normal values are preserved exactly
        let preserved = features.iter()
            .filter(|&&x| x >= 0.0 && x <= 100.0)
            .filter(|&&x| clipped.contains(&x))
            .count();

        let preservation_rate = preserved as f64 / 9800.0;
        println!("\nPreservation rate: {:.1}%", preservation_rate * 100.0);

        assert!(preservation_rate > 0.95,
                "At least 95% of normal data should be preserved");
    }
}