#![allow(
    clippy::assertions_on_constants,
    clippy::assertions_on_result_states,
    clippy::clone_on_copy,
    clippy::decimal_literal_representation,
    clippy::doc_markdown,
    clippy::empty_line_after_doc_comments,
    clippy::field_reassign_with_default,
    clippy::get_unwrap,
    clippy::identity_op,
    clippy::inconsistent_digit_grouping,
    clippy::indexing_slicing,
    clippy::integer_division,
    clippy::len_zero,
    clippy::let_underscore_must_use,
    clippy::manual_div_ceil,
    clippy::manual_let_else,
    clippy::manual_range_contains,
    clippy::modulo_arithmetic,
    clippy::needless_range_loop,
    clippy::non_ascii_literal,
    clippy::redundant_clone,
    clippy::shadow_reuse,
    clippy::shadow_same,
    clippy::shadow_unrelated,
    clippy::single_match_else,
    clippy::str_to_string,
    clippy::string_slice,
    clippy::tests_outside_test_module,
    clippy::too_many_lines,
    clippy::unnecessary_wraps,
    clippy::unseparated_literal_suffix,
    clippy::use_debug,
    clippy::useless_vec,
    clippy::wildcard_enum_match_arm,
    clippy::else_if_without_else,
    clippy::expect_used,
    clippy::missing_const_for_fn,
    clippy::similar_names,
    clippy::type_complexity,
    clippy::collapsible_else_if,
    clippy::doc_lazy_continuation,
    clippy::items_after_test_module,
    clippy::map_clone,
    clippy::multiple_unsafe_ops_per_block,
    clippy::unwrap_or_default,
    clippy::assign_op_pattern,
    clippy::needless_borrow,
    clippy::println_empty_string,
    clippy::unnecessary_cast,
    clippy::used_underscore_binding,
    clippy::create_dir,
    clippy::implicit_saturating_sub,
    clippy::exit,
    clippy::expect_fun_call,
    clippy::too_many_arguments,
    clippy::unnecessary_map_or,
    clippy::unwrap_used,
    dead_code,
    unused_imports,
    unused_variables,
    clippy::cloned_ref_to_slice_refs,
    clippy::neg_multiply,
    clippy::while_let_loop,
    clippy::bool_assert_comparison,
    clippy::excessive_precision,
    clippy::trivially_copy_pass_by_ref,
    clippy::op_ref,
    clippy::redundant_closure,
    clippy::unnecessary_lazy_evaluations,
    clippy::if_then_some_else_none,
    clippy::unnecessary_to_owned,
    clippy::single_component_path_imports,
)]
//! Feature Normalization Tests
//!
//! Tests for percentile-based feature clipping to prevent outliers
//! from crushing the feature distribution during min-max normalization.
//!
//! ## Problem
//!
//! OBV (On-Balance Volume) features accumulate signed volume over time,
//! leading to extreme outliers (e.g., -863K to +863K). When using
//! min-max normalization, these outliers compress 51/54 other features
//! into a narrow range [0.48, 0.52], making them indistinguishable.
//!
//! ## Solution
//!
//! Apply percentile clipping (1st to 99th percentile) BEFORE min-max
//! normalization. This preserves 98% of data while preventing outliers
//! from dominating the normalization scale.

use std::f64;
use tracing::info;

/// Compute percentile value from sorted data
fn percentile(sorted_data: &[f64], p: f64) -> f64 {
    assert!(
        !sorted_data.is_empty(),
        "Cannot compute percentile of empty data"
    );
    assert!(p >= 0.0 && p <= 1.0, "Percentile must be in [0, 1]");

    let idx = (sorted_data.len() as f64 * p).round() as usize;
    let idx = idx.min(sorted_data.len() - 1);
    sorted_data[idx]
}

/// Apply percentile clipping to features
fn clip_features_by_percentile(features: &[f64], p_low: f64, p_high: f64) -> Vec<f64> {
    let mut sorted = features.to_vec();
    sorted.sort_by(|a, b| a.partial_cmp(b).unwrap());

    let p1 = percentile(&sorted, p_low);
    let p99 = percentile(&sorted, p_high);

    info!(p_low, p1, "Percentile lower bound");
    info!(p_high, p99, "Percentile upper bound");

    features.iter().map(|&x| x.clamp(p1, p99)).collect()
}

/// Normalize features to [0, 1] range
fn normalize_min_max(features: &[f64]) -> Vec<f64> {
    let min = features.iter().copied().fold(f64::INFINITY, f64::min);
    let max = features.iter().copied().fold(f64::NEG_INFINITY, f64::max);
    let range = max - min;

    if range.abs() < 1e-10 {
        // All values are the same, return 0.5
        return vec![0.5; features.len()];
    }

    features.iter().map(|&x| (x - min) / range).collect()
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_percentile_computation() {
        let data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];

        // Test extremes
        assert!((percentile(&data, 0.0) - 1.0).abs() < 1e-10);
        assert!((percentile(&data, 1.0) - 10.0).abs() < 1e-10);

        // Test median (50th percentile)
        let p50 = percentile(&data, 0.5);
        assert!(
            p50 >= 5.0 && p50 <= 6.0,
            "Median should be ~5.5, got {}",
            p50
        );

        // Test 99th percentile
        let p99 = percentile(&data, 0.99);
        assert!(
            p99 >= 9.0 && p99 <= 10.0,
            "99th percentile should be ~10, got {}",
            p99
        );
    }

    #[test]
    fn test_clip_features_without_outliers() {
        // Data without outliers - clipping should have minimal effect
        let features = vec![10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0];
        let clipped = clip_features_by_percentile(&features, 0.01, 0.99);

        // Most values should be unchanged
        for i in 1..9 {
            assert!(
                (clipped[i] - features[i]).abs() < 1.0,
                "Value {} should be mostly unchanged",
                i
            );
        }
    }

    #[test]
    fn test_clip_features_with_extreme_outliers() {
        // Test that clipping works when percentiles exclude outliers
        // Key insight: outliers must be OUTSIDE the 1st-99th percentile range
        let mut features = Vec::new();

        // Add 100 normal values in range [-100, 100]
        for i in -50..50 {
            features.push(i as f64 * 2.0);
        }

        // Add extreme outliers at beginning and end
        // These will be at the 0.5% and 99.5% positions
        features.insert(0, -863_000.0);
        features.push(863_000.0);

        info!(total_features = features.len(), "Total features");

        let clipped = clip_features_by_percentile(&features, 0.02, 0.98);

        // Extreme outliers should be clipped to 2nd and 98th percentile values
        let min_clipped = clipped.iter().copied().fold(f64::INFINITY, f64::min);
        let max_clipped = clipped.iter().copied().fold(f64::NEG_INFINITY, f64::max);

        info!(min_clipped, max_clipped, "Clipped range");

        // After clipping, outliers should be replaced with percentile boundary values
        // which are within the normal range
        assert!(
            max_clipped < 200.0,
            "Max should be clipped to reasonable range, got {}",
            max_clipped
        );
        assert!(
            min_clipped > -200.0,
            "Min should be clipped to reasonable range, got {}",
            min_clipped
        );
    }

    #[test]
    fn test_normalize_min_max_basic() {
        let features = vec![0.0, 25.0, 50.0, 75.0, 100.0];
        let normalized = normalize_min_max(&features);

        // Check bounds
        assert!((normalized[0] - 0.0).abs() < 1e-10, "Min should map to 0");
        assert!((normalized[4] - 1.0).abs() < 1e-10, "Max should map to 1");

        // Check midpoint
        assert!(
            (normalized[2] - 0.5).abs() < 1e-10,
            "Midpoint should map to 0.5"
        );

        // Check all values in [0, 1]
        for val in &normalized {
            assert!(
                *val >= 0.0 && *val <= 1.0,
                "Normalized value {} out of range",
                val
            );
        }
    }

    #[test]
    fn test_normalize_constant_features() {
        // All values the same - should return 0.5
        let features = vec![42.0; 10];
        let normalized = normalize_min_max(&features);

        for val in &normalized {
            assert!(
                (val - 0.5).abs() < 1e-10,
                "Constant features should normalize to 0.5"
            );
        }
    }

    #[test]
    fn test_full_pipeline_with_outliers() {
        // Simulate realistic scenario: 54 features with OBV outliers
        let mut features = Vec::new();

        // 51 normal features (range: 0-100)
        for _ in 0..51 {
            for i in 0..10 {
                features.push(i as f64 * 10.0);
            }
        }

        // 3 OBV features with extreme outliers
        for _ in 0..3 {
            features.push(-863_000.0);
            features.push(863_000.0);
            for i in -5..5 {
                features.push(i as f64 * 100.0);
            }
        }

        info!("Feature Normalization Test");
        info!(total_features = features.len(), "Total features");

        // BEFORE: Direct normalization (broken)
        let normalized_before = normalize_min_max(&features);
        let min_before = normalized_before
            .iter()
            .copied()
            .fold(f64::INFINITY, f64::min);
        let max_before = normalized_before
            .iter()
            .copied()
            .fold(f64::NEG_INFINITY, f64::max);

        info!("BEFORE percentile clipping");
        info!(
            feature_min = features.iter().copied().fold(f64::INFINITY, f64::min),
            feature_max = features.iter().copied().fold(f64::NEG_INFINITY, f64::max),
            "Feature range before clipping"
        );
        info!(min_before, max_before, "Normalized range before clipping");

        // Count how many values are in narrow range [0.48, 0.52]
        let crushed_before = normalized_before
            .iter()
            .filter(|&&x| x >= 0.48 && x <= 0.52)
            .count();
        info!(
            crushed_before,
            crushed_before_pct = 100.0 * crushed_before as f64 / normalized_before.len() as f64,
            "Values crushed to [0.48, 0.52] before clipping"
        );

        // AFTER: Percentile clipping + normalization (fixed)
        let clipped = clip_features_by_percentile(&features, 0.01, 0.99);
        let normalized_after = normalize_min_max(&clipped);
        let min_after = normalized_after
            .iter()
            .copied()
            .fold(f64::INFINITY, f64::min);
        let max_after = normalized_after
            .iter()
            .copied()
            .fold(f64::NEG_INFINITY, f64::max);

        info!("AFTER percentile clipping");
        info!(
            clipped_min = clipped.iter().copied().fold(f64::INFINITY, f64::min),
            clipped_max = clipped.iter().copied().fold(f64::NEG_INFINITY, f64::max),
            "Clipped feature range"
        );
        info!(min_after, max_after, "Normalized range after clipping");

        // Count distribution after fix
        let crushed_after = normalized_after
            .iter()
            .filter(|&&x| x >= 0.48 && x <= 0.52)
            .count();
        info!(
            crushed_after,
            crushed_after_pct = 100.0 * crushed_after as f64 / normalized_after.len() as f64,
            "Values crushed to [0.48, 0.52] after clipping"
        );

        // Assert fix works
        assert!(
            crushed_after < crushed_before / 2,
            "Percentile clipping should reduce feature crushing significantly"
        );

        // Verify full utilization of [0, 1] range
        assert!(
            (min_after - 0.0).abs() < 0.1,
            "Min should be close to 0 after fix"
        );
        assert!(
            (max_after - 1.0).abs() < 0.1,
            "Max should be close to 1 after fix"
        );

        info!("Fix Validated");
        info!("Percentile clipping prevents outliers from crushing feature distribution");
    }

    #[test]
    fn test_obv_realistic_scenario() {
        // Realistic OBV outlier scenario from ES_FUT_180d.parquet
        let mut features = Vec::new();

        // Generate OBV-like data: accumulates over time
        let mut obv = 0.0;
        for i in 0..1000 {
            let volume = 100.0 + (i as f64 % 50.0);
            let direction = if i % 3 == 0 { 1.0 } else { -1.0 };
            obv += volume * direction;
            features.push(obv);
        }

        // Add other normal features (RSI, MACD, etc.)
        for _ in 0..53 {
            for i in 0..1000 {
                features.push((i % 100) as f64);
            }
        }

        info!("OBV Realistic Scenario");
        let orig_min = features.iter().copied().fold(f64::INFINITY, f64::min);
        let orig_max = features.iter().copied().fold(f64::NEG_INFINITY, f64::max);
        info!(orig_min, orig_max, "Original feature range");

        // Apply percentile clipping
        let clipped = clip_features_by_percentile(&features, 0.01, 0.99);
        let clipped_min = clipped.iter().copied().fold(f64::INFINITY, f64::min);
        let clipped_max = clipped.iter().copied().fold(f64::NEG_INFINITY, f64::max);
        info!(clipped_min, clipped_max, "Clipped feature range");

        // Range should be much smaller after clipping
        let orig_range = orig_max - orig_min;
        let clipped_range = clipped_max - clipped_min;
        info!(
            orig_range,
            clipped_range,
            reduction_pct = 100.0 * (1.0 - clipped_range / orig_range),
            "Range reduction after clipping"
        );

        assert!(
            clipped_range < orig_range * 0.5,
            "Clipping should reduce range by at least 50%"
        );
    }

    #[test]
    fn test_edge_case_all_same_value() {
        let features = vec![42.0; 100];
        let clipped = clip_features_by_percentile(&features, 0.01, 0.99);
        let normalized = normalize_min_max(&clipped);

        // All values should normalize to 0.5
        for val in &normalized {
            assert!((val - 0.5).abs() < 1e-10);
        }
    }

    #[test]
    fn test_edge_case_two_values() {
        let features = vec![0.0, 100.0];
        let clipped = clip_features_by_percentile(&features, 0.01, 0.99);
        let normalized = normalize_min_max(&clipped);

        assert!((normalized[0] - 0.0).abs() < 1e-10);
        assert!((normalized[1] - 1.0).abs() < 1e-10);
    }

    #[test]
    fn test_preserves_98_percent_of_data() {
        // Generate 10000 normal values + 200 outliers
        let mut features = Vec::new();

        // 98% normal (0-100)
        for i in 0..9800 {
            features.push((i % 100) as f64);
        }

        // 2% outliers (-100000, +100000)
        for _ in 0..100 {
            features.push(-100_000.0);
            features.push(100_000.0);
        }

        let clipped = clip_features_by_percentile(&features, 0.01, 0.99);

        // Count how many normal values are preserved exactly
        let preserved = features
            .iter()
            .filter(|&&x| x >= 0.0 && x <= 100.0)
            .filter(|&&x| clipped.contains(&x))
            .count();

        let preservation_rate = preserved as f64 / 9800.0;
        info!(preservation_rate_pct = preservation_rate * 100.0, "Preservation rate");

        assert!(
            preservation_rate > 0.95,
            "At least 95% of normal data should be preserved"
        );
    }
}