foxhunt/ml/tests/wave15_feature_audit_test.rs

//! WAVE 15 (Agent 33): Feature Audit and Cleanup Test
//!
//! This test documents the baseline 225-feature state before cleanup and validates
//! the 125-feature state after unstable feature removal.
//!
//! **Agent 29 Findings** (Primary instability causes):
//! 1. **Statistical Features (indices 175-200)**: Skewness/kurtosis EXTREMELY UNSTABLE
//!    - Can jump from 0 → 3 in single bar with one outlier
//!    - PRIMARY SUSPECT for gradient explosions
//! 2. **Microstructure Features (indices 115-164)**: Division by zero risk
//!    - `amihud_illiquidity = |Return| / Volume` → ∞ when Volume → 0
//! 3. **Redundant TA Indicators**: 80+ feature pairs with correlation >0.95
//!    - Multiple momentum variants, RSI variants, MACD variants
//!
//! **Removal Plan** (100 features total):
//! - Statistical: Remove 6/26 (skewness × 3, kurtosis × 3)
//! - Microstructure: Remove 30/50 (Amihud + 28 placeholders, keep Roll + Corwin-Schultz)
//! - Price patterns: Remove 45/60 (redundant momentum/trend indicators)
//! - Volume patterns: Remove 19/40 (redundant volume ratios)
//! - **Result**: 225 → 125 features

use anyhow::Result;
use chrono::Utc;
use ml::features::extraction::{extract_ml_features, OHLCVBar};

/// Create test OHLCV bars with controlled characteristics
fn create_test_bars(count: usize) -> Vec<OHLCVBar> {
    (0..count)
        .map(|i| OHLCVBar {
            timestamp: Utc::now(),
            open: 100.0 + (i as f64 * 0.1),
            high: 101.0 + (i as f64 * 0.1),
            low: 99.0 + (i as f64 * 0.1),
            close: 100.5 + (i as f64 * 0.1),
            volume: 10000.0 + (i as f64 * 100.0),
        })
        .collect()
}

/// Create test bars with outlier to demonstrate skewness/kurtosis instability
fn create_bars_with_outlier(
    count: usize,
    outlier_idx: usize,
    outlier_magnitude: f64,
) -> Vec<OHLCVBar> {
    (0..count)
        .map(|i| {
            let base_price = 100.0;
            let price = if i == outlier_idx {
                base_price + outlier_magnitude // Outlier
            } else {
                base_price + (i as f64 * 0.01) // Normal price movement
            };

            OHLCVBar {
                timestamp: Utc::now(),
                open: price,
                high: price * 1.01,
                low: price * 0.99,
                close: price,
                volume: 10000.0,
            }
        })
        .collect()
}

#[test]
#[ignore] // Will fail after cleanup (expected)
fn test_feature_count_before_cleanup() {
    // BASELINE: 225 features before cleanup
    let bars = create_test_bars(60);
    let features = extract_ml_features(&bars).expect("Feature extraction failed");

    assert!(!features.is_empty(), "Should extract features after warmup");

    let feature_vec = features.last().unwrap();
    assert_eq!(
        feature_vec.len(),
        225,
        "Baseline: 225 features before cleanup (indices 0-224)"
    );
}

#[test]
fn test_feature_count_after_cleanup() {
    // AFTER CLEANUP: 125 stable features
    let bars = create_test_bars(60);
    let features = extract_ml_features(&bars).expect("Feature extraction failed");

    assert!(!features.is_empty(), "Should extract features after warmup");

    let feature_vec = features.last().unwrap();
    assert_eq!(feature_vec.len(), 125, "After cleanup: 125 stable features");
}

#[test]
fn test_unstable_features_removed() {
    // Verify specific unstable features are removed
    let bars = create_test_bars(60);
    let features = extract_ml_features(&bars).expect("Feature extraction failed");
    let feature_vec = features.last().unwrap();

    // After cleanup, feature vector should be 125
    assert_eq!(feature_vec.len(), 125, "Feature count should be 125");

    // Verify all features are finite (no NaN/Inf)
    for (i, &val) in feature_vec.iter().enumerate() {
        assert!(
            val.is_finite(),
            "Feature {} should be finite, got: {}",
            i,
            val
        );
    }
}

#[test]
#[ignore] // Demonstrates instability - will be fixed after cleanup
fn test_skewness_instability_demonstration() {
    // Demonstrate that skewness is EXTREMELY UNSTABLE with outliers

    // Test case 1: No outlier
    let bars_normal = create_bars_with_outlier(60, 999, 0.0); // No outlier
    let features_normal = extract_ml_features(&bars_normal).expect("Extraction failed");
    let vec_normal = features_normal.last().unwrap();

    // Test case 2: Single outlier (+50 points)
    let bars_outlier = create_bars_with_outlier(60, 55, 50.0); // Outlier at index 55
    let features_outlier = extract_ml_features(&bars_outlier).expect("Extraction failed");
    let vec_outlier = features_outlier.last().unwrap();

    // In 225-feature system:
    // - Skewness is at indices 178-180 (features 175-200 are statistical)
    // - One outlier can cause skewness to jump from ~0 → ~3
    //
    // This test will PASS before cleanup (demonstrating instability)
    // This test will be REMOVED after cleanup (skewness features removed)

    if vec_normal.len() == 225 {
        // Before cleanup: Statistical features at indices 175-200
        let skew_5_normal = vec_normal[178];
        let skew_5_outlier = vec_outlier[178];

        let skewness_delta = (skew_5_outlier - skew_5_normal).abs();

        // Demonstrate instability: Single outlier causes massive skewness jump
        assert!(
            skewness_delta > 1.0,
            "Skewness should jump by >1.0 with single outlier, got delta: {}",
            skewness_delta
        );

        println!("❌ INSTABILITY DEMONSTRATED:");
        println!("   Skewness (no outlier): {:.4}", skew_5_normal);
        println!("   Skewness (1 outlier):  {:.4}", skew_5_outlier);
        println!(
            "   Delta:                 {:.4} (>1.0 = UNSTABLE)",
            skewness_delta
        );
    }
}

#[test]
fn test_feature_stability_after_cleanup() {
    // After cleanup: Features should be STABLE with outliers

    // Test case 1: No outlier
    let bars_normal = create_bars_with_outlier(60, 999, 0.0);
    let features_normal = extract_ml_features(&bars_normal).expect("Extraction failed");
    let vec_normal = features_normal.last().unwrap();

    // Test case 2: Single outlier (+50 points)
    let bars_outlier = create_bars_with_outlier(60, 55, 50.0);
    let features_outlier = extract_ml_features(&bars_outlier).expect("Extraction failed");
    let vec_outlier = features_outlier.last().unwrap();

    // After cleanup: No feature should jump >3 standard deviations
    let mut max_delta = 0.0;
    let mut unstable_feature_idx = None;

    for i in 0..vec_normal.len().min(vec_outlier.len()) {
        let delta = (vec_outlier[i] - vec_normal[i]).abs();
        if delta > max_delta {
            max_delta = delta;
            unstable_feature_idx = Some(i);
        }
    }

    assert!(
        max_delta < 3.0,
        "Feature {} has excessive jump: {:.2} (threshold: 3.0) - unstable feature not removed!",
        unstable_feature_idx.unwrap_or(0),
        max_delta
    );

    println!("✅ STABILITY VERIFIED:");
    println!("   Max feature delta: {:.4} (threshold: 3.0)", max_delta);
    println!("   All features stable with outlier present");
}

#[test]
fn test_removed_features_documented() {
    // Document which features were removed
    let removed_features = vec![
        ("Statistical: Skewness (5-period)", "Index 178 → REMOVED"),
        ("Statistical: Skewness (10-period)", "Index 179 → REMOVED"),
        ("Statistical: Skewness (20-period)", "Index 180 → REMOVED"),
        ("Statistical: Kurtosis (5-period)", "Index 181 → REMOVED"),
        ("Statistical: Kurtosis (10-period)", "Index 182 → REMOVED"),
        ("Statistical: Kurtosis (20-period)", "Index 183 → REMOVED"),
        (
            "Microstructure: Amihud Illiquidity",
            "Index 116 → REMOVED (div-by-zero risk)",
        ),
        (
            "Microstructure: 28 placeholders",
            "Indices 122-149 → REMOVED",
        ),
        (
            "Price Patterns: 45 redundant TA",
            "Various indices → REMOVED (>0.95 correlation)",
        ),
        ("Volume Patterns: 19 redundant", "Various indices → REMOVED"),
    ];

    println!("\n📋 REMOVED FEATURES SUMMARY:");
    for (feature_name, status) in removed_features {
        println!("   - {}: {}", feature_name, status);
    }
    println!("\n   TOTAL REMOVED: 100 features");
    println!("   REMAINING: 125 stable features\n");
}