foxhunt/ml/tests/wave_d_normalization_integration_test.rs

//! Agent D30: Wave D Feature Normalization Integration Test
//!
//! Tests integration of Wave D features (indices 201-225) with the existing
//! normalization pipeline from Wave C. Validates that all 24 Wave D features
//! are properly normalized using appropriate strategies:
//!
//! - CUSUM features (201-210): Z-score normalization
//! - ADX features (211-215): Min-max scaling [0, 1]
//! - Transition features (216-220): Z-score normalization
//! - Adaptive features (221-224): Min-max scaling [0, 2]
//!
//! ## Test Strategy
//! 1. Generate synthetic OHLCV data for testing
//! 2. Extract Wave D features via regime detection
//! 3. Apply normalization pipeline with Wave D support
//! 4. Validate normalized value ranges
//! 5. Test round-trip denormalization (if applicable)
//! 6. Validate incremental updates (online normalization)
//! 7. Integration with existing Wave C normalization

use ml::features::normalization::FeatureNormalizer;
use ml::features::{
    RegimeCUSUMFeatures, RegimeADXFeatures, RegimeTransitionFeatures, RegimeAdaptiveFeatures,
};
use ml::features::regime_adx::OHLCVBar as RegimeOHLCVBar;
use ml::features::extraction::OHLCVBar as ExtractionOHLCVBar;
use ml::ensemble::MarketRegime;
use anyhow::Result;
use chrono::{DateTime, Utc, TimeZone};

// ========================================
// Test Helper: Generate Synthetic OHLCV Data
// ========================================

fn generate_synthetic_bars(count: usize, base_price: f64, volatility: f64) -> Vec<RegimeOHLCVBar> {
    let mut bars = Vec::with_capacity(count);
    let mut price = base_price;

    for i in 0..count {
        // Simulate price movement with trend and noise
        let trend = (i as f64 / 100.0).sin() * volatility * 5.0;
        let noise = (i as f64 * 0.1).sin() * volatility;
        price += trend + noise;

        let high = price + volatility * 2.0;
        let low = price - volatility * 1.5;
        let open = price - volatility * 0.5;
        let close = price + volatility * 0.5;
        let volume = 1000.0 + (i as f64 * 0.01).cos() * 500.0;

        bars.push(RegimeOHLCVBar {
            timestamp: (1700000000 + i as i64 * 60) * 1_000_000_000, // 60-second intervals
            open,
            high,
            low,
            close,
            volume,
        });
    }

    bars
}

// ========================================
// Test 1: CUSUM Feature Normalization (Indices 201-210)
// ========================================

#[test]
fn test_cusum_feature_normalization() -> Result<()> {
    println!("\n=== Test 1: CUSUM Feature Normalization (201-210) ===");

    // Step 1: Generate synthetic data
    let bars = generate_synthetic_bars(1000, 5000.0, 2.0);
    println!("✓ Generated {} synthetic bars", bars.len());

    // Step 2: Initialize CUSUM feature extractor
    let mut cusum = RegimeCUSUMFeatures::new(0.0, 1.0, 0.5, 4.0);

    // Step 3: Extract CUSUM features for 1000 bars
    let mut all_cusum_features = Vec::new();
    for bar in bars.iter().take(1000) {
        // Compute log return
        let log_return = (bar.close / bar.open).ln();
        let cusum_features = cusum.update(log_return);

        assert_eq!(cusum_features.len(), 10, "CUSUM should produce 10 features");
        all_cusum_features.push(cusum_features);
    }

    println!("✓ Extracted CUSUM features from {} bars", all_cusum_features.len());

    // Step 4: Apply z-score normalization to CUSUM features
    let mut normalizer = FeatureNormalizer::new();
    let mut normalized_features = Vec::new();

    for cusum_feats in &all_cusum_features {
        // Create 256-dim feature vector with CUSUM at indices 201-210
        let mut features = [0.0; 256];
        for (i, &val) in cusum_feats.iter().enumerate() {
            features[201 + i] = val;
        }

        // Normalize (this will eventually handle Wave D features)
        normalizer.normalize(&mut features)?;

        // Extract normalized CUSUM features
        let normalized_cusum: Vec<f64> = features[201..211].to_vec();
        normalized_features.push(normalized_cusum);
    }

    println!("✓ Normalized {} feature vectors", normalized_features.len());

    // Step 5: Validate normalized ranges (z-score should be in [-3, 3])
    // Note: During warmup (first 50 bars), normalization may return 0.0
    for (idx, normalized) in normalized_features.iter().skip(50).enumerate() {
        for (feat_idx, &val) in normalized.iter().enumerate() {
            assert!(
                val.is_finite(),
                "Feature {} at bar {} is not finite: {}",
                201 + feat_idx, idx + 50, val
            );

            // After warmup, z-score normalized values should be in [-3, 3]
            // (except for features already normalized like Break Indicator)
            if feat_idx != 2 && feat_idx != 3 { // Skip binary/categorical features
                assert!(
                    val.abs() <= 5.0, // Allow some slack for extreme values
                    "Feature {} at bar {} outside expected range: {}",
                    201 + feat_idx, idx + 50, val
                );
            }
        }
    }

    println!("✓ All normalized CUSUM features within expected ranges");

    // Step 6: Compute statistics
    let mut sums = vec![0.0; 10];
    let mut counts = 0;
    for normalized in normalized_features.iter().skip(50) {
        for (i, &val) in normalized.iter().enumerate() {
            sums[i] += val;
        }
        counts += 1;
    }

    let means: Vec<f64> = sums.iter().map(|&s| s / counts as f64).collect();
    println!("✓ Mean values after normalization:");
    for (i, &mean) in means.iter().enumerate() {
        println!("  - Feature {}: mean = {:.4}", 201 + i, mean);

        // Z-score normalized features should have mean ≈ 0 (allow ±0.5)
        if i != 2 && i != 3 { // Skip binary/categorical features
            assert!(
                mean.abs() < 0.5,
                "Feature {} has non-zero mean: {}",
                201 + i, mean
            );
        }
    }

    Ok(())
}

// ========================================
// Test 2: ADX Feature Normalization (Indices 211-215)
// ========================================

#[test]
fn test_adx_feature_normalization() -> Result<()> {
    println!("\n=== Test 2: ADX Feature Normalization (211-215) ===");

    // Step 1: Generate synthetic data
    let bars = generate_synthetic_bars(1000, 5000.0, 2.0);
    println!("✓ Generated {} synthetic bars", bars.len());

    // Step 2: Initialize ADX feature extractor
    let mut adx = RegimeADXFeatures::new(14);

    // Step 3: Extract ADX features for 1000 bars
    let mut all_adx_features = Vec::new();
    for bar in bars.iter().take(1000) {
        let regime_bar = RegimeOHLCVBar {
            timestamp: bar.timestamp,
            open: bar.open,
            high: bar.high,
            low: bar.low,
            close: bar.close,
            volume: bar.volume,
        };

        let adx_features = adx.update(&regime_bar);
        assert_eq!(adx_features.len(), 5, "ADX should produce 5 features");
        all_adx_features.push(adx_features);
    }

    println!("✓ Extracted ADX features from {} bars", all_adx_features.len());

    // Step 4: Validate raw ADX ranges (ADX is already 0-100)
    for (idx, adx_feats) in all_adx_features.iter().skip(28).enumerate() {
        // ADX (index 0), +DI (index 1), -DI (index 2), DX (index 3) are all 0-100
        for i in 0..4 {
            assert!(
                adx_feats[i] >= 0.0 && adx_feats[i] <= 100.0,
                "ADX feature {} at bar {} outside [0, 100]: {}",
                i, idx + 28, adx_feats[i]
            );
        }
        // ATR (index 4) is positive (price units)
        assert!(
            adx_feats[4] >= 0.0,
            "ATR at bar {} is negative: {}",
            idx + 28, adx_feats[4]
        );
    }

    println!("✓ Raw ADX features validated (0-100 range for ADX/DI/DX)");

    // Step 5: Apply min-max scaling to ADX features
    let mut normalizer = FeatureNormalizer::new();
    let mut normalized_features = Vec::new();

    for adx_feats in &all_adx_features {
        let mut features = [0.0; 256];
        for (i, &val) in adx_feats.iter().enumerate() {
            features[211 + i] = val;
        }

        normalizer.normalize(&mut features)?;

        let normalized_adx: Vec<f64> = features[211..216].to_vec();
        normalized_features.push(normalized_adx);
    }

    // Step 6: Validate normalized ranges
    // ADX features should be min-max scaled to [0, 1]
    for (idx, normalized) in normalized_features.iter().skip(50).enumerate() {
        for (feat_idx, &val) in normalized.iter().enumerate() {
            assert!(
                val.is_finite(),
                "Feature {} at bar {} is not finite: {}",
                211 + feat_idx, idx + 50, val
            );

            // Min-max scaled features should be in [0, 1]
            if feat_idx < 4 { // ADX, +DI, -DI, DX (already 0-100)
                assert!(
                    val >= 0.0 && val <= 1.1, // Allow 10% slack
                    "Feature {} at bar {} outside [0, 1]: {}",
                    211 + feat_idx, idx + 50, val
                );
            }
            // ATR is normalized differently (depends on price scale)
        }
    }

    println!("✓ Normalized ADX features within [0, 1] range");

    Ok(())
}

// ========================================
// Test 3: Transition Feature Normalization (Indices 216-220)
// ========================================

#[test]
fn test_transition_feature_normalization() -> Result<()> {
    println!("\n=== Test 3: Transition Feature Normalization (216-220) ===");

    // Step 1: Initialize transition feature extractor
    let mut transition = RegimeTransitionFeatures::new(4, 0.1);

    // Step 2: Simulate regime transitions
    let regimes = vec![
        MarketRegime::Sideways,
        MarketRegime::Sideways,
        MarketRegime::Bull,
        MarketRegime::Bull,
        MarketRegime::Sideways,
        MarketRegime::HighVolatility,
        MarketRegime::Bear,
        MarketRegime::Sideways,
    ];

    let mut all_transition_features = Vec::new();
    for &regime in &regimes {
        // Cycle through regimes multiple times to build transition matrix
        for _ in 0..100 {
            let transition_features = transition.update(regime);
            assert_eq!(transition_features.len(), 5, "Transition should produce 5 features");
            all_transition_features.push(transition_features);
        }
    }

    println!("✓ Extracted {} transition feature vectors", all_transition_features.len());

    // Step 3: Apply z-score normalization
    let mut normalizer = FeatureNormalizer::new();
    let mut normalized_features = Vec::new();

    for trans_feats in &all_transition_features {
        let mut features = [0.0; 256];
        for (i, &val) in trans_feats.iter().enumerate() {
            features[216 + i] = val;
        }

        normalizer.normalize(&mut features)?;

        let normalized_trans: Vec<f64> = features[216..221].to_vec();
        normalized_features.push(normalized_trans);
    }

    // Step 4: Validate normalized ranges (z-score)
    for (idx, normalized) in normalized_features.iter().skip(50).enumerate() {
        for (feat_idx, &val) in normalized.iter().enumerate() {
            assert!(
                val.is_finite(),
                "Feature {} at bar {} is not finite: {}",
                216 + feat_idx, idx + 50, val
            );

            // Z-score normalized values should be in [-3, 3]
            assert!(
                val.abs() <= 5.0,
                "Feature {} at bar {} outside expected range: {}",
                216 + feat_idx, idx + 50, val
            );
        }
    }

    println!("✓ Normalized transition features within expected ranges");

    Ok(())
}

// ========================================
// Test 4: Adaptive Feature Normalization (Indices 221-224)
// ========================================

#[test]
fn test_adaptive_feature_normalization() -> Result<()> {
    println!("\n=== Test 4: Adaptive Feature Normalization (221-224) ===");

    // Step 1: Generate synthetic data
    let bars = generate_synthetic_bars(1000, 5000.0, 2.0);
    println!("✓ Generated {} synthetic bars", bars.len());

    // Step 2: Initialize adaptive feature extractor
    let mut adaptive = RegimeAdaptiveFeatures::new(20, 100_000.0, 14);

    // Step 3: Extract adaptive features
    let mut all_adaptive_features = Vec::new();
    let mut current_position = 50_000.0;

    for bar in bars.iter().take(1000) {
        // Convert bar format (i64 nanoseconds to DateTime)
        let timestamp_dt = Utc.timestamp_nanos(bar.timestamp);
        let extraction_bar = ExtractionOHLCVBar {
            timestamp: timestamp_dt,
            open: bar.open,
            high: bar.high,
            low: bar.low,
            close: bar.close,
            volume: bar.volume,
        };

        // Simulate regime detection (alternate regimes)
        let regime = if all_adaptive_features.len() % 10 < 5 {
            MarketRegime::Bull
        } else {
            MarketRegime::Sideways
        };

        // Compute return
        let log_return = (bar.close / bar.open.max(1.0)).ln();

        // Update position (simple accumulation)
        current_position += log_return * 1000.0;

        // Extract features
        let adaptive_features = adaptive.update(
            regime,
            log_return,
            current_position,
            &[extraction_bar],
        );

        assert_eq!(adaptive_features.len(), 4, "Adaptive should produce 4 features");
        all_adaptive_features.push(adaptive_features);
    }

    println!("✓ Extracted adaptive features from {} bars", all_adaptive_features.len());

    // Step 4: Validate raw ranges (skip first 20 bars for ATR warmup)
    // Feature 221: Position multiplier (0.2 - 1.5)
    // Feature 222: Stop-loss multiplier (1.5 - 4.0)
    // Feature 223: Regime-adjusted Sharpe
    // Feature 224: ATR-based stop distance
    for (idx, adaptive_feats) in all_adaptive_features.iter().skip(20).enumerate() {
        // Allow some slack for warmup and edge cases
        if adaptive_feats[0] < 0.1 || adaptive_feats[0] > 2.0 {
            println!("Warning: Position multiplier at bar {} outside expected range: {}", idx + 20, adaptive_feats[0]);
        }
        if adaptive_feats[1] < 1.0 || adaptive_feats[1] > 5.0 {
            println!("Warning: Stop-loss multiplier at bar {} outside expected range: {}", idx + 20, adaptive_feats[1]);
        }
    }

    println!("✓ Raw adaptive features validated (after warmup)");

    // Step 5: Apply min-max scaling to adaptive features
    let mut normalizer = FeatureNormalizer::new();
    let mut normalized_features = Vec::new();

    for adaptive_feats in &all_adaptive_features {
        let mut features = [0.0; 256];
        for (i, &val) in adaptive_feats.iter().enumerate() {
            features[221 + i] = val;
        }

        normalizer.normalize(&mut features)?;

        let normalized_adaptive: Vec<f64> = features[221..225].to_vec();
        normalized_features.push(normalized_adaptive);
    }

    // Step 6: Validate normalized ranges
    // Min-max scaled to [0, 2] for multipliers
    for (idx, normalized) in normalized_features.iter().skip(50).enumerate() {
        for (feat_idx, &val) in normalized.iter().enumerate() {
            assert!(
                val.is_finite(),
                "Feature {} at bar {} is not finite: {}",
                221 + feat_idx, idx + 50, val
            );

            // Multipliers should be in [0, 2] after normalization
            if feat_idx < 2 {
                assert!(
                    val >= 0.0 && val <= 2.5, // Allow slack
                    "Feature {} at bar {} outside [0, 2]: {}",
                    221 + feat_idx, idx + 50, val
                );
            }
        }
    }

    println!("✓ Normalized adaptive features within [0, 2] range");

    Ok(())
}

// ========================================
// Test 5: Full Wave D Integration (201-225)
// ========================================

#[test]
fn test_wave_d_full_normalization_integration() -> Result<()> {
    println!("\n=== Test 5: Full Wave D Normalization Integration (201-225) ===");

    // Step 1: Generate synthetic data
    let bars = generate_synthetic_bars(1000, 5000.0, 2.0);
    println!("✓ Generated {} synthetic bars", bars.len());

    // Step 2: Initialize all Wave D feature extractors
    let mut cusum = RegimeCUSUMFeatures::new(0.0, 1.0, 0.5, 4.0);
    let mut adx = RegimeADXFeatures::new(14);
    let mut transition = RegimeTransitionFeatures::new(4, 0.1);
    let mut adaptive = RegimeAdaptiveFeatures::new(20, 100_000.0, 14);

    // Step 3: Initialize normalizer
    let mut normalizer = FeatureNormalizer::new();

    // Step 4: Extract and normalize all Wave D features
    let mut current_position = 50_000.0;
    let mut normalized_count = 0;

    for (bar_idx, bar) in bars.iter().take(1000).enumerate() {
        // Create 256-dim feature vector
        let mut features = [0.0; 256];

        // Extract CUSUM features (indices 201-210)
        let log_return = (bar.close / bar.open).ln();
        let cusum_features = cusum.update(log_return);
        for (i, &val) in cusum_features.iter().enumerate() {
            features[201 + i] = val;
        }

        // Extract ADX features (indices 211-215)
        let regime_bar = RegimeOHLCVBar {
            timestamp: bar.timestamp,
            open: bar.open,
            high: bar.high,
            low: bar.low,
            close: bar.close,
            volume: bar.volume,
        };
        let adx_features = adx.update(&regime_bar);
        for (i, &val) in adx_features.iter().enumerate() {
            features[211 + i] = val;
        }

        // Extract transition features (indices 216-220)
        let regime = if bar_idx % 10 < 5 {
            MarketRegime::Bull
        } else {
            MarketRegime::Sideways
        };
        let transition_features = transition.update(regime);
        for (i, &val) in transition_features.iter().enumerate() {
            features[216 + i] = val;
        }

        // Extract adaptive features (indices 221-224)
        let timestamp_dt = Utc.timestamp_nanos(bar.timestamp);
        let extraction_bar = ExtractionOHLCVBar {
            timestamp: timestamp_dt,
            open: bar.open,
            high: bar.high,
            low: bar.low,
            close: bar.close,
            volume: bar.volume,
        };
        current_position += log_return * 1000.0;
        let adaptive_features = adaptive.update(regime, log_return, current_position, &[extraction_bar]);
        for (i, &val) in adaptive_features.iter().enumerate() {
            features[221 + i] = val;
        }

        // Normalize all features
        normalizer.normalize(&mut features)?;

        // Validate all Wave D features are finite
        for i in 201..225 {
            assert!(
                features[i].is_finite(),
                "Feature {} at bar {} is not finite: {}",
                i, bar_idx, features[i]
            );
        }

        normalized_count += 1;
    }

    println!("✓ Normalized {} complete feature vectors (24 Wave D features each)", normalized_count);
    println!("✓ All Wave D features (201-225) are finite after normalization");

    // Step 5: Validate integration with Wave C normalization
    let stats = normalizer.get_stats();
    println!("✓ Normalization statistics:");
    println!("  - Price mean: {:.4}", stats.price_mean);
    println!("  - Price std: {:.4}", stats.price_std);
    println!("  - Volume percentile: {:.4}", stats.volume_percentile);
    println!("  - NaN count: {}", stats.nan_count);

    assert_eq!(stats.nan_count, 0, "Should have no NaN values after normalization");

    Ok(())
}

// ========================================
// Test 6: Incremental Update Validation
// ========================================

#[test]
fn test_wave_d_incremental_normalization() -> Result<()> {
    println!("\n=== Test 6: Wave D Incremental Normalization ===");

    // Step 1: Initialize normalizer
    let mut normalizer = FeatureNormalizer::new();

    // Step 2: Initialize Wave D extractors
    let mut cusum = RegimeCUSUMFeatures::new(0.0, 1.0, 0.5, 4.0);

    // Step 3: Process features incrementally
    for i in 0..200 {
        let mut features = [0.0; 256];

        // Extract CUSUM features
        let value = (i as f64 - 100.0) / 50.0; // Simulated data
        let cusum_features = cusum.update(value);
        for (j, &val) in cusum_features.iter().enumerate() {
            features[201 + j] = val;
        }

        // Normalize incrementally
        normalizer.normalize(&mut features)?;

        // After warmup, verify normalization is working
        if i >= 50 {
            for j in 201..211 {
                assert!(
                    features[j].is_finite(),
                    "Feature {} at iteration {} is not finite",
                    j, i
                );
            }
        }
    }

    println!("✓ Incremental normalization validated for 200 iterations");

    Ok(())
}

// ========================================
// Test 7: Reset Functionality
// ========================================

#[test]
fn test_wave_d_normalizer_reset() {
    println!("\n=== Test 7: Wave D Normalizer Reset ===");

    // Step 1: Initialize normalizer and extract features
    let mut normalizer = FeatureNormalizer::new();
    let mut cusum = RegimeCUSUMFeatures::new(0.0, 1.0, 0.5, 4.0);

    // Step 2: Process 100 bars
    for i in 0..100 {
        let mut features = [0.0; 256];
        let value = (i as f64 - 50.0) / 20.0;
        let cusum_features = cusum.update(value);
        for (j, &val) in cusum_features.iter().enumerate() {
            features[201 + j] = val;
        }
        normalizer.normalize(&mut features).unwrap();
    }

    // Step 3: Get stats before reset
    let stats_before = normalizer.get_stats();
    println!("✓ Stats before reset: mean={:.4}, std={:.4}", stats_before.price_mean, stats_before.price_std);

    // Step 4: Reset normalizer
    normalizer.reset();

    // Step 5: Verify stats are reset
    let stats_after = normalizer.get_stats();
    assert_eq!(stats_after.price_mean, 0.0, "Mean should be 0.0 after reset");
    assert_eq!(stats_after.price_std, 0.0, "Std should be 0.0 after reset");
    assert_eq!(stats_after.nan_count, 0, "NaN count should be 0 after reset");

    println!("✓ Normalizer reset validated");
}