foxhunt/ml/tests/dbn_256_feature_validation.rs

//! Agent A18: DBN 256-Feature Extraction Validation
//!
//! Comprehensive validation of 256-dimensional feature extraction using real DBN market data
//! from ES.FUT, NQ.FUT, ZN.FUT, and 6E.FUT datasets.
//!
//! **Test Coverage**:
//! 1. Load real DBN data from test_data/ (1000+ bars per symbol)
//! 2. Run 256-feature extraction on all symbols
//! 3. Validate feature properties:
//!    - All features within valid ranges (no NaN/Inf)
//!    - Correct feature count (256 per bar)
//!    - Reasonable indicator values (RSI 0-100 before norm, MACD sensible, etc.)
//! 4. Statistical validation (mean, std, min, max for each feature)
//! 5. Cross-symbol consistency checks
//!
//! **Real Data Files**:
//! - ES.FUT: E-mini S&P 500 futures (1,674 bars, 2024-01-02)
//! - 6E.FUT: Euro FX futures (29,937 bars total, 2024-01-02 to 2024-01-31)
//! - ZN.FUT: 10-Year Treasury Note futures (28,935 bars from ml_training/)
//! - NQ.FUT: Nasdaq-100 E-mini futures (ml_training/)

use anyhow::Result;
use ml::features::extraction::{extract_ml_features, OHLCVBar};
use ml::real_data_loader::RealDataLoader;
use std::collections::HashMap;

/// Statistical summary for a single feature
#[derive(Debug, Clone)]
struct FeatureStats {
    mean: f64,
    std_dev: f64,
    min: f64,
    max: f64,
    nan_count: usize,
    inf_count: usize,
    valid_count: usize,
}

impl FeatureStats {
    fn new(values: &[f64]) -> Self {
        let mut valid_values = Vec::new();
        let mut nan_count = 0;
        let mut inf_count = 0;

        for &val in values {
            if val.is_nan() {
                nan_count += 1;
            } else if val.is_infinite() {
                inf_count += 1;
            } else {
                valid_values.push(val);
            }
        }

        let valid_count = valid_values.len();

        if valid_values.is_empty() {
            return Self {
                mean: 0.0,
                std_dev: 0.0,
                min: 0.0,
                max: 0.0,
                nan_count,
                inf_count,
                valid_count: 0,
            };
        }

        let mean = valid_values.iter().sum::<f64>() / valid_count as f64;
        let variance = valid_values
            .iter()
            .map(|v| (v - mean).powi(2))
            .sum::<f64>()
            / valid_count as f64;
        let std_dev = variance.sqrt();
        let min = valid_values.iter().fold(f64::INFINITY, |a, &b| a.min(b));
        let max = valid_values.iter().fold(f64::NEG_INFINITY, |a, &b| a.max(b));

        Self {
            mean,
            std_dev,
            min,
            max,
            nan_count,
            inf_count,
            valid_count,
        }
    }

    fn is_valid(&self) -> bool {
        self.nan_count == 0 && self.inf_count == 0 && self.valid_count > 0
    }
}

/// Validation report for a single symbol
#[derive(Debug)]
struct SymbolValidationReport {
    symbol: String,
    total_bars: usize,
    feature_vectors: usize,
    feature_stats: Vec<FeatureStats>,
    passed: bool,
    errors: Vec<String>,
}

impl SymbolValidationReport {
    fn print_summary(&self) {
        println!("\n{:=<80}", "");
        println!("Symbol: {}", self.symbol);
        println!("Total Bars: {}", self.total_bars);
        println!("Feature Vectors: {}", self.feature_vectors);
        println!("Status: {}", if self.passed { "✅ PASS" } else { "❌ FAIL" });

        if !self.errors.is_empty() {
            println!("\nErrors ({}):", self.errors.len());
            for (i, error) in self.errors.iter().enumerate() {
                println!("  {}. {}", i + 1, error);
            }
        }

        // Sample feature statistics (features 0-14: OHLCV + technical indicators)
        println!("\nSample Feature Statistics (0-14: OHLCV + Indicators):");
        println!("{:-<80}", "");
        println!(
            "{:<6} {:>12} {:>12} {:>12} {:>12} {:>8}",
            "Feat", "Mean", "StdDev", "Min", "Max", "Valid%"
        );
        println!("{:-<80}", "");

        for (i, stat) in self.feature_stats.iter().take(15).enumerate() {
            let valid_pct = if self.feature_vectors > 0 {
                (stat.valid_count as f64 / self.feature_vectors as f64) * 100.0
            } else {
                0.0
            };

            println!(
                "{:<6} {:>12.6} {:>12.6} {:>12.6} {:>12.6} {:>7.2}%",
                i, stat.mean, stat.std_dev, stat.min, stat.max, valid_pct
            );
        }

        // Invalid feature summary
        let invalid_features: Vec<usize> = self
            .feature_stats
            .iter()
            .enumerate()
            .filter(|(_, stat)| !stat.is_valid())
            .map(|(i, _)| i)
            .collect();

        if !invalid_features.is_empty() {
            println!("\n⚠️  Invalid Features ({}):", invalid_features.len());
            for feat_idx in invalid_features.iter().take(10) {
                let stat = &self.feature_stats[*feat_idx];
                println!(
                    "  Feature {}: {} NaNs, {} Infs, {} valid",
                    feat_idx, stat.nan_count, stat.inf_count, stat.valid_count
                );
            }
            if invalid_features.len() > 10 {
                println!("  ... and {} more", invalid_features.len() - 10);
            }
        }

        println!("{:=<80}", "");
    }

    fn print_detailed_stats(&self, feature_indices: &[usize]) {
        println!("\nDetailed Statistics for Selected Features:");
        println!("{:-<100}", "");
        println!(
            "{:<8} {:>15} {:>15} {:>15} {:>15} {:>10} {:>10} {:>10}",
            "Feature", "Mean", "StdDev", "Min", "Max", "NaNs", "Infs", "Valid%"
        );
        println!("{:-<100}", "");

        for &idx in feature_indices {
            if idx < self.feature_stats.len() {
                let stat = &self.feature_stats[idx];
                let valid_pct = if self.feature_vectors > 0 {
                    (stat.valid_count as f64 / self.feature_vectors as f64) * 100.0
                } else {
                    0.0
                };

                println!(
                    "{:<8} {:>15.6} {:>15.6} {:>15.6} {:>15.6} {:>10} {:>10} {:>9.2}%",
                    idx,
                    stat.mean,
                    stat.std_dev,
                    stat.min,
                    stat.max,
                    stat.nan_count,
                    stat.inf_count,
                    valid_pct
                );
            }
        }
        println!("{:-<100}", "");
    }
}

/// Validate feature extraction for a single symbol
async fn validate_symbol(symbol: &str, file_path: &str) -> Result<SymbolValidationReport> {
    println!("\n🔍 Loading data for {}...", symbol);

    // Load real DBN data
    let loader = RealDataLoader::new();
    let bars = loader.load_ohlcv_bars_from_file(file_path).await?;

    let total_bars = bars.len();
    println!("   Loaded {} bars", total_bars);

    // Extract 256-dim features
    println!("   Extracting 256-dim features...");
    let start_time = std::time::Instant::now();
    let features = extract_ml_features(&bars)?;
    let duration = start_time.elapsed();

    let feature_vectors = features.len();
    println!(
        "   Extracted {} feature vectors in {:.2}ms ({:.2}μs/bar)",
        feature_vectors,
        duration.as_secs_f64() * 1000.0,
        (duration.as_secs_f64() * 1_000_000.0) / feature_vectors as f64
    );

    // Validate feature count
    let mut errors = Vec::new();
    for (i, fv) in features.iter().enumerate() {
        if fv.len() != 256 {
            errors.push(format!(
                "Feature vector {} has {} features (expected 256)",
                i,
                fv.len()
            ));
        }
    }

    // Compute statistics for each feature dimension
    let mut feature_stats = Vec::with_capacity(256);
    for feat_idx in 0..256 {
        let values: Vec<f64> = features.iter().map(|fv| fv[feat_idx]).collect();
        let stat = FeatureStats::new(&values);
        feature_stats.push(stat);
    }

    // Validation checks
    let mut passed = true;

    // 1. Check for NaN/Inf values
    for (i, stat) in feature_stats.iter().enumerate() {
        if stat.nan_count > 0 {
            errors.push(format!(
                "Feature {} has {} NaN values ({:.2}%)",
                i,
                stat.nan_count,
                (stat.nan_count as f64 / feature_vectors as f64) * 100.0
            ));
            passed = false;
        }
        if stat.inf_count > 0 {
            errors.push(format!(
                "Feature {} has {} Inf values ({:.2}%)",
                i,
                stat.inf_count,
                (stat.inf_count as f64 / feature_vectors as f64) * 100.0
            ));
            passed = false;
        }
    }

    // 2. Check feature vector count
    const WARMUP_PERIOD: usize = 50;
    let expected_vectors = total_bars.saturating_sub(WARMUP_PERIOD);
    if feature_vectors != expected_vectors {
        errors.push(format!(
            "Expected {} feature vectors (bars - warmup), got {}",
            expected_vectors, feature_vectors
        ));
        passed = false;
    }

    // 3. Validate OHLCV features (0-4) are normalized
    for i in 0..5 {
        let stat = &feature_stats[i];
        if stat.min < -10.0 || stat.max > 10.0 {
            errors.push(format!(
                "OHLCV feature {} has suspicious range: [{:.2}, {:.2}]",
                i, stat.min, stat.max
            ));
        }
    }

    // 4. Validate technical indicators (5-14) have reasonable values
    // RSI should be in [0, 100] before normalization, or [-1, 1] after
    // MACD, Bollinger, etc. should have finite ranges

    Ok(SymbolValidationReport {
        symbol: symbol.to_string(),
        total_bars,
        feature_vectors,
        feature_stats,
        passed,
        errors,
    })
}

#[tokio::test]
async fn test_es_fut_256_features() -> Result<()> {
    let report = validate_symbol(
        "ES.FUT",
        "test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn",
    )
    .await?;

    report.print_summary();
    report.print_detailed_stats(&[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]);

    assert!(
        report.passed,
        "ES.FUT validation failed with {} errors",
        report.errors.len()
    );
    assert_eq!(report.feature_stats.len(), 256);

    // Verify all features are valid (no NaN/Inf)
    for (i, stat) in report.feature_stats.iter().enumerate() {
        assert_eq!(
            stat.nan_count, 0,
            "Feature {} has {} NaN values",
            i, stat.nan_count
        );
        assert_eq!(
            stat.inf_count, 0,
            "Feature {} has {} Inf values",
            i, stat.inf_count
        );
    }

    Ok(())
}

#[tokio::test]
async fn test_6e_fut_256_features() -> Result<()> {
    let report = validate_symbol(
        "6E.FUT",
        "test_data/real/databento/6E.FUT_ohlcv-1m_2024-01-02_to_2024-01-31.dbn",
    )
    .await?;

    report.print_summary();

    assert!(
        report.passed,
        "6E.FUT validation failed with {} errors",
        report.errors.len()
    );
    assert_eq!(report.feature_stats.len(), 256);

    // 6E.FUT has the most data (29,937 bars), verify feature extraction scales
    assert!(
        report.feature_vectors > 1000,
        "Expected > 1000 feature vectors for 6E.FUT, got {}",
        report.feature_vectors
    );

    Ok(())
}

#[tokio::test]
#[ignore] // Large dataset, run manually: cargo test --test dbn_256_feature_validation test_zn_fut_256_features -- --ignored
async fn test_zn_fut_256_features() -> Result<()> {
    // ZN.FUT has 360 files in ml_training/, test a sample
    let sample_files = vec![
        "test_data/real/databento/ml_training/ZN.FUT_ohlcv-1m_2024-02-07.dbn",
        "test_data/real/databento/ml_training/ZN.FUT_ohlcv-1m_2024-03-15.dbn",
        "test_data/real/databento/ml_training/ZN.FUT_ohlcv-1m_2024-04-17.dbn",
    ];

    for file_path in sample_files {
        let report = validate_symbol("ZN.FUT", file_path).await?;
        report.print_summary();

        assert!(
            report.passed,
            "ZN.FUT validation failed for {}: {} errors",
            file_path,
            report.errors.len()
        );
    }

    Ok(())
}

#[tokio::test]
#[ignore] // Run manually: cargo test --test dbn_256_feature_validation test_nq_fut_256_features -- --ignored
async fn test_nq_fut_256_features() -> Result<()> {
    let report = validate_symbol(
        "NQ.FUT",
        "test_data/real/databento/NQ.FUT_ohlcv-1m_2024-01-02.dbn",
    )
    .await?;

    report.print_summary();

    assert!(
        report.passed,
        "NQ.FUT validation failed with {} errors",
        report.errors.len()
    );
    assert_eq!(report.feature_stats.len(), 256);

    Ok(())
}

#[tokio::test]
async fn test_cross_symbol_consistency() -> Result<()> {
    println!("\n{:=':<80}", "");
    println!("Cross-Symbol Consistency Validation");
    println!("{:=':<80}", "");

    // Load data for multiple symbols
    let symbols = vec![
        (
            "ES.FUT",
            "test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn",
        ),
        (
            "NQ.FUT",
            "test_data/real/databento/NQ.FUT_ohlcv-1m_2024-01-02.dbn",
        ),
    ];

    let mut reports = Vec::new();
    for (symbol, file_path) in symbols {
        let report = validate_symbol(symbol, file_path).await?;
        reports.push(report);
    }

    // Consistency checks across symbols
    println!("\nConsistency Checks:");

    // 1. All symbols should produce 256 features
    for report in &reports {
        assert_eq!(
            report.feature_stats.len(),
            256,
            "Symbol {} has {} features (expected 256)",
            report.symbol,
            report.feature_stats.len()
        );
        println!("   ✅ {}: 256 features", report.symbol);
    }

    // 2. Feature ranges should be comparable across symbols (normalized features)
    // Check OHLCV features (0-4) have similar ranges
    for feat_idx in 0..5 {
        println!("\n   Feature {} (OHLCV) ranges:", feat_idx);
        for report in &reports {
            let stat = &report.feature_stats[feat_idx];
            println!(
                "      {}: [{:.4}, {:.4}] (mean: {:.4}, std: {:.4})",
                report.symbol, stat.min, stat.max, stat.mean, stat.std_dev
            );
        }
    }

    // 3. All reports should pass
    for report in &reports {
        assert!(
            report.passed,
            "Symbol {} failed validation",
            report.symbol
        );
    }

    println!("\n✅ Cross-symbol consistency validation PASSED");

    Ok(())
}

#[tokio::test]
async fn test_feature_extraction_performance() -> Result<()> {
    println!("\n{:=':<80}", "");
    println!("Feature Extraction Performance Benchmark");
    println!("{:=':<80}", "");

    let loader = RealDataLoader::new();
    let bars = loader
        .load_ohlcv_bars_from_file("test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn")
        .await?;

    println!("Loaded {} bars", bars.len());

    // Benchmark feature extraction
    let iterations = 10;
    let mut durations = Vec::new();

    for i in 0..iterations {
        let start = std::time::Instant::now();
        let features = extract_ml_features(&bars)?;
        let duration = start.elapsed();
        durations.push(duration);

        println!(
            "   Iteration {}: {:.2}ms ({} feature vectors, {:.2}μs/vector)",
            i + 1,
            duration.as_secs_f64() * 1000.0,
            features.len(),
            (duration.as_secs_f64() * 1_000_000.0) / features.len() as f64
        );
    }

    // Statistics
    let total_duration: std::time::Duration = durations.iter().sum();
    let avg_duration = total_duration / iterations as u32;
    let min_duration = durations.iter().min().unwrap();
    let max_duration = durations.iter().max().unwrap();

    println!("\nPerformance Summary:");
    println!("   Average: {:.2}ms", avg_duration.as_secs_f64() * 1000.0);
    println!("   Min:     {:.2}ms", min_duration.as_secs_f64() * 1000.0);
    println!("   Max:     {:.2}ms", max_duration.as_secs_f64() * 1000.0);

    // Target: <1ms per bar for 256 features (from extraction.rs docs)
    let bars_processed = bars.len() - 50; // After warmup
    let avg_time_per_bar = avg_duration.as_secs_f64() * 1000.0 / bars_processed as f64;
    println!(
        "   Time per bar: {:.4}ms (target: <1ms)",
        avg_time_per_bar
    );

    assert!(
        avg_time_per_bar < 2.0,
        "Feature extraction too slow: {:.4}ms/bar (target: <1ms)",
        avg_time_per_bar
    );

    println!("\n✅ Performance benchmark PASSED");

    Ok(())
}

/// Integration test: Full pipeline validation
#[tokio::test]
async fn test_full_pipeline_integration() -> Result<()> {
    println!("\n{:=':<80}", "");
    println!("Full Pipeline Integration Test");
    println!("{:=':<80}", "");

    // 1. Load real data
    println!("\n1. Loading real DBN data...");
    let loader = RealDataLoader::new();
    let bars = loader
        .load_ohlcv_bars_from_file("test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn")
        .await?;
    println!("   ✅ Loaded {} bars", bars.len());

    // 2. Extract features
    println!("\n2. Extracting 256-dim features...");
    let features = extract_ml_features(&bars)?;
    println!("   ✅ Extracted {} feature vectors", features.len());

    // 3. Validate feature properties
    println!("\n3. Validating feature properties...");

    // 3a. Check dimensions
    assert!(!features.is_empty(), "No features extracted");
    for (i, fv) in features.iter().enumerate() {
        assert_eq!(
            fv.len(),
            256,
            "Feature vector {} has {} dimensions (expected 256)",
            i,
            fv.len()
        );
    }
    println!("   ✅ All feature vectors have 256 dimensions");

    // 3b. Check for invalid values
    let mut total_values = 0;
    let mut nan_count = 0;
    let mut inf_count = 0;

    for fv in &features {
        for &val in fv.iter() {
            total_values += 1;
            if val.is_nan() {
                nan_count += 1;
            }
            if val.is_infinite() {
                inf_count += 1;
            }
        }
    }

    println!(
        "   ✅ Validated {} values: {} NaNs, {} Infs",
        total_values, nan_count, inf_count
    );
    assert_eq!(nan_count, 0, "Found {} NaN values", nan_count);
    assert_eq!(inf_count, 0, "Found {} Inf values", inf_count);

    // 4. Sample feature analysis
    println!("\n4. Sample feature analysis (first vector):");
    let first_vector = &features[0];
    println!("   OHLCV (0-4): {:?}", &first_vector[0..5]);
    println!(
        "   Technical Indicators (5-14): {:?}",
        &first_vector[5..15]
    );
    println!(
        "   Price Patterns (15-24, sample): {:?}",
        &first_vector[15..25]
    );

    println!("\n{:=':<80}", "");
    println!("✅ Full Pipeline Integration Test PASSED");
    println!("{:=':<80}", "");

    Ok(())
}