foxhunt/ml/tests/data_validation_tests.rs

//! # Data Quality Validation Tests - TDD
//!
//! Comprehensive test suite for DBN data validation following TDD methodology.
//! Tests are written FIRST and will FAIL until implementation is complete.
//!
//! ## Validation Checks
//!
//! 1. OHLCV Integrity: high≥low, volume≥0, price relationships
//! 2. Price Continuity: No >20% spikes between bars
//! 3. Technical Indicators: RSI 0-100, no NaN values
//! 4. Timestamp Alignment: No gaps, proper ordering
//! 5. Data Completeness: No missing bars in sequence
//! 6. Automatic Correction: Price spikes, outliers
//!
//! ## Expected Behavior
//!
//! All tests should PASS after implementing ml/src/data_validation/ module.

use anyhow::Result;
use ml::data_validation::corrector::DataCorrector;
use ml::data_validation::rules::{
    ContinuityRule, CompletenessRule, IndicatorRule, IntegrityRule, TimestampRule,
};
use ml::data_validation::validator::{DataValidator, ValidationReport, ValidationResult};
use ml::real_data_loader::{Indicators, OHLCVBar, RealDataLoader};

/// Test 1: OHLCV integrity validation (high≥low, volume≥0, price relationships)
#[tokio::test]
async fn test_ohlcv_integrity_validation() -> Result<()> {
    println!("\n🔍 Test 1: OHLCV Integrity Validation");
    println!("════════════════════════════════════════════════════════");

    // Create validator with integrity rule
    let validator = DataValidator::new().with_rule(Box::new(IntegrityRule::new()));

    // Test Case 1: Valid data
    let valid_bars = vec![
        create_test_bar(100.0, 105.0, 95.0, 102.0, 1000.0),
        create_test_bar(102.0, 108.0, 100.0, 106.0, 1200.0),
    ];

    let result = validator.validate(&valid_bars)?;
    assert!(
        result.is_valid(),
        "Valid OHLCV data should pass integrity check"
    );
    assert_eq!(result.errors.len(), 0, "No errors expected for valid data");

    // Test Case 2: Invalid high < low
    let invalid_bars = vec![create_test_bar(100.0, 95.0, 105.0, 102.0, 1000.0)];

    let result = validator.validate(&invalid_bars)?;
    assert!(
        !result.is_valid(),
        "Invalid high < low should fail integrity check"
    );
    assert!(result.errors.len() > 0, "Should report high < low error");
    assert!(result.error_summary().contains("high < low"));

    // Test Case 3: Negative volume
    let negative_volume_bars = vec![create_test_bar(100.0, 105.0, 95.0, 102.0, -100.0)];

    let result = validator.validate(&negative_volume_bars)?;
    assert!(!result.is_valid(), "Negative volume should fail");
    assert!(result.error_summary().contains("negative volume"));

    // Test Case 4: High not highest
    let high_not_highest = vec![create_test_bar(110.0, 105.0, 95.0, 102.0, 1000.0)];

    let result = validator.validate(&high_not_highest)?;
    assert!(!result.is_valid(), "High not highest should fail");

    // Test Case 5: Low not lowest
    let low_not_lowest = vec![create_test_bar(100.0, 105.0, 106.0, 102.0, 1000.0)];

    let result = validator.validate(&low_not_lowest)?;
    assert!(!result.is_valid(), "Low not lowest should fail");

    println!("✅ OHLCV integrity validation working correctly");
    Ok(())
}

/// Test 2: Price continuity validation (no >20% spikes)
#[tokio::test]
async fn test_price_continuity_validation() -> Result<()> {
    println!("\n🔍 Test 2: Price Continuity Validation");
    println!("════════════════════════════════════════════════════════");

    // Create validator with continuity rule (20% spike threshold)
    let validator = DataValidator::new().with_rule(Box::new(ContinuityRule::new(0.20)));

    // Test Case 1: Normal price movement (<20% change)
    let normal_bars = vec![
        create_test_bar(100.0, 105.0, 95.0, 102.0, 1000.0),
        create_test_bar(102.0, 110.0, 100.0, 108.0, 1100.0), // 5.9% change
        create_test_bar(108.0, 115.0, 105.0, 112.0, 1050.0), // 3.7% change
    ];

    let result = validator.validate(&normal_bars)?;
    assert!(
        result.is_valid(),
        "Normal price movement should pass continuity check"
    );
    assert_eq!(result.errors.len(), 0, "No errors for normal movement");

    // Test Case 2: Price spike >20%
    let spike_bars = vec![
        create_test_bar(100.0, 105.0, 95.0, 102.0, 1000.0),
        create_test_bar(125.0, 130.0, 120.0, 127.0, 1100.0), // 24.5% spike
    ];

    let result = validator.validate(&spike_bars)?;
    assert!(
        !result.is_valid(),
        "Price spike >20% should fail continuity check"
    );
    assert!(result.errors.len() > 0, "Should report spike error");
    assert!(result.error_summary().contains("spike"));

    // Test Case 3: Exactly 20% threshold
    let threshold_bars = vec![
        create_test_bar(100.0, 105.0, 95.0, 100.0, 1000.0),
        create_test_bar(120.0, 125.0, 115.0, 120.0, 1100.0), // Exactly 20%
    ];

    let result = validator.validate(&threshold_bars)?;
    // Should pass (threshold is exclusive)
    assert!(
        result.is_valid(),
        "Exactly 20% change should be valid (threshold exclusive)"
    );

    println!("✅ Price continuity validation working correctly");
    Ok(())
}

/// Test 3: Technical indicator validation (RSI 0-100, no NaN)
#[tokio::test]
async fn test_indicator_validation() -> Result<()> {
    println!("\n🔍 Test 3: Technical Indicator Validation");
    println!("════════════════════════════════════════════════════════");

    let validator = DataValidator::new().with_rule(Box::new(IndicatorRule::new()));

    // Test Case 1: Valid indicators
    let valid_indicators = Indicators {
        rsi: vec![30.0, 45.0, 55.0, 70.0],
        macd: vec![0.5, 0.8, 1.2, 0.9],
        macd_signal: vec![0.4, 0.7, 1.0, 0.8],
        bb_upper: vec![105.0, 110.0, 115.0, 112.0],
        bb_middle: vec![100.0, 105.0, 108.0, 106.0],
        bb_lower: vec![95.0, 100.0, 101.0, 100.0],
        atr: vec![2.0, 2.5, 3.0, 2.8],
        ema_fast: vec![100.0, 102.0, 105.0, 107.0],
        ema_slow: vec![98.0, 100.0, 102.0, 104.0],
        volume_ma: vec![1000.0, 1100.0, 1050.0, 1150.0],
    };

    let result = validator.validate_indicators(&valid_indicators)?;
    assert!(
        result.is_valid(),
        "Valid indicators should pass validation"
    );
    assert_eq!(result.errors.len(), 0, "No errors for valid indicators");

    // Test Case 2: RSI out of range (>100)
    let invalid_rsi_high = Indicators {
        rsi: vec![30.0, 105.0, 55.0, 70.0], // RSI = 105 (invalid)
        ..valid_indicators.clone()
    };

    let result = validator.validate_indicators(&invalid_rsi_high)?;
    assert!(!result.is_valid(), "RSI >100 should fail validation");
    assert!(result.error_summary().contains("RSI"));

    // Test Case 3: RSI out of range (<0)
    let invalid_rsi_low = Indicators {
        rsi: vec![30.0, -5.0, 55.0, 70.0], // RSI = -5 (invalid)
        ..valid_indicators.clone()
    };

    let result = validator.validate_indicators(&invalid_rsi_low)?;
    assert!(!result.is_valid(), "RSI <0 should fail validation");

    // Test Case 4: NaN values in indicators
    let nan_indicators = Indicators {
        rsi: vec![30.0, f32::NAN, 55.0, 70.0], // NaN in RSI
        ..valid_indicators.clone()
    };

    let result = validator.validate_indicators(&nan_indicators)?;
    assert!(!result.is_valid(), "NaN values should fail validation");
    assert!(result.error_summary().contains("NaN"));

    // Test Case 5: Infinite values
    let inf_indicators = Indicators {
        macd: vec![0.5, f32::INFINITY, 1.2, 0.9], // Infinity in MACD
        ..valid_indicators.clone()
    };

    let result = validator.validate_indicators(&inf_indicators)?;
    assert!(!result.is_valid(), "Infinite values should fail validation");

    println!("✅ Indicator validation working correctly");
    Ok(())
}

/// Test 4: Timestamp alignment validation (no gaps, proper ordering)
#[tokio::test]
async fn test_timestamp_validation() -> Result<()> {
    println!("\n🔍 Test 4: Timestamp Alignment Validation");
    println!("════════════════════════════════════════════════════════");

    let validator = DataValidator::new().with_rule(Box::new(TimestampRule::new(60))); // 60 second bars

    // Test Case 1: Properly ordered timestamps
    let ordered_bars = vec![
        create_test_bar_with_timestamp(100.0, 105.0, 95.0, 102.0, 1000.0, 1000),
        create_test_bar_with_timestamp(102.0, 108.0, 100.0, 106.0, 1100.0, 1060),
        create_test_bar_with_timestamp(106.0, 110.0, 104.0, 108.0, 1050.0, 1120),
    ];

    let result = validator.validate(&ordered_bars)?;
    assert!(
        result.is_valid(),
        "Properly ordered timestamps should pass"
    );

    // Test Case 2: Unordered timestamps
    let unordered_bars = vec![
        create_test_bar_with_timestamp(100.0, 105.0, 95.0, 102.0, 1000.0, 1000),
        create_test_bar_with_timestamp(102.0, 108.0, 100.0, 106.0, 1100.0, 900), // Out of order
    ];

    let result = validator.validate(&unordered_bars)?;
    assert!(
        !result.is_valid(),
        "Unordered timestamps should fail validation"
    );
    assert!(result.error_summary().contains("timestamp"));

    // Test Case 3: Large gap in timestamps (missing bars)
    let gap_bars = vec![
        create_test_bar_with_timestamp(100.0, 105.0, 95.0, 102.0, 1000.0, 1000),
        create_test_bar_with_timestamp(102.0, 108.0, 100.0, 106.0, 1100.0, 1300), // 300s gap (5 bars)
    ];

    let result = validator.validate(&gap_bars)?;
    assert!(
        !result.is_valid(),
        "Large gaps should fail validation"
    );
    assert!(result.error_summary().contains("gap"));

    println!("✅ Timestamp validation working correctly");
    Ok(())
}

/// Test 5: Data completeness validation (no missing bars)
#[tokio::test]
async fn test_completeness_validation() -> Result<()> {
    println!("\n🔍 Test 5: Data Completeness Validation");
    println!("════════════════════════════════════════════════════════");

    let validator = DataValidator::new().with_rule(Box::new(CompletenessRule::new(60, 0.95))); // 95% completeness

    // Test Case 1: Complete data (no gaps)
    let complete_bars = vec![
        create_test_bar_with_timestamp(100.0, 105.0, 95.0, 102.0, 1000.0, 1000),
        create_test_bar_with_timestamp(102.0, 108.0, 100.0, 106.0, 1100.0, 1060),
        create_test_bar_with_timestamp(106.0, 110.0, 104.0, 108.0, 1050.0, 1120),
        create_test_bar_with_timestamp(108.0, 112.0, 106.0, 110.0, 1200.0, 1180),
    ];

    let result = validator.validate(&complete_bars)?;
    assert!(
        result.is_valid(),
        "Complete data should pass completeness check"
    );

    // Test Case 2: Missing bars (below threshold)
    let incomplete_bars = vec![
        create_test_bar_with_timestamp(100.0, 105.0, 95.0, 102.0, 1000.0, 1000),
        create_test_bar_with_timestamp(102.0, 108.0, 100.0, 106.0, 1100.0, 1060),
        // Missing bar at 1120
        create_test_bar_with_timestamp(108.0, 112.0, 106.0, 110.0, 1200.0, 1180),
        // Missing bar at 1240
        create_test_bar_with_timestamp(110.0, 115.0, 108.0, 113.0, 1250.0, 1300),
    ];

    let result = validator.validate(&incomplete_bars)?;
    assert!(
        !result.is_valid(),
        "Incomplete data should fail completeness check"
    );
    assert!(result.error_summary().contains("completeness"));

    println!("✅ Completeness validation working correctly");
    Ok(())
}

/// Test 6: Automatic price spike correction
#[tokio::test]
async fn test_automatic_spike_correction() -> Result<()> {
    println!("\n🔍 Test 6: Automatic Price Spike Correction");
    println!("════════════════════════════════════════════════════════");

    let corrector = DataCorrector::new();

    // Test Case 1: Correct price spike using interpolation
    let bars_with_spike = vec![
        create_test_bar(100.0, 105.0, 95.0, 102.0, 1000.0),
        create_test_bar(200.0, 205.0, 195.0, 202.0, 1100.0), // 100% spike (outlier)
        create_test_bar(104.0, 108.0, 100.0, 106.0, 1050.0),
    ];

    let corrected = corrector.correct_price_spikes(&bars_with_spike, 0.20)?;

    // Corrected middle bar should be interpolated (~103)
    assert!(
        corrected[1].close > 100.0 && corrected[1].close < 110.0,
        "Spike should be interpolated to reasonable value"
    );
    assert_ne!(
        corrected[1].close, bars_with_spike[1].close,
        "Price should be corrected"
    );

    // Test Case 2: No correction for valid data
    let normal_bars = vec![
        create_test_bar(100.0, 105.0, 95.0, 102.0, 1000.0),
        create_test_bar(102.0, 108.0, 100.0, 106.0, 1100.0),
        create_test_bar(106.0, 110.0, 104.0, 108.0, 1050.0),
    ];

    let corrected = corrector.correct_price_spikes(&normal_bars, 0.20)?;

    // No changes should be made
    assert_eq!(
        corrected[0].close, normal_bars[0].close,
        "Valid data should not be modified"
    );
    assert_eq!(
        corrected[1].close, normal_bars[1].close,
        "Valid data should not be modified"
    );

    println!("✅ Automatic spike correction working correctly");
    Ok(())
}

/// Test 7: Automatic outlier removal
#[tokio::test]
async fn test_automatic_outlier_removal() -> Result<()> {
    println!("\n🔍 Test 7: Automatic Outlier Removal");
    println!("════════════════════════════════════════════════════════");

    let corrector = DataCorrector::new();

    // Test Case 1: Remove volume outliers
    let bars_with_outliers = vec![
        create_test_bar(100.0, 105.0, 95.0, 102.0, 1000.0),
        create_test_bar(102.0, 108.0, 100.0, 106.0, 1100.0),
        create_test_bar(106.0, 110.0, 104.0, 108.0, 50000.0), // Volume outlier (50x normal)
        create_test_bar(108.0, 112.0, 106.0, 110.0, 1050.0),
    ];

    let corrected = corrector.remove_outliers(&bars_with_outliers, 3.0)?; // 3 std devs

    // Outlier volume should be capped or interpolated
    assert!(
        corrected[2].volume < 10000.0,
        "Outlier volume should be corrected"
    );
    assert_ne!(
        corrected[2].volume, bars_with_outliers[2].volume,
        "Volume should be modified"
    );

    println!("✅ Automatic outlier removal working correctly");
    Ok(())
}

/// Test 8: Validation report generation
#[tokio::test]
async fn test_validation_report_generation() -> Result<()> {
    println!("\n🔍 Test 8: Validation Report Generation");
    println!("════════════════════════════════════════════════════════");

    let validator = DataValidator::new()
        .with_rule(Box::new(IntegrityRule::new()))
        .with_rule(Box::new(ContinuityRule::new(0.20)))
        .with_rule(Box::new(IndicatorRule::new()));

    // Create data with multiple issues
    let problematic_bars = vec![
        create_test_bar(100.0, 95.0, 105.0, 102.0, 1000.0), // High < low
        create_test_bar(200.0, 205.0, 195.0, 202.0, 1100.0), // 100% spike
        create_test_bar(104.0, 108.0, 100.0, 106.0, -50.0), // Negative volume
    ];

    let result = validator.validate(&problematic_bars)?;

    // Should have multiple errors
    assert!(!result.is_valid(), "Should fail with multiple issues");
    assert!(result.errors.len() >= 3, "Should report all issues");

    // Check report generation
    let report = result.generate_report();
    assert!(report.contains("integrity"), "Report should mention integrity");
    assert!(report.contains("continuity"), "Report should mention continuity");
    assert!(report.contains("FAIL"), "Report should show failure");

    println!("✅ Report:");
    println!("{}", report);

    println!("✅ Validation report generation working correctly");
    Ok(())
}

/// Test 9: Integration with real DBN data
#[tokio::test]
async fn test_real_data_validation_integration() -> Result<()> {
    println!("\n🔍 Test 9: Real Data Validation Integration");
    println!("════════════════════════════════════════════════════════");

    let mut loader = RealDataLoader::new_from_workspace()?;
    let bars = loader.load_symbol_data("ZN.FUT").await?;

    // Create comprehensive validator
    let validator = DataValidator::new()
        .with_rule(Box::new(IntegrityRule::new()))
        .with_rule(Box::new(ContinuityRule::new(0.20)))
        .with_rule(Box::new(TimestampRule::new(60)));

    // Validate real data
    let result = validator.validate(&bars)?;

    println!("📊 Validation Results for ZN.FUT:");
    println!("  Total bars: {}", bars.len());
    println!("  Valid: {}", result.is_valid());
    println!("  Errors: {}", result.errors.len());
    println!("  Warnings: {}", result.warnings.len());

    if !result.is_valid() {
        println!("\n⚠️ Issues found:");
        println!("{}", result.generate_report());
    }

    // Real data should be mostly valid (allow some warnings)
    assert!(
        result.errors.len() < bars.len() / 100,
        "Error rate should be <1%"
    );

    println!("✅ Real data validation integration working");
    Ok(())
}

/// Test 10: Prometheus metrics integration
#[tokio::test]
async fn test_validation_metrics() -> Result<()> {
    println!("\n🔍 Test 10: Prometheus Metrics Integration");
    println!("════════════════════════════════════════════════════════");

    let validator = DataValidator::new()
        .with_rule(Box::new(IntegrityRule::new()))
        .with_metrics_enabled(true);

    let bars = vec![
        create_test_bar(100.0, 105.0, 95.0, 102.0, 1000.0),
        create_test_bar(102.0, 108.0, 100.0, 106.0, 1100.0),
    ];

    let result = validator.validate(&bars)?;

    // Check that metrics were recorded
    let metrics = validator.get_metrics();
    assert!(
        metrics.total_validations > 0,
        "Should record validation count"
    );
    assert!(
        metrics.total_bars_validated >= bars.len(),
        "Should count validated bars"
    );

    println!("📊 Validation Metrics:");
    println!("  Total validations: {}", metrics.total_validations);
    println!("  Total bars: {}", metrics.total_bars_validated);
    println!("  Errors detected: {}", metrics.total_errors);
    println!("  Corrections applied: {}", metrics.total_corrections);

    println!("✅ Prometheus metrics working correctly");
    Ok(())
}

// Helper functions for test data creation

fn create_test_bar(open: f64, high: f64, low: f64, close: f64, volume: f64) -> OHLCVBar {
    OHLCVBar {
        timestamp: chrono::Utc::now(),
        open,
        high,
        low,
        close,
        volume,
    }
}

fn create_test_bar_with_timestamp(
    open: f64,
    high: f64,
    low: f64,
    close: f64,
    volume: f64,
    timestamp_secs: i64,
) -> OHLCVBar {
    OHLCVBar {
        timestamp: chrono::DateTime::from_timestamp(timestamp_secs, 0)
            .unwrap_or_else(chrono::Utc::now),
        open,
        high,
        low,
        close,
        volume,
    }
}