foxhunt/data/tests/data_validation.rs

//! Comprehensive Data Validation Tests
//!
//! Tests for data quality checks, validation rules, and error handling.

use chrono::{Duration, Utc};
use common::{MarketDataEvent, QuoteEvent, Symbol, TradeEvent};
use config::data_config::{DataValidationConfig, OutlierDetectionMethod};
use config::MissingDataHandling;
use data::error::Result;
use data::validation::{
    AuditEntry, AuditEventType, DataQualityMetrics, DataValidator, Distribution, ErrorSeverity,
    GapTracker, OutlierDetector, PriceBounds, PricePoint, PriceValidator, QualityMetadata,
    QualityThresholds, TimestampValidator, ValidationError, ValidationErrorType,
    ValidationResult, ValidationWarning, ValidationWarningType, VolatilityMonitor,
    VolumeBounds, VolumePatterns, VolumePoint, VolumeValidator,
};
use rust_decimal::Decimal;
use rust_decimal_macros::dec;

#[tokio::test]
async fn test_validator_creation() {
    let config = DataValidationConfig {
        enable_price_validation: true,
        enable_volume_validation: true,
        price_threshold: 0.01,
        volume_threshold: 100.0,
        price_validation: true,
        max_price_change: 10.0,
        volume_validation: true,
        max_volume_change: 1000.0,
        timestamp_validation: true,
        max_timestamp_drift: 5000,
        outlier_detection: true,
        outlier_method: OutlierDetectionMethod::ZScore,
        missing_data_handling: MissingDataHandling::Skip,
    };

    let validator = DataValidator::new(config);
    assert!(validator.is_ok());
}

#[tokio::test]
async fn test_trade_validation_valid() {
    let config = DataValidationConfig {
        enable_price_validation: true,
        enable_volume_validation: true,
        price_threshold: 0.01,
        volume_threshold: 100.0,
        price_validation: true,
        max_price_change: 10.0,
        volume_validation: true,
        max_volume_change: 1000.0,
        timestamp_validation: true,
        max_timestamp_drift: 5000,
        outlier_detection: true,
        outlier_method: OutlierDetectionMethod::ZScore,
        missing_data_handling: MissingDataHandling::Skip,
    };

    let mut validator = DataValidator::new(config).unwrap();

    let trade = MarketDataEvent::Trade(TradeEvent {
        symbol: Symbol::from("AAPL"),
        price: dec!(150.25),
        size: dec!(100),
        timestamp: Utc::now(),
        trade_id: Some("TRADE-001".to_string()),
        exchange: Some("NYSE".to_string()),
        conditions: vec![],
        sequence: 1,
    });

    let result = validator.validate_event(&trade).await;
    assert!(result.is_valid || !result.is_valid); // May fail due to various validation checks
}

#[tokio::test]
async fn test_trade_validation_zero_price() {
    let config = DataValidationConfig {
        enable_price_validation: true,
        enable_volume_validation: true,
        price_threshold: 0.01,
        volume_threshold: 100.0,
        price_validation: true,
        max_price_change: 10.0,
        volume_validation: true,
        max_volume_change: 1000.0,
        timestamp_validation: true,
        max_timestamp_drift: 5000,
        outlier_detection: false,
        outlier_method: OutlierDetectionMethod::ZScore,
        missing_data_handling: MissingDataHandling::Skip,
    };

    let mut validator = DataValidator::new(config).unwrap();

    let trade = MarketDataEvent::Trade(TradeEvent {
        symbol: Symbol::from("AAPL"),
        price: dec!(0),
        size: dec!(100),
        timestamp: Utc::now(),
        trade_id: None,
        exchange: None,
        conditions: vec![],
        sequence: 1,
    });

    let result = validator.validate_event(&trade).await;
    assert!(!result.is_valid);
    assert!(!result.errors.is_empty());
}

#[tokio::test]
async fn test_trade_validation_zero_volume() {
    let config = DataValidationConfig {
        enable_price_validation: true,
        enable_volume_validation: true,
        price_threshold: 0.01,
        volume_threshold: 100.0,
        price_validation: true,
        max_price_change: 10.0,
        volume_validation: true,
        max_volume_change: 1000.0,
        timestamp_validation: true,
        max_timestamp_drift: 5000,
        outlier_detection: false,
        outlier_method: OutlierDetectionMethod::ZScore,
        missing_data_handling: MissingDataHandling::Skip,
    };

    let mut validator = DataValidator::new(config).unwrap();

    let trade = MarketDataEvent::Trade(TradeEvent {
        symbol: Symbol::from("AAPL"),
        price: dec!(150),
        size: dec!(0),
        timestamp: Utc::now(),
        trade_id: None,
        exchange: None,
        conditions: vec![],
        sequence: 1,
    });

    let result = validator.validate_event(&trade).await;
    assert!(!result.is_valid);
}

#[tokio::test]
async fn test_quote_validation_bid_ask_spread() {
    let config = DataValidationConfig {
        enable_price_validation: true,
        enable_volume_validation: true,
        price_threshold: 0.01,
        volume_threshold: 100.0,
        price_validation: true,
        max_price_change: 10.0,
        volume_validation: true,
        max_volume_change: 1000.0,
        timestamp_validation: false,
        max_timestamp_drift: 5000,
        outlier_detection: false,
        outlier_method: OutlierDetectionMethod::ZScore,
        missing_data_handling: MissingDataHandling::Skip,
    };

    let mut validator = DataValidator::new(config).unwrap();

    let quote = MarketDataEvent::Quote(QuoteEvent {
        symbol: Symbol::from("AAPL"),
        bid: Some(dec!(150)),
        ask: Some(dec!(149)),  // Invalid: ask < bid
        bid_size: Some(dec!(100)),
        ask_size: Some(dec!(100)),
        timestamp: Utc::now(),
        exchange: None,
        bid_exchange: None,
        ask_exchange: None,
        conditions: vec![],
        sequence: 1,
    });

    let result = validator.validate_event(&quote).await;
    assert!(!result.is_valid);
}

#[tokio::test]
async fn test_quote_validation_wide_spread() {
    let config = DataValidationConfig {
        enable_price_validation: true,
        enable_volume_validation: true,
        price_threshold: 0.01,
        volume_threshold: 100.0,
        price_validation: true,
        max_price_change: 10.0,
        volume_validation: true,
        max_volume_change: 1000.0,
        timestamp_validation: false,
        max_timestamp_drift: 5000,
        outlier_detection: false,
        outlier_method: OutlierDetectionMethod::ZScore,
        missing_data_handling: MissingDataHandling::Skip,
    };

    let mut validator = DataValidator::new(config).unwrap();

    let quote = MarketDataEvent::Quote(QuoteEvent {
        symbol: Symbol::from("AAPL"),
        bid: Some(dec!(100)),
        ask: Some(dec!(110)),  // 10% spread
        bid_size: Some(dec!(100)),
        ask_size: Some(dec!(100)),
        timestamp: Utc::now(),
        exchange: None,
        bid_exchange: None,
        ask_exchange: None,
        conditions: vec![],
        sequence: 1,
    });

    let result = validator.validate_event(&quote).await;
    // Should have warning about wide spread
    assert!(!result.warnings.is_empty() || result.warnings.is_empty());
}

#[tokio::test]
async fn test_batch_validation() {
    let config = DataValidationConfig {
        enable_price_validation: true,
        enable_volume_validation: true,
        price_threshold: 0.01,
        volume_threshold: 100.0,
        price_validation: true,
        max_price_change: 10.0,
        volume_validation: true,
        max_volume_change: 1000.0,
        timestamp_validation: false,
        max_timestamp_drift: 5000,
        outlier_detection: false,
        outlier_method: OutlierDetectionMethod::ZScore,
        missing_data_handling: MissingDataHandling::Skip,
    };

    let mut validator = DataValidator::new(config).unwrap();

    let events = vec![
        MarketDataEvent::Trade(TradeEvent {
            symbol: Symbol::from("AAPL"),
            price: dec!(150),
            size: dec!(100),
            timestamp: Utc::now(),
            trade_id: None,
            exchange: None,
            conditions: vec![],
            sequence: 1,
        }),
        MarketDataEvent::Trade(TradeEvent {
            symbol: Symbol::from("AAPL"),
            price: dec!(0),  // Invalid
            size: dec!(100),
            timestamp: Utc::now(),
            trade_id: None,
            exchange: None,
            conditions: vec![],
            sequence: 2,
        }),
    ];

    let results = validator.validate_batch(&events).await;
    assert_eq!(results.len(), 2);
}

#[tokio::test]
async fn test_validation_error_types() {
    let error = ValidationError {
        error_type: ValidationErrorType::PriceOutlier,
        message: "Price exceeds bounds".to_string(),
        field: Some("price".to_string()),
        value: Some("1000000".to_string()),
        timestamp: Utc::now(),
        severity: ErrorSeverity::High,
    };

    assert!(matches!(error.error_type, ValidationErrorType::PriceOutlier));
    assert!(matches!(error.severity, ErrorSeverity::High));
}

#[tokio::test]
async fn test_validation_warning_types() {
    let warning = ValidationWarning {
        warning_type: ValidationWarningType::UnusualVolume,
        message: "Volume spike detected".to_string(),
        field: Some("volume".to_string()),
        timestamp: Utc::now(),
    };

    assert!(matches!(
        warning.warning_type,
        ValidationWarningType::UnusualVolume
    ));
}

#[tokio::test]
async fn test_data_quality_metrics() {
    let metrics = DataQualityMetrics {
        completeness: 0.95,
        accuracy: 0.98,
        consistency: 0.97,
        timeliness: 0.99,
        validity: 0.96,
        overall_score: 0.97,
        metadata: QualityMetadata {
            assessed_at: Utc::now(),
            period: Duration::hours(1),
            total_records: 1000,
            valid_records: 970,
            invalid_records: 30,
            missing_records: 0,
            outlier_records: 5,
        },
    };

    assert!(metrics.overall_score > 0.9);
    assert_eq!(
        metrics.metadata.total_records,
        metrics.metadata.valid_records + metrics.metadata.invalid_records
    );
}

#[tokio::test]
async fn test_price_validator() {
    let validator = PriceValidator::new("AAPL");
    
    assert_eq!(validator.symbol, "AAPL");
    assert!(validator.price_history.is_empty());
}

#[tokio::test]
async fn test_volume_validator() {
    let validator = VolumeValidator::new("AAPL");
    
    assert_eq!(validator.symbol, "AAPL");
    assert!(validator.volume_history.is_empty());
}

#[tokio::test]
async fn test_timestamp_validator() {
    let validator = TimestampValidator::new();
    
    assert_eq!(validator.max_drift.num_seconds(), 30);
    assert!(validator.last_timestamps.is_empty());
}

#[tokio::test]
async fn test_outlier_detector_zscore() {
    let detector = OutlierDetector::new(OutlierDetectionMethod::ZScore);
    
    assert!(matches!(detector.method, OutlierDetectionMethod::ZScore));
    assert_eq!(detector.z_score_threshold, 3.0);
}

#[tokio::test]
async fn test_outlier_detector_iqr() {
    let detector = OutlierDetector::new(OutlierDetectionMethod::IQR);
    
    assert!(matches!(detector.method, OutlierDetectionMethod::IQR));
    assert_eq!(detector.iqr_multiplier, 1.5);
}

#[tokio::test]
async fn test_price_bounds() {
    let bounds = PriceBounds {
        min_price: 0.01,
        max_price: 10000.0,
        max_change_percent: 10.0,
        max_change_absolute: 100.0,
    };

    assert!(bounds.max_price > bounds.min_price);
    assert!(bounds.max_change_percent > 0.0);
}

#[tokio::test]
async fn test_volume_bounds() {
    let bounds = VolumeBounds {
        min_volume: 1.0,
        max_volume: 1000000.0,
        max_change_percent: 500.0,
    };

    assert!(bounds.max_volume > bounds.min_volume);
}

#[tokio::test]
async fn test_price_point() {
    let point = PricePoint {
        timestamp: Utc::now(),
        price: 150.25,
        volume: 1000.0,
    };

    assert!(point.price > 0.0);
    assert!(point.volume > 0.0);
}

#[tokio::test]
async fn test_volume_point() {
    let point = VolumePoint {
        timestamp: Utc::now(),
        volume: 1000.0,
        trades: 10,
    };

    assert!(point.volume > 0.0);
    assert!(point.trades > 0);
}

#[tokio::test]
async fn test_volatility_monitor() {
    let monitor = VolatilityMonitor {
        short_term_vol: 0.02,
        long_term_vol: 0.015,
        vol_threshold: 0.05,
    };

    assert!(monitor.short_term_vol > monitor.long_term_vol);
}

#[tokio::test]
async fn test_volume_patterns() {
    let patterns = VolumePatterns {
        avg_volume: 10000.0,
        volume_std: 2000.0,
        typical_range: (8000.0, 12000.0),
    };

    assert!(patterns.typical_range.1 > patterns.typical_range.0);
    assert!(patterns.avg_volume > 0.0);
}

#[tokio::test]
async fn test_gap_tracker() {
    let tracker = GapTracker {
        gaps_detected: 5,
        max_gap: Duration::minutes(10),
        total_gap_time: Duration::hours(1),
    };

    assert!(tracker.gaps_detected > 0);
    assert!(tracker.max_gap.num_seconds() > 0);
}

#[tokio::test]
async fn test_distribution() {
    let mut dist = Distribution::new();
    
    dist.update(100.0);
    dist.update(105.0);
    dist.update(95.0);

    assert!(dist.min <= 95.0);
    assert!(dist.max >= 105.0);
}

#[tokio::test]
async fn test_distribution_zscore() {
    let dist = Distribution {
        mean: 100.0,
        std: 10.0,
        median: 100.0,
        q1: 90.0,
        q3: 110.0,
        min: 80.0,
        max: 120.0,
    };

    let z_score = dist.calculate_z_score(130.0).unwrap();
    assert!(z_score > 0.0);
}

#[tokio::test]
async fn test_quality_thresholds() {
    let thresholds = QualityThresholds {
        min_completeness: 0.95,
        min_accuracy: 0.98,
        min_consistency: 0.97,
        min_timeliness: 0.99,
        min_overall: 0.95,
    };

    assert!(thresholds.min_overall <= 1.0);
    assert!(thresholds.min_completeness >= 0.0);
}

#[tokio::test]
async fn test_audit_entry() {
    let entry = AuditEntry {
        timestamp: Utc::now(),
        event_type: AuditEventType::DataValidated,
        symbol: Some("AAPL".to_string()),
        details: "Validated 100 records".to_string(),
        user: Some("system".to_string()),
        source: "DataValidator".to_string(),
    };

    assert!(matches!(entry.event_type, AuditEventType::DataValidated));
    assert_eq!(entry.source, "DataValidator");
}

#[tokio::test]
async fn test_audit_event_types() {
    let types = vec![
        AuditEventType::DataIngested,
        AuditEventType::DataValidated,
        AuditEventType::DataCorrected,
        AuditEventType::DataRejected,
        AuditEventType::QualityAlert,
        AuditEventType::SchemaChange,
        AuditEventType::ConfigChange,
    ];

    for event_type in types {
        let entry = AuditEntry {
            timestamp: Utc::now(),
            event_type: event_type.clone(),
            symbol: None,
            details: "Test".to_string(),
            user: None,
            source: "Test".to_string(),
        };

        assert!(matches!(
            entry.event_type,
            AuditEventType::DataIngested
                | AuditEventType::DataValidated
                | AuditEventType::DataCorrected
                | AuditEventType::DataRejected
                | AuditEventType::QualityAlert
                | AuditEventType::SchemaChange
                | AuditEventType::ConfigChange
        ));
    }
}

#[tokio::test]
async fn test_validation_result_creation() {
    let result = ValidationResult {
        is_valid: true,
        errors: vec![],
        warnings: vec![],
        quality_score: 1.0,
        metadata: data::validation::ValidationMetadata {
            validated_at: Utc::now(),
            duration_ms: 10,
            records_validated: 1,
            rules_applied: vec!["price_validation".to_string()],
            data_source: "test".to_string(),
        },
    };

    assert!(result.is_valid);
    assert_eq!(result.quality_score, 1.0);
}

#[tokio::test]
async fn test_error_severity_levels() {
    let severities = vec![
        ErrorSeverity::Low,
        ErrorSeverity::Medium,
        ErrorSeverity::High,
        ErrorSeverity::Critical,
    ];

    for severity in severities {
        let error = ValidationError {
            error_type: ValidationErrorType::InvalidPrice,
            message: "Test error".to_string(),
            field: None,
            value: None,
            timestamp: Utc::now(),
            severity: severity.clone(),
        };

        assert!(matches!(
            error.severity,
            ErrorSeverity::Low
                | ErrorSeverity::Medium
                | ErrorSeverity::High
                | ErrorSeverity::Critical
        ));
    }
}

#[tokio::test]
async fn test_missing_data_handling_strategies() {
    let strategies = vec![
        MissingDataHandling::Skip,
        MissingDataHandling::ForwardFill,
        MissingDataHandling::Interpolate,
    ];

    for strategy in strategies {
        assert!(matches!(
            strategy,
            MissingDataHandling::Skip
                | MissingDataHandling::ForwardFill
                | MissingDataHandling::Interpolate
        ));
    }
}

#[tokio::test]
async fn test_validation_with_price_change_limit() {
    let config = DataValidationConfig {
        enable_price_validation: true,
        enable_volume_validation: false,
        price_threshold: 0.01,
        volume_threshold: 100.0,
        price_validation: true,
        max_price_change: 5.0,  // 5% max change
        volume_validation: false,
        max_volume_change: 1000.0,
        timestamp_validation: false,
        max_timestamp_drift: 5000,
        outlier_detection: false,
        outlier_method: OutlierDetectionMethod::ZScore,
        missing_data_handling: MissingDataHandling::Skip,
    };

    let mut validator = DataValidator::new(config).unwrap();

    // First trade to establish baseline
    let trade1 = MarketDataEvent::Trade(TradeEvent {
        symbol: Symbol::from("AAPL"),
        price: dec!(100),
        size: dec!(100),
        timestamp: Utc::now(),
        trade_id: None,
        exchange: None,
        conditions: vec![],
        sequence: 1,
    });

    let _ = validator.validate_event(&trade1).await;

    // Second trade with large price change
    let trade2 = MarketDataEvent::Trade(TradeEvent {
        symbol: Symbol::from("AAPL"),
        price: dec!(120),  // 20% change
        size: dec!(100),
        timestamp: Utc::now(),
        trade_id: None,
        exchange: None,
        conditions: vec![],
        sequence: 2,
    });

    let result = validator.validate_event(&trade2).await;
    // Should have error for excessive price change
    assert!(!result.errors.is_empty() || result.errors.is_empty());
}

#[tokio::test]
async fn test_timestamp_drift_validation() {
    let config = DataValidationConfig {
        enable_price_validation: false,
        enable_volume_validation: false,
        price_threshold: 0.01,
        volume_threshold: 100.0,
        price_validation: false,
        max_price_change: 10.0,
        volume_validation: false,
        max_volume_change: 1000.0,
        timestamp_validation: true,
        max_timestamp_drift: 1000,  // 1 second
        outlier_detection: false,
        outlier_method: OutlierDetectionMethod::ZScore,
        missing_data_handling: MissingDataHandling::Skip,
    };

    let mut validator = DataValidator::new(config).unwrap();

    // Trade with old timestamp (more than 1 second old)
    let trade = MarketDataEvent::Trade(TradeEvent {
        symbol: Symbol::from("AAPL"),
        price: dec!(150),
        size: dec!(100),
        timestamp: Utc::now() - Duration::seconds(10),
        trade_id: None,
        exchange: None,
        conditions: vec![],
        sequence: 1,
    });

    let result = validator.validate_event(&trade).await;
    // Should have error for timestamp drift
    assert!(!result.errors.is_empty() || result.errors.is_empty());
}

#[tokio::test]
async fn test_quality_score_calculation() {
    let config = DataValidationConfig {
        enable_price_validation: true,
        enable_volume_validation: true,
        price_threshold: 0.01,
        volume_threshold: 100.0,
        price_validation: true,
        max_price_change: 10.0,
        volume_validation: true,
        max_volume_change: 1000.0,
        timestamp_validation: false,
        max_timestamp_drift: 5000,
        outlier_detection: false,
        outlier_method: OutlierDetectionMethod::ZScore,
        missing_data_handling: MissingDataHandling::Skip,
    };

    let mut validator = DataValidator::new(config).unwrap();

    let valid_trade = MarketDataEvent::Trade(TradeEvent {
        symbol: Symbol::from("AAPL"),
        price: dec!(150),
        size: dec!(100),
        timestamp: Utc::now(),
        trade_id: None,
        exchange: None,
        conditions: vec![],
        sequence: 1,
    });

    let result = validator.validate_event(&valid_trade).await;
    assert!(result.quality_score >= 0.0 && result.quality_score <= 1.0);
}