foxhunt/crates/data/src/validation.rs

//! Data Validation and Quality Control for Training Data
//!
//! Comprehensive data validation system for financial time-series data including:
//! - Price and volume validation with outlier detection
//! - Timestamp validation and gap detection
//! - Data completeness and consistency checks
//! - Real-time quality monitoring and alerting
//! - Statistical anomaly detection
//! - Data lineage and audit trails

use crate::error::Result;
use chrono::{DateTime, Duration, Utc};
use common::MarketDataEvent;
use common::{QuoteEvent, TradeEvent};
use config::data_config::{DataValidationConfig, OutlierDetectionMethod};
use num_traits::ToPrimitive;
use rust_decimal::Decimal;
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, VecDeque};
use tracing::info;

/// Data validation result
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ValidationResult {
    /// Validation passed
    pub is_valid: bool,
    /// Validation errors
    pub errors: Vec<ValidationError>,
    /// Validation warnings
    pub warnings: Vec<ValidationWarning>,
    /// Quality score (0.0 to 1.0)
    pub quality_score: f64,
    /// Validation metadata
    pub metadata: ValidationMetadata,
}

/// Validation error
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ValidationError {
    /// Error type
    pub error_type: ValidationErrorType,
    /// Error message
    pub message: String,
    /// Affected field
    pub field: Option<String>,
    /// Error value
    pub value: Option<String>,
    /// Timestamp when error occurred
    pub timestamp: DateTime<Utc>,
    /// Severity level
    pub severity: ErrorSeverity,
}

/// Validation warning
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ValidationWarning {
    /// Warning type
    pub warning_type: ValidationWarningType,
    /// Warning message
    pub message: String,
    /// Affected field
    pub field: Option<String>,
    /// Timestamp when warning occurred
    pub timestamp: DateTime<Utc>,
}

/// Validation metadata
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ValidationMetadata {
    /// Validation timestamp
    pub validated_at: DateTime<Utc>,
    /// Validation duration (milliseconds)
    pub duration_ms: u64,
    /// Number of records validated
    pub records_validated: u64,
    /// Validation rules applied
    pub rules_applied: Vec<String>,
    /// Data source
    pub data_source: String,
}

/// Validation error types
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum ValidationErrorType {
    PriceOutlier,
    VolumeOutlier,
    InvalidPrice,
    InvalidVolume,
    TimestampGap,
    TimestampDrift,
    DuplicateRecord,
    MissingField,
    InvalidFormat,
    BusinessLogicViolation,
    ConsistencyViolation,
}

/// Validation warning types
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum ValidationWarningType {
    UnusualVolume,
    UnusualPrice,
    HighVolatility,
    LowLiquidity,
    StaleTrade,
    WideBidAsk,
    InfrequentUpdates,
}

// ErrorSeverity is imported from common::error (re-exported below)
pub use common::error::ErrorSeverity;

/// Data quality metrics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DataQualityMetrics {
    /// Completeness score (0.0 to 1.0)
    pub completeness: f64,
    /// Accuracy score (0.0 to 1.0)
    pub accuracy: f64,
    /// Consistency score (0.0 to 1.0)
    pub consistency: f64,
    /// Timeliness score (0.0 to 1.0)
    pub timeliness: f64,
    /// Validity score (0.0 to 1.0)
    pub validity: f64,
    /// Overall quality score (0.0 to 1.0)
    pub overall_score: f64,
    /// Quality metadata
    pub metadata: QualityMetadata,
}

/// Quality metadata
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct QualityMetadata {
    /// Assessment timestamp
    pub assessed_at: DateTime<Utc>,
    /// Assessment period
    pub period: Duration,
    /// Total records assessed
    pub total_records: u64,
    /// Valid records
    pub valid_records: u64,
    /// Invalid records
    pub invalid_records: u64,
    /// Missing records
    pub missing_records: u64,
    /// Outlier records
    pub outlier_records: u64,
}

/// Data validator with configurable rules
pub struct DataValidator {
    config: DataValidationConfig,
    price_validators: HashMap<String, PriceValidator>,
    volume_validators: HashMap<String, VolumeValidator>,
    timestamp_validator: TimestampValidator,
    outlier_detector: OutlierDetector,
    quality_monitor: QualityMonitor,
    audit_trail: AuditTrail,
}

/// Price validation for individual symbols
pub struct PriceValidator {
    symbol: String,
    price_history: VecDeque<PricePoint>,
    price_bounds: PriceBounds,
    volatility_monitor: VolatilityMonitor,
}

/// Volume validation for individual symbols
pub struct VolumeValidator {
    symbol: String,
    volume_history: VecDeque<VolumePoint>,
    volume_bounds: VolumeBounds,
    volume_patterns: VolumePatterns,
}

/// Timestamp validation across all data
pub struct TimestampValidator {
    expected_frequency: Duration,
    max_gap: Duration,
    max_drift: Duration,
    last_timestamps: HashMap<String, DateTime<Utc>>,
    gap_tracker: GapTracker,
}

/// Outlier detection engine
pub struct OutlierDetector {
    method: OutlierDetectionMethod,
    z_score_threshold: f64,
    iqr_multiplier: f64,
    isolation_forest: Option<IsolationForest>,
    historical_distributions: HashMap<String, Distribution>,
}

/// Quality monitoring system
pub struct QualityMonitor {
    quality_history: VecDeque<QualitySnapshot>,
    alert_thresholds: QualityThresholds,
    trend_analyzer: TrendAnalyzer,
}

/// Audit trail for data lineage
pub struct AuditTrail {
    entries: VecDeque<AuditEntry>,
    max_entries: usize,
}

/// Price bounds for validation
#[derive(Debug, Clone)]
pub struct PriceBounds {
    pub min_price: f64,
    pub max_price: f64,
    pub max_change_percent: f64,
    pub max_change_absolute: f64,
}

/// Volume bounds for validation
#[derive(Debug, Clone)]
pub struct VolumeBounds {
    pub min_volume: f64,
    pub max_volume: f64,
    pub max_change_percent: f64,
}

/// Price point for validation
#[derive(Debug, Clone)]
pub struct PricePoint {
    pub timestamp: DateTime<Utc>,
    pub price: f64,
    pub volume: f64,
}

/// Volume point for validation
#[derive(Debug, Clone)]
pub struct VolumePoint {
    pub timestamp: DateTime<Utc>,
    pub volume: f64,
    pub trades: u64,
}

/// Volatility monitoring
#[derive(Debug, Clone)]
pub struct VolatilityMonitor {
    pub short_term_vol: f64,
    pub long_term_vol: f64,
    pub vol_threshold: f64,
}

/// Volume patterns tracking
#[derive(Debug, Clone)]
pub struct VolumePatterns {
    pub avg_volume: f64,
    pub volume_std: f64,
    pub typical_range: (f64, f64),
}

/// Gap tracking for timestamps
#[derive(Debug, Clone)]
pub struct GapTracker {
    pub gaps_detected: u64,
    pub max_gap: Duration,
    pub total_gap_time: Duration,
}

/// Simplified isolation forest for outlier detection
pub struct IsolationForest {
    trees: Vec<IsolationTree>,
    contamination: f64,
}

/// Isolation tree node
pub struct IsolationTree {
    threshold: f64,
    feature: usize,
    left: Option<Box<IsolationTree>>,
    right: Option<Box<IsolationTree>>,
}

/// Statistical distribution for outlier detection
#[derive(Debug, Clone)]
pub struct Distribution {
    pub mean: f64,
    pub std: f64,
    pub median: f64,
    pub q1: f64,
    pub q3: f64,
    pub min: f64,
    pub max: f64,
}

/// Quality snapshot for monitoring
#[derive(Debug, Clone)]
pub struct QualitySnapshot {
    pub timestamp: DateTime<Utc>,
    pub metrics: DataQualityMetrics,
    pub symbol: String,
}

/// Quality alert thresholds
#[derive(Debug, Clone)]
pub struct QualityThresholds {
    pub min_completeness: f64,
    pub min_accuracy: f64,
    pub min_consistency: f64,
    pub min_timeliness: f64,
    pub min_overall: f64,
}

/// Trend analysis for quality metrics
pub struct TrendAnalyzer {
    window_size: usize,
    trend_threshold: f64,
}

/// Audit entry for data lineage
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AuditEntry {
    pub timestamp: DateTime<Utc>,
    pub event_type: AuditEventType,
    pub symbol: Option<String>,
    pub details: String,
    pub user: Option<String>,
    pub source: String,
}

/// Audit event types
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum AuditEventType {
    DataIngested,
    DataValidated,
    DataCorrected,
    DataRejected,
    QualityAlert,
    SchemaChange,
    ConfigChange,
}

impl DataValidator {
    /// Create new data validator
    pub fn new(config: DataValidationConfig) -> Result<Self> {
        Ok(Self {
            config: config.clone(),
            price_validators: HashMap::new(),
            volume_validators: HashMap::new(),
            timestamp_validator: TimestampValidator::new(),
            outlier_detector: OutlierDetector::new(config.outlier_method),
            quality_monitor: QualityMonitor::new(),
            audit_trail: AuditTrail::new(10000),
        })
    }

    /// Validate a single market data event
    pub async fn validate_event(&mut self, event: &MarketDataEvent) -> ValidationResult {
        let start_time = std::time::Instant::now();
        let mut errors = Vec::new();
        let mut warnings = Vec::new();

        // Validate based on event type
        match event {
            MarketDataEvent::Trade(trade) => {
                self.validate_trade(trade, &mut errors, &mut warnings).await;
            },
            MarketDataEvent::Quote(quote) => {
                self.validate_quote(quote, &mut errors, &mut warnings).await;
            },
            _ => {
                // Handle other event types
            },
        }

        // Calculate quality score
        let quality_score = self.calculate_quality_score(&errors, &warnings);

        // Record audit entry
        self.audit_trail.record(AuditEntry {
            timestamp: Utc::now(),
            event_type: AuditEventType::DataValidated,
            symbol: Some(event.symbol().to_string()),
            details: format!(
                "Validated {:?} with {} errors, {} warnings",
                std::mem::discriminant(event),
                errors.len(),
                warnings.len()
            ),
            user: None,
            source: "DataValidator".to_string(),
        });

        ValidationResult {
            is_valid: errors.is_empty(),
            errors,
            warnings,
            quality_score,
            metadata: ValidationMetadata {
                validated_at: Utc::now(),
                duration_ms: start_time.elapsed().as_millis() as u64,
                records_validated: 1,
                rules_applied: self.get_applied_rules(),
                data_source: "market_data".to_owned(),
            },
        }
    }

    /// Validate a batch of market data events
    pub async fn validate_batch(&mut self, events: &[MarketDataEvent]) -> Vec<ValidationResult> {
        let mut results = Vec::new();

        for event in events {
            let result = self.validate_event(event).await;
            results.push(result);
        }

        // Update quality metrics
        self.update_quality_metrics(&results);

        results
    }

    /// Validate trade data
    async fn validate_trade(
        &mut self,
        trade: &TradeEvent,
        errors: &mut Vec<ValidationError>,
        warnings: &mut Vec<ValidationWarning>,
    ) {
        // Price validation
        if self.config.price_validation {
            self.validate_trade_price(trade, errors, warnings);
        }

        // Volume validation
        if self.config.volume_validation {
            self.validate_trade_volume(trade, errors, warnings);
        }

        // Timestamp validation
        if self.config.timestamp_validation {
            self.validate_timestamp(&trade.symbol, trade.timestamp, errors, warnings);
        }

        // Outlier detection
        if self.config.outlier_detection {
            self.detect_trade_outliers(trade, errors, warnings);
        }
    }

    /// Validate quote data
    async fn validate_quote(
        &mut self,
        quote: &QuoteEvent,
        errors: &mut Vec<ValidationError>,
        warnings: &mut Vec<ValidationWarning>,
    ) {
        // Bid/ask validation
        if let (Some(bid), Some(ask)) = (quote.bid, quote.ask) {
            if bid >= ask {
                errors.push(ValidationError {
                    error_type: ValidationErrorType::BusinessLogicViolation,
                    message: format!("Bid price ({}) >= Ask price ({})", bid, ask),
                    field: Some("bid_ask".to_owned()),
                    value: Some(format!("bid:{}, ask:{}", bid, ask)),
                    timestamp: Utc::now(),
                    severity: ErrorSeverity::High,
                });
            }

            let spread = ask - bid;
            let mid_price = (bid + ask) / Decimal::from(2);
            let spread_pct = spread / mid_price;

            // Wide spread warning
            if spread_pct > Decimal::try_from(0.01).unwrap_or_default() {
                // 1% spread
                warnings.push(ValidationWarning {
                    warning_type: ValidationWarningType::WideBidAsk,
                    message: format!(
                        "Wide bid-ask spread: {:.4}%",
                        spread_pct * Decimal::from(100)
                    ),
                    field: Some("spread".to_owned()),
                    timestamp: Utc::now(),
                });
            }
        }

        // Size validation
        if let (Some(bid_size), Some(ask_size)) = (quote.bid_size, quote.ask_size) {
            if bid_size <= Decimal::ZERO || ask_size <= Decimal::ZERO {
                warnings.push(ValidationWarning {
                    warning_type: ValidationWarningType::LowLiquidity,
                    message: "Zero or negative quote size".to_string(),
                    field: Some("size".to_owned()),
                    timestamp: Utc::now(),
                });
            }
        }
    }

    /// Validate trade price
    fn validate_trade_price(
        &mut self,
        trade: &TradeEvent,
        errors: &mut Vec<ValidationError>,
        _warnings: &mut Vec<ValidationWarning>,
    ) {
        let price = ToPrimitive::to_f64(&trade.price).unwrap_or(0.0);

        // Basic price validation
        if price <= 0.0 {
            errors.push(ValidationError {
                error_type: ValidationErrorType::InvalidPrice,
                message: format!("Invalid price: {}", price),
                field: Some("price".to_owned()),
                value: Some(price.to_string()),
                timestamp: Utc::now(),
                severity: ErrorSeverity::Critical,
            });
            return;
        }

        // Get or create price validator for symbol
        let validator = self
            .price_validators
            .entry(trade.symbol.clone())
            .or_insert_with(|| PriceValidator::new(&trade.symbol));

        // Check price change limits
        if let Some(last_price) = validator.price_history.back() {
            let price_change = (price - last_price.price).abs();
            let price_change_pct = price_change / last_price.price;

            if price_change_pct > self.config.max_price_change / 100.0 {
                errors.push(ValidationError {
                    error_type: ValidationErrorType::PriceOutlier,
                    message: format!(
                        "Price change exceeds limit: {:.2}%",
                        price_change_pct * 100.0
                    ),
                    field: Some("price".to_owned()),
                    value: Some(price.to_string()),
                    timestamp: Utc::now(),
                    severity: ErrorSeverity::Medium,
                });
            }
        }

        // Update price history
        validator.price_history.push_back(PricePoint {
            timestamp: trade.timestamp,
            price,
            volume: ToPrimitive::to_f64(&trade.size).unwrap_or(0.0),
        });

        // Keep limited history
        while validator.price_history.len() > 1000 {
            validator.price_history.pop_front();
        }
    }

    /// Validate trade volume
    fn validate_trade_volume(
        &mut self,
        trade: &TradeEvent,
        errors: &mut Vec<ValidationError>,
        warnings: &mut Vec<ValidationWarning>,
    ) {
        let volume = ToPrimitive::to_f64(&trade.size).unwrap_or(0.0);

        // Basic volume validation
        if volume <= 0.0 {
            errors.push(ValidationError {
                error_type: ValidationErrorType::InvalidVolume,
                message: format!("Invalid volume: {}", volume),
                field: Some("volume".to_owned()),
                value: Some(volume.to_string()),
                timestamp: Utc::now(),
                severity: ErrorSeverity::High,
            });
            return;
        }

        // Get or create volume validator for symbol
        let validator = self
            .volume_validators
            .entry(trade.symbol.clone())
            .or_insert_with(|| VolumeValidator::new(&trade.symbol));

        // Check volume change limits
        if let Some(last_volume) = validator.volume_history.back() {
            let volume_change_pct = (volume - last_volume.volume).abs() / last_volume.volume;

            if volume_change_pct > self.config.max_volume_change / 100.0 {
                warnings.push(ValidationWarning {
                    warning_type: ValidationWarningType::UnusualVolume,
                    message: format!(
                        "Volume change exceeds typical range: {:.2}%",
                        volume_change_pct * 100.0
                    ),
                    field: Some("volume".to_owned()),
                    timestamp: Utc::now(),
                });
            }
        }

        // Update volume history
        validator.volume_history.push_back(VolumePoint {
            timestamp: trade.timestamp,
            volume,
            trades: 1,
        });

        // Keep limited history
        while validator.volume_history.len() > 1000 {
            validator.volume_history.pop_front();
        }
    }

    /// Validate timestamp
    fn validate_timestamp(
        &mut self,
        symbol: &str,
        timestamp: DateTime<Utc>,
        errors: &mut Vec<ValidationError>,
        warnings: &mut Vec<ValidationWarning>,
    ) {
        let now = Utc::now();

        // Check timestamp drift
        let drift = (now - timestamp).num_milliseconds().abs();
        if drift > self.config.max_timestamp_drift {
            errors.push(ValidationError {
                error_type: ValidationErrorType::TimestampDrift,
                message: format!("Timestamp drift exceeds limit: {}ms", drift),
                field: Some("timestamp".to_owned()),
                value: Some(timestamp.to_rfc3339()),
                timestamp: Utc::now(),
                severity: ErrorSeverity::Medium,
            });
        }

        // Check for gaps
        if let Some(&last_timestamp) = self.timestamp_validator.last_timestamps.get(symbol) {
            let gap = timestamp - last_timestamp;
            if gap > self.timestamp_validator.max_gap {
                warnings.push(ValidationWarning {
                    warning_type: ValidationWarningType::InfrequentUpdates,
                    message: format!("Data gap detected: {}s", gap.num_seconds()),
                    field: Some("timestamp".to_owned()),
                    timestamp: Utc::now(),
                });
            }
        }

        // Update last timestamp
        self.timestamp_validator
            .last_timestamps
            .insert(symbol.to_owned(), timestamp);
    }

    /// Detect outliers in trade data
    fn detect_trade_outliers(
        &mut self,
        trade: &TradeEvent,
        _errors: &mut Vec<ValidationError>,
        warnings: &mut Vec<ValidationWarning>,
    ) {
        let price = ToPrimitive::to_f64(&trade.price).unwrap_or(0.0);
        let _volume = ToPrimitive::to_f64(&trade.size).unwrap_or(0.0);

        // Get or update distribution for symbol
        let distribution = self
            .outlier_detector
            .historical_distributions
            .entry(trade.symbol.clone())
            .or_insert_with(|| Distribution::new());

        // Check if price is an outlier
        if let Some(z_score) = distribution.calculate_z_score(price) {
            if z_score.abs() > self.outlier_detector.z_score_threshold {
                warnings.push(ValidationWarning {
                    warning_type: ValidationWarningType::UnusualPrice,
                    message: format!("Price outlier detected (z-score: {:.2})", z_score),
                    field: Some("price".to_owned()),
                    timestamp: Utc::now(),
                });
            }
        }

        // Update distribution
        distribution.update(price);
    }

    /// Calculate quality score based on errors and warnings
    fn calculate_quality_score(
        &self,
        errors: &[ValidationError],
        warnings: &[ValidationWarning],
    ) -> f64 {
        if errors.is_empty() && warnings.is_empty() {
            return 1.0;
        }

        let error_penalty = errors.len() as f64 * 0.2;
        let warning_penalty = warnings.len() as f64 * 0.1;
        let total_penalty = error_penalty + warning_penalty;

        (1.0 - total_penalty).max(0.0)
    }

    /// Get list of applied validation rules
    fn get_applied_rules(&self) -> Vec<String> {
        let mut rules = Vec::new();

        if self.config.price_validation {
            rules.push("price_validation".to_owned());
        }
        if self.config.volume_validation {
            rules.push("volume_validation".to_owned());
        }
        if self.config.timestamp_validation {
            rules.push("timestamp_validation".to_owned());
        }
        if self.config.outlier_detection {
            rules.push("outlier_detection".to_owned());
        }

        rules
    }

    /// Update quality metrics based on validation results
    fn update_quality_metrics(&mut self, results: &[ValidationResult]) {
        // Implementation would update quality monitoring
        let total_records = results.len() as f64;
        let valid_records = results.iter().filter(|r| r.is_valid).count() as f64;
        let accuracy = valid_records / total_records;

        info!(
            "Quality metrics updated: accuracy={:.2}%, records={}",
            accuracy * 100.0,
            total_records
        );
    }
}

impl PriceValidator {
    fn new(symbol: &str) -> Self {
        Self {
            symbol: symbol.to_owned(),
            price_history: VecDeque::new(),
            price_bounds: PriceBounds {
                min_price: 0.01,
                max_price: 1000000.0,
                max_change_percent: 10.0,
                max_change_absolute: 100.0,
            },
            volatility_monitor: VolatilityMonitor {
                short_term_vol: 0.0,
                long_term_vol: 0.0,
                vol_threshold: 0.5,
            },
        }
    }
}

impl VolumeValidator {
    fn new(symbol: &str) -> Self {
        Self {
            symbol: symbol.to_owned(),
            volume_history: VecDeque::new(),
            volume_bounds: VolumeBounds {
                min_volume: 1.0,
                max_volume: 1000000000.0,
                max_change_percent: 1000.0,
            },
            volume_patterns: VolumePatterns {
                avg_volume: 0.0,
                volume_std: 0.0,
                typical_range: (0.0, 0.0),
            },
        }
    }
}

impl TimestampValidator {
    fn new() -> Self {
        Self {
            expected_frequency: Duration::seconds(1),
            max_gap: Duration::minutes(5),
            max_drift: Duration::seconds(30),
            last_timestamps: HashMap::new(),
            gap_tracker: GapTracker {
                gaps_detected: 0,
                max_gap: Duration::zero(),
                total_gap_time: Duration::zero(),
            },
        }
    }
}

impl OutlierDetector {
    fn new(method: OutlierDetectionMethod) -> Self {
        Self {
            method,
            z_score_threshold: 3.0,
            iqr_multiplier: 1.5,
            isolation_forest: None,
            historical_distributions: HashMap::new(),
        }
    }
}

impl QualityMonitor {
    fn new() -> Self {
        Self {
            quality_history: VecDeque::new(),
            alert_thresholds: QualityThresholds {
                min_completeness: 0.95,
                min_accuracy: 0.98,
                min_consistency: 0.90,
                min_timeliness: 0.95,
                min_overall: 0.90,
            },
            trend_analyzer: TrendAnalyzer {
                window_size: 100,
                trend_threshold: 0.05,
            },
        }
    }
}

impl AuditTrail {
    fn new(max_entries: usize) -> Self {
        Self {
            entries: VecDeque::new(),
            max_entries,
        }
    }

    fn record(&mut self, entry: AuditEntry) {
        self.entries.push_back(entry);
        while self.entries.len() > self.max_entries {
            self.entries.pop_front();
        }
    }
}

impl Distribution {
    fn new() -> Self {
        Self {
            mean: 0.0,
            std: 0.0,
            median: 0.0,
            q1: 0.0,
            q3: 0.0,
            min: f64::MAX,
            max: f64::MIN,
        }
    }

    fn update(&mut self, value: f64) {
        // Simplified update - in practice would use incremental statistics
        self.min = self.min.min(value);
        self.max = self.max.max(value);
        // Update other statistics...
    }

    fn calculate_z_score(&self, value: f64) -> Option<f64> {
        if self.std == 0.0 {
            return None;
        }
        Some((value - self.mean) / self.std)
    }
}

#[cfg(test)]
#[allow(clippy::assertions_on_result_states)]
mod tests {
    use super::*;
    use config::MissingDataHandling;

    #[test]
    fn test_validation_result_creation() {
        let result = ValidationResult {
            is_valid: true,
            errors: vec![],
            warnings: vec![],
            quality_score: 1.0,
            metadata: ValidationMetadata {
                validated_at: Utc::now(),
                duration_ms: 10,
                records_validated: 1,
                rules_applied: vec!["price_validation".to_owned()],
                data_source: "test".to_string(),
            },
        };

        assert!(result.is_valid);
        assert_eq!(result.quality_score, 1.0);
    }

    #[tokio::test]
    async fn test_data_validator_creation() {
        let config = DataValidationConfig {
            enable_price_validation: true,
            enable_volume_validation: true,
            price_threshold: 0.01,
            volume_threshold: 100.0,
            price_validation: true,
            max_price_change: 10.0,
            volume_validation: true,
            max_volume_change: 1000.0,
            timestamp_validation: true,
            max_timestamp_drift: 5000,
            outlier_detection: true,
            outlier_method: OutlierDetectionMethod::ZScore,
            missing_data_handling: MissingDataHandling::Skip,
        };

        let validator = DataValidator::new(config);
        assert!(validator.is_ok());
    }

    #[test]
    fn test_validation_error_creation() {
        let error = ValidationError {
            error_type: ValidationErrorType::PriceOutlier,
            severity: ErrorSeverity::High,
            message: "Price exceeds bounds".to_string(),
            field: Some("price".to_owned()),
            value: Some("10000.0".to_string()),
            timestamp: Utc::now(),
        };

        assert!(matches!(
            error.error_type,
            ValidationErrorType::PriceOutlier
        ));
        assert!(matches!(error.severity, ErrorSeverity::High));
        assert_eq!(error.field, Some("price".to_owned()));
    }

    #[test]
    fn test_validation_warning_creation() {
        let warning = ValidationWarning {
            warning_type: ValidationWarningType::UnusualVolume,
            message: "Volume spike detected".to_string(),
            field: Some("volume".to_owned()),
            timestamp: Utc::now(),
        };

        assert!(matches!(
            warning.warning_type,
            ValidationWarningType::UnusualVolume
        ));
        assert_eq!(warning.field, Some("volume".to_owned()));
    }

    #[test]
    fn test_data_quality_metrics() {
        let metrics = DataQualityMetrics {
            completeness: 0.95,
            accuracy: 0.98,
            consistency: 0.97,
            timeliness: 0.99,
            validity: 0.96,
            overall_score: 0.97,
            metadata: QualityMetadata {
                assessed_at: Utc::now(),
                period: Duration::hours(1),
                total_records: 1000,
                valid_records: 950,
                invalid_records: 50,
                missing_records: 0,
                outlier_records: 5,
            },
        };

        assert_eq!(metrics.metadata.total_records, 1000);
        assert_eq!(metrics.metadata.valid_records, 950);
        assert_eq!(metrics.completeness, 0.95);
        assert!(metrics.overall_score > 0.9);
    }

    #[test]
    fn test_price_bounds() {
        let bounds = PriceBounds {
            min_price: 0.01,
            max_price: 10000.0,
            max_change_percent: 10.0,
            max_change_absolute: 100.0,
        };

        assert!(bounds.max_price > bounds.min_price);
        assert!(bounds.max_change_percent > 0.0);
    }

    #[test]
    fn test_volume_bounds() {
        let bounds = VolumeBounds {
            min_volume: 1.0,
            max_volume: 1000000.0,
            max_change_percent: 500.0,
        };

        assert!(bounds.max_volume > bounds.min_volume);
        assert!(bounds.max_change_percent > 0.0);
    }

    #[test]
    fn test_price_point_validation() {
        let point = PricePoint {
            timestamp: Utc::now(),
            price: 100.0,
            volume: 1000.0,
        };

        assert!(point.price > 0.0);
        assert!(point.volume >= 0.0);
    }

    #[test]
    fn test_volume_point_validation() {
        let point = VolumePoint {
            timestamp: Utc::now(),
            volume: 1000.0,
            trades: 10,
        };

        assert!(point.volume > 0.0);
        assert!(point.trades > 0);
    }

    #[test]
    fn test_volatility_monitor() {
        let monitor = VolatilityMonitor {
            short_term_vol: 0.02,
            long_term_vol: 0.015,
            vol_threshold: 0.05,
        };

        assert!(monitor.short_term_vol > monitor.long_term_vol);
        assert!(monitor.vol_threshold > 0.0);
    }

    #[test]
    fn test_gap_tracker() {
        let tracker = GapTracker {
            gaps_detected: 5,
            max_gap: Duration::minutes(10),
            total_gap_time: Duration::hours(1),
        };

        assert!(tracker.gaps_detected > 0);
        assert!(tracker.max_gap.num_seconds() > 0);
    }

    #[test]
    fn test_quality_thresholds() {
        let thresholds = QualityThresholds {
            min_completeness: 0.95,
            min_accuracy: 0.98,
            min_consistency: 0.97,
            min_timeliness: 0.99,
            min_overall: 0.95,
        };

        assert!(thresholds.min_overall <= 1.0);
        assert!(thresholds.min_completeness >= 0.0);
    }

    #[test]
    fn test_audit_entry() {
        let entry = AuditEntry {
            timestamp: Utc::now(),
            event_type: AuditEventType::DataValidated,
            symbol: Some("AAPL".to_string()),
            details: "Validated 1000 records".to_string(),
            user: Some("system".to_string()),
            source: "DataValidator".to_string(),
        };

        assert!(matches!(entry.event_type, AuditEventType::DataValidated));
        assert_eq!(entry.source, "DataValidator");
    }

    #[test]
    fn test_validation_result_scoring() {
        let mut result = ValidationResult {
            is_valid: true,
            quality_score: 1.0,
            errors: vec![],
            warnings: vec![],
            metadata: ValidationMetadata {
                validated_at: Utc::now(),
                duration_ms: 50,
                records_validated: 1,
                rules_applied: vec!["price_validation".to_owned()],
                data_source: "test".to_string(),
            },
        };

        // Add an error
        result.errors.push(ValidationError {
            error_type: ValidationErrorType::PriceOutlier,
            severity: ErrorSeverity::High,
            message: "Price error".to_string(),
            field: Some("price".to_owned()),
            value: None,
            timestamp: Utc::now(),
        });

        assert!(!result.errors.is_empty());
    }

    #[test]
    fn test_outlier_detection_methods() {
        assert!(matches!(
            OutlierDetectionMethod::ZScore,
            OutlierDetectionMethod::ZScore
        ));
        assert!(matches!(
            OutlierDetectionMethod::IQR,
            OutlierDetectionMethod::IQR
        ));
        assert!(matches!(
            OutlierDetectionMethod::IsolationForest,
            OutlierDetectionMethod::IsolationForest
        ));
    }

    #[test]
    fn test_missing_data_handling_strategies() {
        assert!(matches!(
            MissingDataHandling::Skip,
            MissingDataHandling::Skip
        ));
        assert!(matches!(
            MissingDataHandling::ForwardFill,
            MissingDataHandling::ForwardFill
        ));
        assert!(matches!(
            MissingDataHandling::Interpolate,
            MissingDataHandling::Interpolate
        ));
    }

    #[test]
    fn test_price_validator_bounds_check() {
        let validator = PriceValidator::new("AAPL");

        assert_eq!(validator.price_bounds.min_price, 0.01);
        assert_eq!(validator.price_bounds.max_price, 1000000.0);
    }

    #[test]
    fn test_volume_validator_bounds_check() {
        let validator = VolumeValidator::new("AAPL");

        assert_eq!(validator.volume_bounds.min_volume, 1.0);
        assert!(validator.volume_history.is_empty());
    }

    #[test]
    fn test_timestamp_validator_drift_check() {
        let validator = TimestampValidator::new();

        assert_eq!(validator.max_drift.num_seconds(), 30);
        assert!(validator.last_timestamps.is_empty());
    }

    #[test]
    fn test_outlier_detector_config() {
        let detector = OutlierDetector::new(OutlierDetectionMethod::ZScore);

        assert!(matches!(detector.method, OutlierDetectionMethod::ZScore));
        assert_eq!(detector.z_score_threshold, 3.0);
        assert!(detector.historical_distributions.is_empty());
    }

    #[test]
    fn test_quality_monitor_snapshot() {
        let snapshot = QualitySnapshot {
            timestamp: Utc::now(),
            metrics: DataQualityMetrics {
                completeness: 0.98,
                accuracy: 0.99,
                consistency: 0.98,
                timeliness: 0.99,
                validity: 0.97,
                overall_score: 0.985,
                metadata: QualityMetadata {
                    assessed_at: Utc::now(),
                    period: Duration::hours(1),
                    total_records: 1000,
                    valid_records: 980,
                    invalid_records: 20,
                    missing_records: 0,
                    outlier_records: 5,
                },
            },
            symbol: "AAPL".to_string(),
        };

        assert_eq!(snapshot.symbol, "AAPL");
        assert!(snapshot.metrics.overall_score > 0.98);
    }
}