foxhunt/services/data_acquisition_service/src/validator.rs

//! Data validation module
//!
//! Validates downloaded data for quality issues:
//! - Missing values
//! - Price anomalies
//! - Volume outliers
//! - Timestamp gaps
//! - Schema conformance

use crate::error::{AcquisitionError, AcquisitionResult};
use std::path::Path;

/// Validation configuration
#[derive(Debug, Clone)]
pub struct ValidationConfig {
    pub max_price_change_percent: f64,
    pub max_volume_std_dev: f64,
    pub max_timestamp_gap_seconds: i64,
    pub min_bars_required: usize,
}

impl Default for ValidationConfig {
    fn default() -> Self {
        Self {
            max_price_change_percent: 10.0,
            max_volume_std_dev: 5.0,
            max_timestamp_gap_seconds: 3600,
            min_bars_required: 100,
        }
    }
}

/// Validation result
#[derive(Debug, Clone)]
pub struct ValidationResult {
    pub is_valid: bool,
    pub records_count: u64,
    pub issues_found: Vec<ValidationIssue>,
    pub quality_score: f64,
}

/// Validation issue
#[derive(Debug, Clone)]
pub struct ValidationIssue {
    pub severity: IssueSeverity,
    pub category: String,
    pub description: String,
    pub record_index: Option<usize>,
}

/// Issue severity
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum IssueSeverity {
    Critical,
    High,
    Medium,
    Low,
}

/// Recognized file extensions for market data files.
const RECOGNIZED_EXTENSIONS: &[&str] = &["csv", "parquet", "dbn", "json", "zst"];

/// Data validator
pub struct DataValidator {
    config: ValidationConfig,
}

impl DataValidator {
    /// Create new validator
    pub fn new(config: ValidationConfig) -> Self {
        Self { config }
    }

    /// Validate a downloaded file
    ///
    /// Performs full validation including file existence, size, extension,
    /// estimated record count, and quality scoring.
    pub async fn validate(&self, file_path: &Path) -> AcquisitionResult<ValidationResult> {
        let mut issues: Vec<ValidationIssue> = Vec::new();

        // Check file exists and is readable
        let metadata = match tokio::fs::metadata(file_path).await {
            Ok(m) => m,
            Err(e) => {
                return Err(AcquisitionError::Validation {
                    message: format!("Cannot access file '{}': {}", file_path.display(), e),
                });
            }
        };

        // Check file is not empty
        let file_size = metadata.len();
        if file_size == 0 {
            issues.push(ValidationIssue {
                severity: IssueSeverity::Critical,
                category: "file_size".to_string(),
                description: "File is empty (0 bytes)".to_string(),
                record_index: None,
            });

            return Ok(ValidationResult {
                is_valid: false,
                records_count: 0,
                issues_found: issues,
                quality_score: 0.0,
            });
        }

        // Check file has recognized extension
        let has_recognized_ext = file_path
            .extension()
            .and_then(|ext| ext.to_str())
            .map(|ext| RECOGNIZED_EXTENSIONS.contains(&ext))
            .unwrap_or(false);

        if !has_recognized_ext {
            issues.push(ValidationIssue {
                severity: IssueSeverity::Medium,
                category: "file_extension".to_string(),
                description: format!(
                    "Unrecognized file extension for '{}'",
                    file_path.display()
                ),
                record_index: None,
            });
        }

        // Estimate record count from file size (rough: ~100 bytes per record)
        let records_count = file_size / 100;

        // Check minimum bars requirement
        if (records_count as usize) < self.config.min_bars_required {
            issues.push(ValidationIssue {
                severity: IssueSeverity::High,
                category: "record_count".to_string(),
                description: format!(
                    "Estimated record count ({}) is below minimum required ({})",
                    records_count, self.config.min_bars_required
                ),
                record_index: None,
            });
        }

        // Calculate quality score: 1.0 minus 0.1 per issue, floored at 0.0
        let quality_score = 1.0 - (issues.len() as f64 * 0.1).min(1.0);

        // is_valid = no Critical issues
        let is_valid = !issues
            .iter()
            .any(|issue| issue.severity == IssueSeverity::Critical);

        Ok(ValidationResult {
            is_valid,
            records_count,
            issues_found: issues,
            quality_score,
        })
    }

    /// Quick validation (basic checks only)
    ///
    /// Checks file existence, non-empty size, and recognized extension.
    /// Returns `Ok(false)` for any check failure, `Ok(true)` when all pass.
    /// Only returns `Err` on unexpected I/O errors.
    pub async fn quick_validate(&self, file_path: &Path) -> AcquisitionResult<bool> {
        // Check file exists and is readable
        let metadata = match tokio::fs::metadata(file_path).await {
            Ok(m) => m,
            Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(false),
            Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => return Ok(false),
            Err(e) => {
                return Err(AcquisitionError::Validation {
                    message: format!("I/O error reading '{}': {}", file_path.display(), e),
                });
            }
        };

        // Check file is not empty
        if metadata.len() == 0 {
            return Ok(false);
        }

        // Check file has recognized extension
        let has_recognized_ext = file_path
            .extension()
            .and_then(|ext| ext.to_str())
            .map(|ext| RECOGNIZED_EXTENSIONS.contains(&ext))
            .unwrap_or(false);

        if !has_recognized_ext {
            return Ok(false);
        }

        Ok(true)
    }
}