//! Data validation module //! //! Validates downloaded data for quality issues: //! - Missing values //! - Price anomalies //! - Volume outliers //! - Timestamp gaps //! - Schema conformance use crate::error::{AcquisitionError, AcquisitionResult}; use std::path::Path; /// Validation configuration #[derive(Debug, Clone)] pub struct ValidationConfig { pub max_price_change_percent: f64, pub max_volume_std_dev: f64, pub max_timestamp_gap_seconds: i64, pub min_bars_required: usize, } impl Default for ValidationConfig { fn default() -> Self { Self { max_price_change_percent: 10.0, max_volume_std_dev: 5.0, max_timestamp_gap_seconds: 3600, min_bars_required: 100, } } } /// Validation result #[derive(Debug, Clone)] pub struct ValidationResult { pub is_valid: bool, pub records_count: u64, pub issues_found: Vec, pub quality_score: f64, } /// Validation issue #[derive(Debug, Clone)] pub struct ValidationIssue { pub severity: IssueSeverity, pub category: String, pub description: String, pub record_index: Option, } /// Issue severity #[derive(Debug, Clone, PartialEq, Eq)] pub enum IssueSeverity { Critical, High, Medium, Low, } /// Recognized file extensions for market data files. const RECOGNIZED_EXTENSIONS: &[&str] = &["csv", "parquet", "dbn", "json", "zst"]; /// Data validator pub struct DataValidator { config: ValidationConfig, } impl DataValidator { /// Create new validator pub fn new(config: ValidationConfig) -> Self { Self { config } } /// Validate a downloaded file /// /// Performs full validation including file existence, size, extension, /// estimated record count, and quality scoring. pub async fn validate(&self, file_path: &Path) -> AcquisitionResult { let mut issues: Vec = Vec::new(); // Check file exists and is readable let metadata = match tokio::fs::metadata(file_path).await { Ok(m) => m, Err(e) => { return Err(AcquisitionError::Validation { message: format!("Cannot access file '{}': {}", file_path.display(), e), }); } }; // Check file is not empty let file_size = metadata.len(); if file_size == 0 { issues.push(ValidationIssue { severity: IssueSeverity::Critical, category: "file_size".to_string(), description: "File is empty (0 bytes)".to_string(), record_index: None, }); return Ok(ValidationResult { is_valid: false, records_count: 0, issues_found: issues, quality_score: 0.0, }); } // Check file has recognized extension let has_recognized_ext = file_path .extension() .and_then(|ext| ext.to_str()) .map(|ext| RECOGNIZED_EXTENSIONS.contains(&ext)) .unwrap_or(false); if !has_recognized_ext { issues.push(ValidationIssue { severity: IssueSeverity::Medium, category: "file_extension".to_string(), description: format!( "Unrecognized file extension for '{}'", file_path.display() ), record_index: None, }); } // Estimate record count from file size (rough: ~100 bytes per record) let records_count = file_size / 100; // Check minimum bars requirement if (records_count as usize) < self.config.min_bars_required { issues.push(ValidationIssue { severity: IssueSeverity::High, category: "record_count".to_string(), description: format!( "Estimated record count ({}) is below minimum required ({})", records_count, self.config.min_bars_required ), record_index: None, }); } // Calculate quality score: 1.0 minus 0.1 per issue, floored at 0.0 let quality_score = 1.0 - (issues.len() as f64 * 0.1).min(1.0); // is_valid = no Critical issues let is_valid = !issues .iter() .any(|issue| issue.severity == IssueSeverity::Critical); Ok(ValidationResult { is_valid, records_count, issues_found: issues, quality_score, }) } /// Quick validation (basic checks only) /// /// Checks file existence, non-empty size, and recognized extension. /// Returns `Ok(false)` for any check failure, `Ok(true)` when all pass. /// Only returns `Err` on unexpected I/O errors. pub async fn quick_validate(&self, file_path: &Path) -> AcquisitionResult { // Check file exists and is readable let metadata = match tokio::fs::metadata(file_path).await { Ok(m) => m, Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(false), Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => return Ok(false), Err(e) => { return Err(AcquisitionError::Validation { message: format!("I/O error reading '{}': {}", file_path.display(), e), }); } }; // Check file is not empty if metadata.len() == 0 { return Ok(false); } // Check file has recognized extension let has_recognized_ext = file_path .extension() .and_then(|ext| ext.to_str()) .map(|ext| RECOGNIZED_EXTENSIONS.contains(&ext)) .unwrap_or(false); if !has_recognized_ext { return Ok(false); } Ok(true) } }