//! # Validation Test Helpers //! //! Common helper functions for data validation pipeline tests. //! Provides utilities for creating test data, configuring validators, //! and asserting validation results. //! //! ## Usage //! //! ```rust,no_run //! use common::validation_helpers::*; //! //! // Create validator with standard configuration //! let validator = create_test_validator_config(true, true, true); //! //! // Generate anomalous data for testing error detection //! let bars = generate_anomalous_data(100, AnomalyType::PriceSpikes); //! //! // Test validation and assert results //! let result = validator.validate(&bars)?; //! assert_validation_result(&result, false, 5, 0); // Expect 5 errors //! ``` use ml::data_validation::corrector::DataCorrector; use ml::data_validation::rules::{ CompletenessRule, ContinuityRule, IndicatorRule, IntegrityRule, TimestampRule, }; use ml::data_validation::validator::{DataValidator, ValidationResult}; use ml::data_loader::Indicators; use ml::types::OHLCVBar; // ============================================================================ // Test Configuration Helpers // ============================================================================ /// Create validator with standard test configuration /// /// # Arguments /// /// * `with_integrity` - Enable OHLCV integrity checks (high≥low, volume≥0) /// * `with_continuity` - Enable price continuity checks (20% spike threshold) /// * `with_indicators` - Enable technical indicator validation /// /// # Returns /// /// Configured `DataValidator` ready for testing /// /// # Example /// /// ```rust,no_run /// # use common::validation_helpers::*; /// // Create validator with all checks enabled /// let validator = create_test_validator_config(true, true, true); /// /// // Create validator with only integrity checks /// let validator = create_test_validator_config(true, false, false); /// ``` pub(crate) fn create_test_validator_config( with_integrity: bool, with_continuity: bool, with_indicators: bool, ) -> DataValidator { let mut validator = DataValidator::new(); if with_integrity { validator = validator.with_rule(Box::new(IntegrityRule::new())); } if with_continuity { // 20% spike threshold (standard for production) validator = validator.with_rule(Box::new(ContinuityRule::new(0.20))); } if with_indicators { validator = validator.with_rule(Box::new(IndicatorRule::new())); } validator } /// Create validator with custom spike threshold /// /// # Arguments /// /// * `spike_threshold` - Maximum allowed price change between bars (e.g., 0.20 = 20%) /// /// # Example /// /// ```rust,no_run /// # use common::validation_helpers::*; /// // Create validator that allows 30% spikes /// let validator = create_test_validator_with_threshold(0.30); /// ``` pub(crate) fn create_test_validator_with_threshold(spike_threshold: f64) -> DataValidator { DataValidator::new() .with_rule(Box::new(IntegrityRule::new())) .with_rule(Box::new(ContinuityRule::new(spike_threshold))) } /// Create validator with timestamp validation /// /// # Arguments /// /// * `bar_interval_secs` - Expected interval between bars (e.g., 60 for 1-minute bars) /// /// # Example /// /// ```rust,no_run /// # use common::validation_helpers::*; /// // Create validator for 5-minute bars /// let validator = create_test_validator_with_timestamps(300); /// ``` pub(crate) fn create_test_validator_with_timestamps(bar_interval_secs: i64) -> DataValidator { DataValidator::new() .with_rule(Box::new(IntegrityRule::new())) .with_rule(Box::new(TimestampRule::new(bar_interval_secs))) } /// Create validator with completeness checking /// /// # Arguments /// /// * `bar_interval_secs` - Expected interval between bars /// * `min_completeness` - Minimum completeness ratio (e.g., 0.95 = 95%) /// /// # Example /// /// ```rust,no_run /// # use common::validation_helpers::*; /// // Require 99% data completeness for 1-minute bars /// let validator = create_test_validator_with_completeness(60, 0.99); /// ``` pub(crate) fn create_test_validator_with_completeness( bar_interval_secs: i64, min_completeness: f64, ) -> DataValidator { DataValidator::new() .with_rule(Box::new(IntegrityRule::new())) .with_rule(Box::new(CompletenessRule::new( bar_interval_secs, min_completeness, ))) } /// Create data corrector with standard configuration /// /// # Example /// /// ```rust,no_run /// # use common::validation_helpers::*; /// let corrector = create_test_corrector(); /// let corrected_bars = corrector.correct_price_spikes(&bars, 0.20)?; /// ``` pub(crate) fn create_test_corrector() -> DataCorrector { DataCorrector::new() } // ============================================================================ // Anomalous Data Generation // ============================================================================ /// Type of anomaly to inject into test data #[derive(Debug, Clone, Copy)] pub(crate) enum AnomalyType { /// Price spikes >20% between bars PriceSpikes, /// Invalid OHLCV relationships (high < low, etc.) IntegrityViolations, /// Negative or zero volumes NegativeVolume, /// Timestamp gaps or out-of-order timestamps TimestampGaps, /// Missing bars in sequence MissingBars, /// Multiple anomaly types combined Mixed, } /// Generate anomalous data for testing error detection /// /// # Arguments /// /// * `bar_count` - Total number of bars to generate /// * `anomaly_type` - Type of anomaly to inject /// /// # Returns /// /// Vector of `OHLCVBar` with injected anomalies /// /// # Example /// /// ```rust,no_run /// # use common::validation_helpers::*; /// // Generate 100 bars with price spikes /// let bars = generate_anomalous_data(100, AnomalyType::PriceSpikes); /// /// // Test that validator detects the anomalies /// let result = validator.validate(&bars)?; /// assert!(!result.is_valid()); /// ``` pub(crate) fn generate_anomalous_data( bar_count: usize, anomaly_type: AnomalyType, ) -> Vec { let mut bars = generate_clean_data(bar_count); match anomaly_type { AnomalyType::PriceSpikes => inject_price_spikes(&mut bars), AnomalyType::IntegrityViolations => inject_integrity_violations(&mut bars), AnomalyType::NegativeVolume => inject_negative_volume(&mut bars), AnomalyType::TimestampGaps => inject_timestamp_gaps(&mut bars), AnomalyType::MissingBars => return generate_bars_with_gaps(bar_count), AnomalyType::Mixed => { inject_price_spikes(&mut bars); inject_integrity_violations(&mut bars); inject_negative_volume(&mut bars); }, } bars } /// Generate clean, valid OHLCV data for testing /// /// # Arguments /// /// * `bar_count` - Number of bars to generate /// /// # Returns /// /// Vector of valid `OHLCVBar` with realistic price movements /// /// # Example /// /// ```rust,no_run /// # use common::validation_helpers::*; /// // Generate 50 bars of clean data /// let bars = generate_clean_data(50); /// /// // Should pass all validation checks /// let result = validator.validate(&bars)?; /// assert!(result.is_valid()); /// ``` pub(crate) fn generate_clean_data(bar_count: usize) -> Vec { let mut bars = Vec::with_capacity(bar_count); let mut base_price = 100.0; let base_timestamp = chrono::Utc::now(); for i in 0..bar_count { // Realistic price movement: ±2% per bar let change = (i as f64 % 10.0 - 5.0) / 100.0; // Oscillates between -5% and +5% base_price *= 1.0 + change * 0.4; // Max ±2% per bar let open = base_price; let high = base_price * 1.01; // High is 1% above base let low = base_price * 0.99; // Low is 1% below base let close = base_price * (1.0 + change * 0.2); // Close varies within ±0.4% bars.push(OHLCVBar { timestamp: base_timestamp + chrono::Duration::seconds(i as i64 * 60), open, high, low, close, volume: 1000.0 + (i as f64 * 10.0), }); } bars } // ============================================================================ // Anomaly Injection Functions // ============================================================================ /// Inject price spikes into test data fn inject_price_spikes(bars: &mut [OHLCVBar]) { // Inject spikes at 20%, 50%, and 80% through the data let spike_indices = [bars.len() / 5, bars.len() / 2, bars.len() * 4 / 5]; for &idx in &spike_indices { if idx < bars.len() { // Create 50% price spike (well above 20% threshold) bars[idx].open *= 1.5; bars[idx].high *= 1.5; bars[idx].low *= 1.5; bars[idx].close *= 1.5; } } } /// Inject OHLCV integrity violations fn inject_integrity_violations(bars: &mut [OHLCVBar]) { // Inject violations at 10%, 30%, 60% through the data let violation_indices = [bars.len() / 10, bars.len() * 3 / 10, bars.len() * 6 / 10]; for (i, &idx) in violation_indices.iter().enumerate() { if idx < bars.len() { match i % 3 { 0 => { // High < low bars[idx].high = bars[idx].low * 0.95; }, 1 => { // High < close bars[idx].close = bars[idx].high * 1.05; }, 2 => { // Low > open bars[idx].open = bars[idx].low * 0.95; }, _ => {}, } } } } /// Inject negative volumes fn inject_negative_volume(bars: &mut [OHLCVBar]) { // Inject negative volumes at 15%, 45%, 75% through the data let volume_indices = [ bars.len() * 15 / 100, bars.len() * 45 / 100, bars.len() * 75 / 100, ]; for &idx in &volume_indices { if idx < bars.len() { bars[idx].volume = -100.0; } } } /// Inject timestamp gaps (out of order or large gaps) fn inject_timestamp_gaps(bars: &mut [OHLCVBar]) { // Inject gaps at 25% and 75% through the data let gap_indices = [bars.len() / 4, bars.len() * 3 / 4]; for &idx in &gap_indices { if idx < bars.len() && idx > 0 { // Create 5-minute gap (5x normal 1-minute interval) bars[idx].timestamp = bars[idx - 1].timestamp + chrono::Duration::seconds(300); } } } /// Generate bars with missing data (gaps in sequence) fn generate_bars_with_gaps(bar_count: usize) -> Vec { let mut bars = Vec::with_capacity(bar_count); let mut base_price = 100.0; let base_timestamp = chrono::Utc::now(); for i in 0..bar_count { // Skip every 10th bar to create gaps if i % 10 == 5 { continue; } let change = (i as f64 % 10.0 - 5.0) / 100.0; base_price *= 1.0 + change * 0.4; let open = base_price; let high = base_price * 1.01; let low = base_price * 0.99; let close = base_price * (1.0 + change * 0.2); bars.push(OHLCVBar { timestamp: base_timestamp + chrono::Duration::seconds(i as i64 * 60), open, high, low, close, volume: 1000.0 + (i as f64 * 10.0), }); } bars } // ============================================================================ // Indicator Data Generation // ============================================================================ /// Generate valid technical indicators for testing /// /// # Arguments /// /// * `bar_count` - Number of indicator values to generate /// /// # Returns /// /// Valid `Indicators` struct with realistic values /// /// # Example /// /// ```rust,no_run /// # use common::validation_helpers::*; /// let indicators = generate_clean_indicators(100); /// let result = validator.validate_indicators(&indicators)?; /// assert!(result.is_valid()); /// ``` pub(crate) fn generate_clean_indicators(bar_count: usize) -> Indicators { Indicators { rsi: (0..bar_count).map(|i| 30.0 + (i % 40) as f32).collect(), // Oscillates 30-70 macd: (0..bar_count).map(|i| (i as f32 % 10.0) - 5.0).collect(), // -5 to +5 macd_signal: (0..bar_count).map(|i| (i as f32 % 8.0) - 4.0).collect(), // -4 to +4 bb_upper: (0..bar_count).map(|i| 100.0 + i as f32 * 0.1).collect(), bb_middle: (0..bar_count).map(|i| 98.0 + i as f32 * 0.1).collect(), bb_lower: (0..bar_count).map(|i| 96.0 + i as f32 * 0.1).collect(), atr: (0..bar_count).map(|i| 2.0 + (i % 5) as f32 * 0.2).collect(), // 2.0-3.0 ema_fast: (0..bar_count).map(|i| 99.0 + i as f32 * 0.05).collect(), ema_slow: (0..bar_count).map(|i| 98.0 + i as f32 * 0.05).collect(), volume_ma: (0..bar_count).map(|i| 1000.0 + i as f32 * 10.0).collect(), } } /// Generate indicators with anomalies for testing error detection /// /// # Arguments /// /// * `bar_count` - Number of indicator values /// * `anomaly_type` - Type of anomaly to inject (RSI out of range, NaN, Inf) /// /// # Returns /// /// `Indicators` with injected anomalies /// /// # Example /// /// ```rust,no_run /// # use common::validation_helpers::*; /// // Generate indicators with NaN values /// let indicators = generate_anomalous_indicators(100, IndicatorAnomalyType::NaN); /// let result = validator.validate_indicators(&indicators)?; /// assert!(!result.is_valid()); /// ``` pub(crate) fn generate_anomalous_indicators( bar_count: usize, anomaly_type: IndicatorAnomalyType, ) -> Indicators { let mut indicators = generate_clean_indicators(bar_count); match anomaly_type { IndicatorAnomalyType::RsiOutOfRange => { if bar_count > 10 { indicators.rsi[bar_count / 4] = 105.0; // RSI > 100 indicators.rsi[bar_count / 2] = -5.0; // RSI < 0 } }, IndicatorAnomalyType::NaN => { if bar_count > 10 { indicators.rsi[bar_count / 3] = f32::NAN; indicators.macd[bar_count / 2] = f32::NAN; } }, IndicatorAnomalyType::Infinity => { if bar_count > 10 { indicators.macd[bar_count / 4] = f32::INFINITY; indicators.atr[bar_count / 2] = f32::NEG_INFINITY; } }, } indicators } /// Type of indicator anomaly #[derive(Debug, Clone, Copy)] pub(crate) enum IndicatorAnomalyType { /// RSI values outside 0-100 range RsiOutOfRange, /// NaN values in indicators NaN, /// Infinite values in indicators Infinity, } // ============================================================================ // Validation Result Assertions // ============================================================================ /// Assert validation result matches expected values /// /// # Arguments /// /// * `result` - ValidationResult to check /// * `should_be_valid` - Expected validation status /// * `expected_errors` - Expected number of errors /// * `expected_warnings` - Expected number of warnings /// /// # Panics /// /// Panics if validation result doesn't match expectations /// /// # Example /// /// ```rust,no_run /// # use common::validation_helpers::*; /// let result = validator.validate(&bars)?; /// /// // Assert validation passed with no errors/warnings /// assert_validation_result(&result, true, 0, 0); /// /// // Assert validation failed with 5 errors and 2 warnings /// assert_validation_result(&result, false, 5, 2); /// ``` pub(crate) fn assert_validation_result( result: &ValidationResult, should_be_valid: bool, expected_errors: usize, expected_warnings: usize, ) { assert_eq!( result.is_valid(), should_be_valid, "Validation status mismatch. Expected valid={}, got valid={}. Errors: {}", should_be_valid, result.is_valid(), result.error_summary() ); assert_eq!( result.error_count(), expected_errors, "Error count mismatch. Expected {}, got {}. Errors: {}", expected_errors, result.error_count(), result.error_summary() ); assert_eq!( result.warning_count(), expected_warnings, "Warning count mismatch. Expected {}, got {}", expected_warnings, result.warning_count() ); } /// Assert validation result contains specific error category /// /// # Arguments /// /// * `result` - ValidationResult to check /// * `error_category` - Expected error category string (e.g., "integrity", "continuity") /// /// # Example /// /// ```rust,no_run /// # use common::validation_helpers::*; /// let result = validator.validate(&bars)?; /// assert_has_error_category(&result, "integrity"); /// assert_has_error_category(&result, "continuity"); /// ``` pub(crate) fn assert_has_error_category(result: &ValidationResult, error_category: &str) { assert!( result.error_summary().contains(error_category), "Expected error category '{}' not found in errors: {}", error_category, result.error_summary() ); } /// Assert validation passed with no errors /// /// # Example /// /// ```rust,no_run /// # use common::validation_helpers::*; /// let result = validator.validate(&clean_bars)?; /// assert_validation_passed(&result); /// ``` pub(crate) fn assert_validation_passed(result: &ValidationResult) { assert!( result.is_valid(), "Validation should pass but failed. Errors: {}", result.error_summary() ); assert_eq!( result.error_count(), 0, "Expected no errors but got {}", result.error_count() ); } /// Assert validation failed with at least one error /// /// # Example /// /// ```rust,no_run /// # use common::validation_helpers::*; /// let result = validator.validate(&bad_bars)?; /// assert_validation_failed(&result); /// ``` pub(crate) fn assert_validation_failed(result: &ValidationResult) { assert!(!result.is_valid(), "Validation should fail but passed"); assert!(result.error_count() > 0, "Expected errors but got none"); } // ============================================================================ // Test Data Builders (Fluent API) // ============================================================================ /// Builder for creating custom test bars /// /// Provides fluent API for constructing test data with specific characteristics. /// /// # Example /// /// ```rust,no_run /// # use common::validation_helpers::*; /// let bars = TestBarBuilder::new() /// .count(100) /// .base_price(150.0) /// .volatility(0.05) // 5% volatility /// .trend(0.001) // 0.1% uptrend per bar /// .build(); /// ``` pub(crate) struct TestBarBuilder { count: usize, base_price: f64, volatility: f64, trend: f64, base_timestamp: chrono::DateTime, interval_secs: i64, } impl TestBarBuilder { /// Create new builder with defaults pub(crate) fn new() -> Self { Self { count: 100, base_price: 100.0, volatility: 0.02, // 2% trend: 0.0, // No trend base_timestamp: chrono::Utc::now(), interval_secs: 60, // 1 minute } } /// Set number of bars to generate pub(crate) fn count(mut self, count: usize) -> Self { self.count = count; self } /// Set starting price pub(crate) fn base_price(mut self, price: f64) -> Self { self.base_price = price; self } /// Set price volatility (e.g., 0.02 = ±2% per bar) pub(crate) fn volatility(mut self, volatility: f64) -> Self { self.volatility = volatility; self } /// Set price trend (e.g., 0.001 = +0.1% per bar) pub(crate) fn trend(mut self, trend: f64) -> Self { self.trend = trend; self } /// Set starting timestamp pub(crate) fn base_timestamp(mut self, timestamp: chrono::DateTime) -> Self { self.base_timestamp = timestamp; self } /// Set interval between bars in seconds pub(crate) fn interval_secs(mut self, interval: i64) -> Self { self.interval_secs = interval; self } /// Build the bars pub(crate) fn build(self) -> Vec { let mut bars = Vec::with_capacity(self.count); let mut price = self.base_price; for i in 0..self.count { // Apply trend price *= 1.0 + self.trend; // Add volatility (sine wave pattern) let vol_factor = (i as f64 * 0.1).sin() * self.volatility; price *= 1.0 + vol_factor; let open = price; let high = price * (1.0 + self.volatility * 0.5); let low = price * (1.0 - self.volatility * 0.5); let close = price * (1.0 + vol_factor * 0.5); bars.push(OHLCVBar { timestamp: self.base_timestamp + chrono::Duration::seconds(i as i64 * self.interval_secs), open, high, low, close, volume: 1000.0 + (i as f64 * 10.0), }); } bars } } impl Default for TestBarBuilder { fn default() -> Self { Self::new() } } #[cfg(test)] mod tests { use super::*; #[test] fn test_generate_clean_data() { let bars = generate_clean_data(100); assert_eq!(bars.len(), 100); // All bars should have valid OHLCV relationships for bar in &bars { assert!(bar.high >= bar.low); assert!(bar.high >= bar.open); assert!(bar.high >= bar.close); assert!(bar.low <= bar.open); assert!(bar.low <= bar.close); assert!(bar.volume >= 0.0); } } #[test] fn test_generate_anomalous_data_price_spikes() { let bars = generate_anomalous_data(100, AnomalyType::PriceSpikes); assert_eq!(bars.len(), 100); // Should have some price spikes >20% let mut has_spike = false; for i in 1..bars.len() { let change = (bars[i].close - bars[i - 1].close).abs() / bars[i - 1].close; if change > 0.20 { has_spike = true; break; } } assert!(has_spike, "Expected to find price spikes >20%"); } #[test] fn test_test_bar_builder() { let bars = TestBarBuilder::new() .count(50) .base_price(200.0) .volatility(0.03) .build(); assert_eq!(bars.len(), 50); assert!(bars[0].close > 195.0 && bars[0].close < 205.0); } }