//! Integration tests for data acquisition pipeline //! //! Tests the DataValidator component for correctness across various input //! scenarios: empty files, valid CSV data, missing files, and default config. use data_acquisition_service::validator::{DataValidator, IssueSeverity, ValidationConfig}; use std::io::Write; /// Empty file should be reported as invalid with a Critical issue. #[tokio::test] async fn test_validate_empty_file_reports_invalid() { let validator = DataValidator::new(ValidationConfig::default()); let file = tempfile::NamedTempFile::new().expect("create temp file"); let result = validator.validate(file.path()).await; assert!(result.is_ok(), "validate should not error on empty file"); let validation = result.expect("checked above"); // Empty file should not be valid assert!(!validation.is_valid, "empty file must be invalid"); assert_eq!(validation.records_count, 0, "empty file has zero records"); assert!( validation .issues_found .iter() .any(|i| i.severity == IssueSeverity::Critical), "empty file should have a Critical issue" ); } /// A non-empty CSV file with a recognized extension should pass validation. /// The file must be large enough to meet the min_bars_required threshold /// (default 100, estimated at ~100 bytes per record). #[tokio::test] async fn test_validate_nonempty_csv_file() { let validator = DataValidator::new(ValidationConfig::default()); let dir = tempfile::tempdir().expect("create temp dir"); let csv_path = dir.path().join("test_data.csv"); { let mut file = std::fs::File::create(&csv_path).expect("create csv"); writeln!(file, "timestamp,price,volume").expect("write header"); // Write enough rows to exceed min_bars_required (100 records at ~100 bytes each) for i in 0..200 { writeln!( file, "2024-01-01T{:02}:{:02}:00Z,{:.2},{}", i % 24, i % 60, 100.0 + i as f64 * 0.1, 1000 + i ) .expect("write row"); } } let result = validator.validate(&csv_path).await; assert!(result.is_ok(), "validate should succeed for valid CSV"); let validation = result.expect("checked above"); assert!(validation.is_valid, "non-empty CSV with recognized extension should be valid"); assert!(validation.records_count > 0, "should have estimated records"); assert!( validation.quality_score > 0.0, "quality score should be positive" ); } /// A file without a recognized extension should still validate (not crash) /// but should report an extension issue. #[tokio::test] async fn test_validate_unrecognized_extension_reports_issue() { let validator = DataValidator::new(ValidationConfig::default()); let dir = tempfile::tempdir().expect("create temp dir"); let path = dir.path().join("data.xyz"); // Write enough bytes so the file is not empty and meets min_bars let content = "x".repeat(20_000); std::fs::write(&path, &content).expect("write file"); let result = validator.validate(&path).await; assert!(result.is_ok(), "validate should not error"); let validation = result.expect("checked above"); assert!( validation .issues_found .iter() .any(|i| i.category == "file_extension"), "should report unrecognized file extension issue" ); } /// quick_validate on a valid CSV file with recognized extension should return true. #[tokio::test] async fn test_quick_validate_valid_csv() { let validator = DataValidator::new(ValidationConfig::default()); let dir = tempfile::tempdir().expect("create temp dir"); let path = dir.path().join("test.csv"); std::fs::write(&path, "header\ndata\n").expect("write file"); let result = validator.quick_validate(&path).await; assert!(result.is_ok(), "quick_validate should not error"); assert!( result.expect("checked above"), "valid CSV file should pass quick validation" ); } /// quick_validate on a nonexistent file should return Ok(false) or Err. #[tokio::test] async fn test_quick_validate_nonexistent_file() { let validator = DataValidator::new(ValidationConfig::default()); let result = validator .quick_validate(std::path::Path::new("/nonexistent/path.csv")) .await; if let Ok(valid) = result { assert!(!valid, "nonexistent file should not be valid"); } // Err is also acceptable -- nonexistent file can legitimately error } /// quick_validate on an empty file should return false. #[tokio::test] async fn test_quick_validate_empty_file() { let validator = DataValidator::new(ValidationConfig::default()); let dir = tempfile::tempdir().expect("create temp dir"); let path = dir.path().join("empty.csv"); std::fs::write(&path, "").expect("write empty file"); let result = validator.quick_validate(&path).await; assert!(result.is_ok(), "quick_validate should not error on empty file"); assert!( !result.expect("checked above"), "empty file should fail quick validation" ); } /// quick_validate on a file without recognized extension should return false. #[tokio::test] async fn test_quick_validate_unrecognized_extension() { let validator = DataValidator::new(ValidationConfig::default()); let dir = tempfile::tempdir().expect("create temp dir"); let path = dir.path().join("data.txt"); std::fs::write(&path, "some content\n").expect("write file"); let result = validator.quick_validate(&path).await; assert!(result.is_ok(), "quick_validate should not error"); assert!( !result.expect("checked above"), "unrecognized extension should fail quick validation" ); } /// ValidationConfig::default() should have sensible positive values. #[tokio::test] async fn test_validation_config_defaults() { let config = ValidationConfig::default(); assert!( config.max_price_change_percent > 0.0, "max_price_change_percent must be positive" ); assert!( config.max_volume_std_dev > 0.0, "max_volume_std_dev must be positive" ); assert!( config.max_timestamp_gap_seconds > 0, "max_timestamp_gap_seconds must be positive" ); assert!( config.min_bars_required > 0, "min_bars_required must be positive" ); } /// validate on a nonexistent file should return an error. #[tokio::test] async fn test_validate_nonexistent_file_returns_error() { let validator = DataValidator::new(ValidationConfig::default()); let result = validator .validate(std::path::Path::new("/nonexistent/path.csv")) .await; assert!( result.is_err(), "validate on nonexistent file should return an error" ); } /// A small file below the min_bars_required threshold should report an issue /// but still be considered valid (no Critical issue, just High). #[tokio::test] async fn test_validate_small_file_below_min_bars() { let config = ValidationConfig { min_bars_required: 1000, ..ValidationConfig::default() }; let validator = DataValidator::new(config); let dir = tempfile::tempdir().expect("create temp dir"); let path = dir.path().join("small.csv"); // Write a small amount of data (~50 bytes, well below 1000 * 100) std::fs::write(&path, "timestamp,price,volume\n1,2,3\n").expect("write"); let result = validator.validate(&path).await; assert!(result.is_ok(), "validate should not error"); let validation = result.expect("checked above"); assert!( validation .issues_found .iter() .any(|i| i.category == "record_count"), "should report insufficient record count issue" ); } /// Validate that quality_score degrades as more issues are found. #[tokio::test] async fn test_quality_score_degrades_with_issues() { let config = ValidationConfig { min_bars_required: 100_000, // Very high threshold to trigger record_count issue ..ValidationConfig::default() }; let validator = DataValidator::new(config); let dir = tempfile::tempdir().expect("create temp dir"); // Use unrecognized extension AND small file to trigger multiple issues let path = dir.path().join("data.xyz"); std::fs::write(&path, "small content").expect("write"); let result = validator.validate(&path).await; assert!(result.is_ok(), "validate should not error"); let validation = result.expect("checked above"); assert!( validation.issues_found.len() >= 2, "should have at least 2 issues (extension + record count)" ); assert!( validation.quality_score < 1.0, "quality score should be below 1.0 when issues exist" ); }