From 27cc968e941129e64a122b11088ebec0dbfc237a Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sat, 21 Feb 2026 19:25:43 +0100 Subject: [PATCH] test(data-acquisition): add pipeline integration tests for validator Co-Authored-By: Claude Opus 4.6 --- .../tests/pipeline_integration_tests.rs | 234 ++++++++++++++++++ 1 file changed, 234 insertions(+) create mode 100644 services/data_acquisition_service/tests/pipeline_integration_tests.rs diff --git a/services/data_acquisition_service/tests/pipeline_integration_tests.rs b/services/data_acquisition_service/tests/pipeline_integration_tests.rs new file mode 100644 index 000000000..ef031f937 --- /dev/null +++ b/services/data_acquisition_service/tests/pipeline_integration_tests.rs @@ -0,0 +1,234 @@ +//! Integration tests for data acquisition pipeline +//! +//! Tests the DataValidator component for correctness across various input +//! scenarios: empty files, valid CSV data, missing files, and default config. + +use data_acquisition_service::validator::{DataValidator, IssueSeverity, ValidationConfig}; +use std::io::Write; + +/// Empty file should be reported as invalid with a Critical issue. +#[tokio::test] +async fn test_validate_empty_file_reports_invalid() { + let validator = DataValidator::new(ValidationConfig::default()); + let file = tempfile::NamedTempFile::new().expect("create temp file"); + let result = validator.validate(file.path()).await; + assert!(result.is_ok(), "validate should not error on empty file"); + let validation = result.expect("checked above"); + // Empty file should not be valid + assert!(!validation.is_valid, "empty file must be invalid"); + assert_eq!(validation.records_count, 0, "empty file has zero records"); + assert!( + validation + .issues_found + .iter() + .any(|i| i.severity == IssueSeverity::Critical), + "empty file should have a Critical issue" + ); +} + +/// A non-empty CSV file with a recognized extension should pass validation. +/// The file must be large enough to meet the min_bars_required threshold +/// (default 100, estimated at ~100 bytes per record). +#[tokio::test] +async fn test_validate_nonempty_csv_file() { + let validator = DataValidator::new(ValidationConfig::default()); + let dir = tempfile::tempdir().expect("create temp dir"); + let csv_path = dir.path().join("test_data.csv"); + + { + let mut file = std::fs::File::create(&csv_path).expect("create csv"); + writeln!(file, "timestamp,price,volume").expect("write header"); + // Write enough rows to exceed min_bars_required (100 records at ~100 bytes each) + for i in 0..200 { + writeln!( + file, + "2024-01-01T{:02}:{:02}:00Z,{:.2},{}", + i % 24, + i % 60, + 100.0 + i as f64 * 0.1, + 1000 + i + ) + .expect("write row"); + } + } + + let result = validator.validate(&csv_path).await; + assert!(result.is_ok(), "validate should succeed for valid CSV"); + let validation = result.expect("checked above"); + assert!(validation.is_valid, "non-empty CSV with recognized extension should be valid"); + assert!(validation.records_count > 0, "should have estimated records"); + assert!( + validation.quality_score > 0.0, + "quality score should be positive" + ); +} + +/// A file without a recognized extension should still validate (not crash) +/// but should report an extension issue. +#[tokio::test] +async fn test_validate_unrecognized_extension_reports_issue() { + let validator = DataValidator::new(ValidationConfig::default()); + let dir = tempfile::tempdir().expect("create temp dir"); + let path = dir.path().join("data.xyz"); + + // Write enough bytes so the file is not empty and meets min_bars + let content = "x".repeat(20_000); + std::fs::write(&path, &content).expect("write file"); + + let result = validator.validate(&path).await; + assert!(result.is_ok(), "validate should not error"); + let validation = result.expect("checked above"); + assert!( + validation + .issues_found + .iter() + .any(|i| i.category == "file_extension"), + "should report unrecognized file extension issue" + ); +} + +/// quick_validate on a valid CSV file with recognized extension should return true. +#[tokio::test] +async fn test_quick_validate_valid_csv() { + let validator = DataValidator::new(ValidationConfig::default()); + let dir = tempfile::tempdir().expect("create temp dir"); + let path = dir.path().join("test.csv"); + std::fs::write(&path, "header\ndata\n").expect("write file"); + let result = validator.quick_validate(&path).await; + assert!(result.is_ok(), "quick_validate should not error"); + assert!( + result.expect("checked above"), + "valid CSV file should pass quick validation" + ); +} + +/// quick_validate on a nonexistent file should return Ok(false) or Err. +#[tokio::test] +async fn test_quick_validate_nonexistent_file() { + let validator = DataValidator::new(ValidationConfig::default()); + let result = validator + .quick_validate(std::path::Path::new("/nonexistent/path.csv")) + .await; + match result { + Ok(valid) => assert!(!valid, "nonexistent file should not be valid"), + Err(_) => {} // Also acceptable + } +} + +/// quick_validate on an empty file should return false. +#[tokio::test] +async fn test_quick_validate_empty_file() { + let validator = DataValidator::new(ValidationConfig::default()); + let dir = tempfile::tempdir().expect("create temp dir"); + let path = dir.path().join("empty.csv"); + std::fs::write(&path, "").expect("write empty file"); + let result = validator.quick_validate(&path).await; + assert!(result.is_ok(), "quick_validate should not error on empty file"); + assert!( + !result.expect("checked above"), + "empty file should fail quick validation" + ); +} + +/// quick_validate on a file without recognized extension should return false. +#[tokio::test] +async fn test_quick_validate_unrecognized_extension() { + let validator = DataValidator::new(ValidationConfig::default()); + let dir = tempfile::tempdir().expect("create temp dir"); + let path = dir.path().join("data.txt"); + std::fs::write(&path, "some content\n").expect("write file"); + let result = validator.quick_validate(&path).await; + assert!(result.is_ok(), "quick_validate should not error"); + assert!( + !result.expect("checked above"), + "unrecognized extension should fail quick validation" + ); +} + +/// ValidationConfig::default() should have sensible positive values. +#[tokio::test] +async fn test_validation_config_defaults() { + let config = ValidationConfig::default(); + assert!( + config.max_price_change_percent > 0.0, + "max_price_change_percent must be positive" + ); + assert!( + config.max_volume_std_dev > 0.0, + "max_volume_std_dev must be positive" + ); + assert!( + config.max_timestamp_gap_seconds > 0, + "max_timestamp_gap_seconds must be positive" + ); + assert!( + config.min_bars_required > 0, + "min_bars_required must be positive" + ); +} + +/// validate on a nonexistent file should return an error. +#[tokio::test] +async fn test_validate_nonexistent_file_returns_error() { + let validator = DataValidator::new(ValidationConfig::default()); + let result = validator + .validate(std::path::Path::new("/nonexistent/path.csv")) + .await; + assert!( + result.is_err(), + "validate on nonexistent file should return an error" + ); +} + +/// A small file below the min_bars_required threshold should report an issue +/// but still be considered valid (no Critical issue, just High). +#[tokio::test] +async fn test_validate_small_file_below_min_bars() { + let config = ValidationConfig { + min_bars_required: 1000, + ..ValidationConfig::default() + }; + let validator = DataValidator::new(config); + let dir = tempfile::tempdir().expect("create temp dir"); + let path = dir.path().join("small.csv"); + + // Write a small amount of data (~50 bytes, well below 1000 * 100) + std::fs::write(&path, "timestamp,price,volume\n1,2,3\n").expect("write"); + + let result = validator.validate(&path).await; + assert!(result.is_ok(), "validate should not error"); + let validation = result.expect("checked above"); + assert!( + validation + .issues_found + .iter() + .any(|i| i.category == "record_count"), + "should report insufficient record count issue" + ); +} + +/// Validate that quality_score degrades as more issues are found. +#[tokio::test] +async fn test_quality_score_degrades_with_issues() { + let config = ValidationConfig { + min_bars_required: 100_000, // Very high threshold to trigger record_count issue + ..ValidationConfig::default() + }; + let validator = DataValidator::new(config); + let dir = tempfile::tempdir().expect("create temp dir"); + // Use unrecognized extension AND small file to trigger multiple issues + let path = dir.path().join("data.xyz"); + std::fs::write(&path, "small content").expect("write"); + + let result = validator.validate(&path).await; + assert!(result.is_ok(), "validate should not error"); + let validation = result.expect("checked above"); + assert!( + validation.issues_found.len() >= 2, + "should have at least 2 issues (extension + record count)" + ); + assert!( + validation.quality_score < 1.0, + "quality score should be below 1.0 when issues exist" + ); +}