test(data-acquisition): add pipeline integration tests for validator

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

services/data_acquisition_service/tests/pipeline_integration_tests.rs

@@ -0,0 +1,234 @@
+//! Integration tests for data acquisition pipeline
+//!
+//! Tests the DataValidator component for correctness across various input
+//! scenarios: empty files, valid CSV data, missing files, and default config.
+
+use data_acquisition_service::validator::{DataValidator, IssueSeverity, ValidationConfig};
+use std::io::Write;
+
+/// Empty file should be reported as invalid with a Critical issue.
+#[tokio::test]
+async fn test_validate_empty_file_reports_invalid() {
+    let validator = DataValidator::new(ValidationConfig::default());
+    let file = tempfile::NamedTempFile::new().expect("create temp file");
+    let result = validator.validate(file.path()).await;
+    assert!(result.is_ok(), "validate should not error on empty file");
+    let validation = result.expect("checked above");
+    // Empty file should not be valid
+    assert!(!validation.is_valid, "empty file must be invalid");
+    assert_eq!(validation.records_count, 0, "empty file has zero records");
+    assert!(
+        validation
+            .issues_found
+            .iter()
+            .any(|i| i.severity == IssueSeverity::Critical),
+        "empty file should have a Critical issue"
+    );
+}
+
+/// A non-empty CSV file with a recognized extension should pass validation.
+/// The file must be large enough to meet the min_bars_required threshold
+/// (default 100, estimated at ~100 bytes per record).
+#[tokio::test]
+async fn test_validate_nonempty_csv_file() {
+    let validator = DataValidator::new(ValidationConfig::default());
+    let dir = tempfile::tempdir().expect("create temp dir");
+    let csv_path = dir.path().join("test_data.csv");
+
+    {
+        let mut file = std::fs::File::create(&csv_path).expect("create csv");
+        writeln!(file, "timestamp,price,volume").expect("write header");
+        // Write enough rows to exceed min_bars_required (100 records at ~100 bytes each)
+        for i in 0..200 {
+            writeln!(
+                file,
+                "2024-01-01T{:02}:{:02}:00Z,{:.2},{}",
+                i % 24,
+                i % 60,
+                100.0 + i as f64 * 0.1,
+                1000 + i
+            )
+            .expect("write row");
+        }
+    }
+
+    let result = validator.validate(&csv_path).await;
+    assert!(result.is_ok(), "validate should succeed for valid CSV");
+    let validation = result.expect("checked above");
+    assert!(validation.is_valid, "non-empty CSV with recognized extension should be valid");
+    assert!(validation.records_count > 0, "should have estimated records");
+    assert!(
+        validation.quality_score > 0.0,
+        "quality score should be positive"
+    );
+}
+
+/// A file without a recognized extension should still validate (not crash)
+/// but should report an extension issue.
+#[tokio::test]
+async fn test_validate_unrecognized_extension_reports_issue() {
+    let validator = DataValidator::new(ValidationConfig::default());
+    let dir = tempfile::tempdir().expect("create temp dir");
+    let path = dir.path().join("data.xyz");
+
+    // Write enough bytes so the file is not empty and meets min_bars
+    let content = "x".repeat(20_000);
+    std::fs::write(&path, &content).expect("write file");
+
+    let result = validator.validate(&path).await;
+    assert!(result.is_ok(), "validate should not error");
+    let validation = result.expect("checked above");
+    assert!(
+        validation
+            .issues_found
+            .iter()
+            .any(|i| i.category == "file_extension"),
+        "should report unrecognized file extension issue"
+    );
+}
+
+/// quick_validate on a valid CSV file with recognized extension should return true.
+#[tokio::test]
+async fn test_quick_validate_valid_csv() {
+    let validator = DataValidator::new(ValidationConfig::default());
+    let dir = tempfile::tempdir().expect("create temp dir");
+    let path = dir.path().join("test.csv");
+    std::fs::write(&path, "header\ndata\n").expect("write file");
+    let result = validator.quick_validate(&path).await;
+    assert!(result.is_ok(), "quick_validate should not error");
+    assert!(
+        result.expect("checked above"),
+        "valid CSV file should pass quick validation"
+    );
+}
+
+/// quick_validate on a nonexistent file should return Ok(false) or Err.
+#[tokio::test]
+async fn test_quick_validate_nonexistent_file() {
+    let validator = DataValidator::new(ValidationConfig::default());
+    let result = validator
+        .quick_validate(std::path::Path::new("/nonexistent/path.csv"))
+        .await;
+    match result {
+        Ok(valid) => assert!(!valid, "nonexistent file should not be valid"),
+        Err(_) => {} // Also acceptable
+    }
+}
+
+/// quick_validate on an empty file should return false.
+#[tokio::test]
+async fn test_quick_validate_empty_file() {
+    let validator = DataValidator::new(ValidationConfig::default());
+    let dir = tempfile::tempdir().expect("create temp dir");
+    let path = dir.path().join("empty.csv");
+    std::fs::write(&path, "").expect("write empty file");
+    let result = validator.quick_validate(&path).await;
+    assert!(result.is_ok(), "quick_validate should not error on empty file");
+    assert!(
+        !result.expect("checked above"),
+        "empty file should fail quick validation"
+    );
+}
+
+/// quick_validate on a file without recognized extension should return false.
+#[tokio::test]
+async fn test_quick_validate_unrecognized_extension() {
+    let validator = DataValidator::new(ValidationConfig::default());
+    let dir = tempfile::tempdir().expect("create temp dir");
+    let path = dir.path().join("data.txt");
+    std::fs::write(&path, "some content\n").expect("write file");
+    let result = validator.quick_validate(&path).await;
+    assert!(result.is_ok(), "quick_validate should not error");
+    assert!(
+        !result.expect("checked above"),
+        "unrecognized extension should fail quick validation"
+    );
+}
+
+/// ValidationConfig::default() should have sensible positive values.
+#[tokio::test]
+async fn test_validation_config_defaults() {
+    let config = ValidationConfig::default();
+    assert!(
+        config.max_price_change_percent > 0.0,
+        "max_price_change_percent must be positive"
+    );
+    assert!(
+        config.max_volume_std_dev > 0.0,
+        "max_volume_std_dev must be positive"
+    );
+    assert!(
+        config.max_timestamp_gap_seconds > 0,
+        "max_timestamp_gap_seconds must be positive"
+    );
+    assert!(
+        config.min_bars_required > 0,
+        "min_bars_required must be positive"
+    );
+}
+
+/// validate on a nonexistent file should return an error.
+#[tokio::test]
+async fn test_validate_nonexistent_file_returns_error() {
+    let validator = DataValidator::new(ValidationConfig::default());
+    let result = validator
+        .validate(std::path::Path::new("/nonexistent/path.csv"))
+        .await;
+    assert!(
+        result.is_err(),
+        "validate on nonexistent file should return an error"
+    );
+}
+
+/// A small file below the min_bars_required threshold should report an issue
+/// but still be considered valid (no Critical issue, just High).
+#[tokio::test]
+async fn test_validate_small_file_below_min_bars() {
+    let config = ValidationConfig {
+        min_bars_required: 1000,
+        ..ValidationConfig::default()
+    };
+    let validator = DataValidator::new(config);
+    let dir = tempfile::tempdir().expect("create temp dir");
+    let path = dir.path().join("small.csv");
+
+    // Write a small amount of data (~50 bytes, well below 1000 * 100)
+    std::fs::write(&path, "timestamp,price,volume\n1,2,3\n").expect("write");
+
+    let result = validator.validate(&path).await;
+    assert!(result.is_ok(), "validate should not error");
+    let validation = result.expect("checked above");
+    assert!(
+        validation
+            .issues_found
+            .iter()
+            .any(|i| i.category == "record_count"),
+        "should report insufficient record count issue"
+    );
+}
+
+/// Validate that quality_score degrades as more issues are found.
+#[tokio::test]
+async fn test_quality_score_degrades_with_issues() {
+    let config = ValidationConfig {
+        min_bars_required: 100_000, // Very high threshold to trigger record_count issue
+        ..ValidationConfig::default()
+    };
+    let validator = DataValidator::new(config);
+    let dir = tempfile::tempdir().expect("create temp dir");
+    // Use unrecognized extension AND small file to trigger multiple issues
+    let path = dir.path().join("data.xyz");
+    std::fs::write(&path, "small content").expect("write");
+
+    let result = validator.validate(&path).await;
+    assert!(result.is_ok(), "validate should not error");
+    let validation = result.expect("checked above");
+    assert!(
+        validation.issues_found.len() >= 2,
+        "should have at least 2 issues (extension + record count)"
+    );
+    assert!(
+        validation.quality_score < 1.0,
+        "quality score should be below 1.0 when issues exist"
+    );
+}