foxhunt/services/data_acquisition_service/tests/pipeline_integration_tests.rs

//! Integration tests for data acquisition pipeline
//!
//! Tests the DataValidator component for correctness across various input
//! scenarios: empty files, valid CSV data, missing files, and default config.

use data_acquisition_service::validator::{DataValidator, IssueSeverity, ValidationConfig};
use std::io::Write;

/// Empty file should be reported as invalid with a Critical issue.
#[tokio::test]
async fn test_validate_empty_file_reports_invalid() {
    let validator = DataValidator::new(ValidationConfig::default());
    let file = tempfile::NamedTempFile::new().expect("create temp file");
    let result = validator.validate(file.path()).await;
    assert!(result.is_ok(), "validate should not error on empty file");
    let validation = result.expect("checked above");
    // Empty file should not be valid
    assert!(!validation.is_valid, "empty file must be invalid");
    assert_eq!(validation.records_count, 0, "empty file has zero records");
    assert!(
        validation
            .issues_found
            .iter()
            .any(|i| i.severity == IssueSeverity::Critical),
        "empty file should have a Critical issue"
    );
}

/// A non-empty CSV file with a recognized extension should pass validation.
/// The file must be large enough to meet the min_bars_required threshold
/// (default 100, estimated at ~100 bytes per record).
#[tokio::test]
async fn test_validate_nonempty_csv_file() {
    let validator = DataValidator::new(ValidationConfig::default());
    let dir = tempfile::tempdir().expect("create temp dir");
    let csv_path = dir.path().join("test_data.csv");

    {
        let mut file = std::fs::File::create(&csv_path).expect("create csv");
        writeln!(file, "timestamp,price,volume").expect("write header");
        // Write enough rows to exceed min_bars_required (100 records at ~100 bytes each)
        for i in 0..200 {
            writeln!(
                file,
                "2024-01-01T{:02}:{:02}:00Z,{:.2},{}",
                i % 24,
                i % 60,
                100.0 + i as f64 * 0.1,
                1000 + i
            )
            .expect("write row");
        }
    }

    let result = validator.validate(&csv_path).await;
    assert!(result.is_ok(), "validate should succeed for valid CSV");
    let validation = result.expect("checked above");
    assert!(validation.is_valid, "non-empty CSV with recognized extension should be valid");
    assert!(validation.records_count > 0, "should have estimated records");
    assert!(
        validation.quality_score > 0.0,
        "quality score should be positive"
    );
}

/// A file without a recognized extension should still validate (not crash)
/// but should report an extension issue.
#[tokio::test]
async fn test_validate_unrecognized_extension_reports_issue() {
    let validator = DataValidator::new(ValidationConfig::default());
    let dir = tempfile::tempdir().expect("create temp dir");
    let path = dir.path().join("data.xyz");

    // Write enough bytes so the file is not empty and meets min_bars
    let content = "x".repeat(20_000);
    std::fs::write(&path, &content).expect("write file");

    let result = validator.validate(&path).await;
    assert!(result.is_ok(), "validate should not error");
    let validation = result.expect("checked above");
    assert!(
        validation
            .issues_found
            .iter()
            .any(|i| i.category == "file_extension"),
        "should report unrecognized file extension issue"
    );
}

/// quick_validate on a valid CSV file with recognized extension should return true.
#[tokio::test]
async fn test_quick_validate_valid_csv() {
    let validator = DataValidator::new(ValidationConfig::default());
    let dir = tempfile::tempdir().expect("create temp dir");
    let path = dir.path().join("test.csv");
    std::fs::write(&path, "header\ndata\n").expect("write file");
    let result = validator.quick_validate(&path).await;
    assert!(result.is_ok(), "quick_validate should not error");
    assert!(
        result.expect("checked above"),
        "valid CSV file should pass quick validation"
    );
}

/// quick_validate on a nonexistent file should return Ok(false) or Err.
#[tokio::test]
async fn test_quick_validate_nonexistent_file() {
    let validator = DataValidator::new(ValidationConfig::default());
    let result = validator
        .quick_validate(std::path::Path::new("/nonexistent/path.csv"))
        .await;
    if let Ok(valid) = result {
        assert!(!valid, "nonexistent file should not be valid");
    }
    // Err is also acceptable -- nonexistent file can legitimately error
}

/// quick_validate on an empty file should return false.
#[tokio::test]
async fn test_quick_validate_empty_file() {
    let validator = DataValidator::new(ValidationConfig::default());
    let dir = tempfile::tempdir().expect("create temp dir");
    let path = dir.path().join("empty.csv");
    std::fs::write(&path, "").expect("write empty file");
    let result = validator.quick_validate(&path).await;
    assert!(result.is_ok(), "quick_validate should not error on empty file");
    assert!(
        !result.expect("checked above"),
        "empty file should fail quick validation"
    );
}

/// quick_validate on a file without recognized extension should return false.
#[tokio::test]
async fn test_quick_validate_unrecognized_extension() {
    let validator = DataValidator::new(ValidationConfig::default());
    let dir = tempfile::tempdir().expect("create temp dir");
    let path = dir.path().join("data.txt");
    std::fs::write(&path, "some content\n").expect("write file");
    let result = validator.quick_validate(&path).await;
    assert!(result.is_ok(), "quick_validate should not error");
    assert!(
        !result.expect("checked above"),
        "unrecognized extension should fail quick validation"
    );
}

/// ValidationConfig::default() should have sensible positive values.
#[tokio::test]
async fn test_validation_config_defaults() {
    let config = ValidationConfig::default();
    assert!(
        config.max_price_change_percent > 0.0,
        "max_price_change_percent must be positive"
    );
    assert!(
        config.max_volume_std_dev > 0.0,
        "max_volume_std_dev must be positive"
    );
    assert!(
        config.max_timestamp_gap_seconds > 0,
        "max_timestamp_gap_seconds must be positive"
    );
    assert!(
        config.min_bars_required > 0,
        "min_bars_required must be positive"
    );
}

/// validate on a nonexistent file should return an error.
#[tokio::test]
async fn test_validate_nonexistent_file_returns_error() {
    let validator = DataValidator::new(ValidationConfig::default());
    let result = validator
        .validate(std::path::Path::new("/nonexistent/path.csv"))
        .await;
    assert!(
        result.is_err(),
        "validate on nonexistent file should return an error"
    );
}

/// A small file below the min_bars_required threshold should report an issue
/// but still be considered valid (no Critical issue, just High).
#[tokio::test]
async fn test_validate_small_file_below_min_bars() {
    let config = ValidationConfig {
        min_bars_required: 1000,
        ..ValidationConfig::default()
    };
    let validator = DataValidator::new(config);
    let dir = tempfile::tempdir().expect("create temp dir");
    let path = dir.path().join("small.csv");

    // Write a small amount of data (~50 bytes, well below 1000 * 100)
    std::fs::write(&path, "timestamp,price,volume\n1,2,3\n").expect("write");

    let result = validator.validate(&path).await;
    assert!(result.is_ok(), "validate should not error");
    let validation = result.expect("checked above");
    assert!(
        validation
            .issues_found
            .iter()
            .any(|i| i.category == "record_count"),
        "should report insufficient record count issue"
    );
}

/// Validate that quality_score degrades as more issues are found.
#[tokio::test]
async fn test_quality_score_degrades_with_issues() {
    let config = ValidationConfig {
        min_bars_required: 100_000, // Very high threshold to trigger record_count issue
        ..ValidationConfig::default()
    };
    let validator = DataValidator::new(config);
    let dir = tempfile::tempdir().expect("create temp dir");
    // Use unrecognized extension AND small file to trigger multiple issues
    let path = dir.path().join("data.xyz");
    std::fs::write(&path, "small content").expect("write");

    let result = validator.validate(&path).await;
    assert!(result.is_ok(), "validate should not error");
    let validation = result.expect("checked above");
    assert!(
        validation.issues_found.len() >= 2,
        "should have at least 2 issues (extension + record count)"
    );
    assert!(
        validation.quality_score < 1.0,
        "quality score should be below 1.0 when issues exist"
    );
}